diff --git a/ci/jenkinsfile/cluster_deploy2dev.groovy b/ci/jenkinsfile/cluster_deploy2dev.groovy index cde361b9ea..c96e16ffa9 100644 --- a/ci/jenkinsfile/cluster_deploy2dev.groovy +++ b/ci/jenkinsfile/cluster_deploy2dev.groovy @@ -5,15 +5,17 @@ try { dir ("milvus-helm") { checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:megasearch/milvus-helm.git", name: 'origin', refspec: "+refs/heads/${SEMVER}:refs/remotes/origin/${SEMVER}"]]]) dir ("milvus/milvus-cluster") { - sh "helm install --set roServers.image.tag=${DOCKER_VERSION} --set woServers.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP -f ci/values.yaml --name ${env.JOB_NAME}-${env.BUILD_NUMBER}-cluster --namespace milvus-cluster --version 0.3.1 . " + sh "helm install --wait --timeout 300 --set roServers.image.tag=${DOCKER_VERSION} --set woServers.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP -f ci/values.yaml --name ${env.JOB_NAME}-${env.BUILD_NUMBER}-cluster --namespace milvus-cluster --version 0.3.1 . " } } + /* timeout(time: 2, unit: 'MINUTES') { waitUntil { def result = sh script: "nc -z -w 3 ${env.JOB_NAME}-${env.BUILD_NUMBER}-cluster-milvus-cluster-proxy.milvus-cluster.svc.cluster.local 19530", returnStatus: true return !result } } + */ } catch (exc) { echo 'Helm running failed!' sh "helm del --purge ${env.JOB_NAME}-${env.BUILD_NUMBER}-cluster" diff --git a/ci/jenkinsfile/cluster_dev_test.groovy b/ci/jenkinsfile/cluster_dev_test.groovy index ad7b838e4f..2d8854ca71 100644 --- a/ci/jenkinsfile/cluster_dev_test.groovy +++ b/ci/jenkinsfile/cluster_dev_test.groovy @@ -2,7 +2,7 @@ timeout(time: 10, unit: 'MINUTES') { try { dir ("${PROJECT_NAME}_test") { checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:Test/milvus_test.git", name: 'origin', refspec: "+refs/heads/${SEMVER}:refs/remotes/origin/${SEMVER}"]]]) - sh 'python3 -m pip install -r requirements.txt' + sh 'python3 -m pip install -r requirements_cluster.txt' sh "pytest . --alluredir=cluster_test_out --ip ${env.JOB_NAME}-${env.BUILD_NUMBER}-cluster-milvus-cluster-proxy.milvus-cluster.svc.cluster.local" } } catch (exc) { diff --git a/ci/jenkinsfile/deploy2dev.groovy b/ci/jenkinsfile/deploy2dev.groovy index b0c537f2e1..52cbae2bfe 100644 --- a/ci/jenkinsfile/deploy2dev.groovy +++ b/ci/jenkinsfile/deploy2dev.groovy @@ -5,15 +5,17 @@ try { dir ("milvus-helm") { checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:megasearch/milvus-helm.git", name: 'origin', refspec: "+refs/heads/${SEMVER}:refs/remotes/origin/${SEMVER}"]]]) dir ("milvus/milvus-gpu") { - sh "helm install --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.JOB_NAME}-${env.BUILD_NUMBER} --version 0.3.1 ." + sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.JOB_NAME}-${env.BUILD_NUMBER} -f ci/values.yaml --namespace milvus-1 --version 0.3.1 ." } } + /* timeout(time: 2, unit: 'MINUTES') { waitUntil { def result = sh script: "nc -z -w 3 ${env.JOB_NAME}-${env.BUILD_NUMBER}-milvus-gpu-engine.kube-opt.svc.cluster.local 19530", returnStatus: true return !result } } + */ } catch (exc) { echo 'Helm running failed!' sh "helm del --purge ${env.JOB_NAME}-${env.BUILD_NUMBER}" diff --git a/ci/jenkinsfile/dev_test.groovy b/ci/jenkinsfile/dev_test.groovy index 836ddbad05..fb5b4749cc 100644 --- a/ci/jenkinsfile/dev_test.groovy +++ b/ci/jenkinsfile/dev_test.groovy @@ -1,10 +1,27 @@ -timeout(time: 10, unit: 'MINUTES') { +timeout(time: 20, unit: 'MINUTES') { try { dir ("${PROJECT_NAME}_test") { checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:Test/milvus_test.git", name: 'origin', refspec: "+refs/heads/${SEMVER}:refs/remotes/origin/${SEMVER}"]]]) sh 'python3 -m pip install -r requirements.txt' sh "pytest . --alluredir=test_out --ip ${env.JOB_NAME}-${env.BUILD_NUMBER}-milvus-gpu-engine.kube-opt.svc.cluster.local" } + + // mysql database backend test + load "${env.WORKSPACE}/ci/jenkinsfile/cleanup_dev.groovy" + + if (!fileExists('milvus-helm')) { + dir ("milvus-helm") { + checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:megasearch/milvus-helm.git", name: 'origin', refspec: "+refs/heads/${SEMVER}:refs/remotes/origin/${SEMVER}"]]]) + } + } + dir ("milvus-helm") { + dir ("milvus/milvus-gpu") { + sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.JOB_NAME}-${env.BUILD_NUMBER} -f ci/db_backend/mysql_values.yaml --namespace milvus-2 --version 0.3.1 ." + } + } + dir ("${PROJECT_NAME}_test") { + sh "pytest . --alluredir=test_out --ip ${env.JOB_NAME}-${env.BUILD_NUMBER}-milvus-gpu-engine.kube-opt.svc.cluster.local" + } } catch (exc) { echo 'Milvus Test Failed !' throw exc diff --git a/ci/main_jenkinsfile b/ci/main_jenkinsfile index 9577837bfd..d4132e9ff1 100644 --- a/ci/main_jenkinsfile +++ b/ci/main_jenkinsfile @@ -35,7 +35,7 @@ pipeline { defaultContainer 'jnlp' containerTemplate { name 'milvus-build-env' - image 'registry.zilliz.com/milvus/milvus-build-env:v0.10' + image 'registry.zilliz.com/milvus/milvus-build-env:v0.11' ttyEnabled true command 'cat' } diff --git a/ci/main_jenkinsfile_no_ut b/ci/main_jenkinsfile_no_ut index 082433650c..0b2f90fd63 100644 --- a/ci/main_jenkinsfile_no_ut +++ b/ci/main_jenkinsfile_no_ut @@ -35,7 +35,7 @@ pipeline { defaultContainer 'jnlp' containerTemplate { name 'milvus-build-env' - image 'registry.zilliz.com/milvus/milvus-build-env:v0.10' + image 'registry.zilliz.com/milvus/milvus-build-env:v0.11' ttyEnabled true command 'cat' } diff --git a/ci/nightly_main_jenkinsfile b/ci/nightly_main_jenkinsfile index 7f59561f82..5458cf7632 100644 --- a/ci/nightly_main_jenkinsfile +++ b/ci/nightly_main_jenkinsfile @@ -35,7 +35,7 @@ pipeline { defaultContainer 'jnlp' containerTemplate { name 'milvus-build-env' - image 'registry.zilliz.com/milvus/milvus-build-env:v0.10' + image 'registry.zilliz.com/milvus/milvus-build-env:v0.11' ttyEnabled true command 'cat' } diff --git a/cpp/CHANGELOG.md b/cpp/CHANGELOG.md index 3ac9db25a1..77cbe055e3 100644 --- a/cpp/CHANGELOG.md +++ b/cpp/CHANGELOG.md @@ -17,6 +17,10 @@ Please mark all change in change log and use the ticket from JIRA. - MS-232 - Add MySQLMetaImpl::UpdateTableFilesToIndex and set maximum_memory to default if config value = 0 - MS-233 - Remove mem manager log - MS-230 - Change parameter name: Maximum_memory to insert_buffer_size +- MS-234 - Some case cause background merge thread stop +- MS-235 - Some test cases random fail +- MS-236 - Add MySQLMetaImpl::HasNonIndexFiles +- MS-257 - Update bzip2 download url ## Improvement - MS-156 - Add unittest for merge result functions @@ -25,6 +29,13 @@ Please mark all change in change log and use the ticket from JIRA. - MS-206 - Support SQ8 index type - MS-208 - Add buildinde interface for C++ SDK - MS-212 - Support Inner product metric type +- MS-241 - Build Faiss with MKL if using Intel CPU; else build with OpenBlas +- MS-242 - Clean up cmake and change MAKE_BUILD_ARGS to be user defined variable +- MS-245 - Improve search result transfer performance +- MS-248 - Support AddVector/SearchVector profiling +- MS-256 - Add more cache config +- MS-260 - Refine log +- MS-261 - update faiss version to 1.5.3 and add BUILD_FAISS_WITH_MKL as an option ## New Feature - MS-180 - Add new mem manager diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 3a0e8900a6..07b4719790 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -116,6 +116,11 @@ set(MILVUS_ENGINE_SRC ${PROJECT_SOURCE_DIR}/src) add_compile_definitions(PROFILER=${PROFILER}) +message("MILVUS_ENABLE_PROFILING = ${MILVUS_ENABLE_PROFILING}") +if (MILVUS_ENABLE_PROFILING STREQUAL "ON") + ADD_DEFINITIONS(-DMILVUS_ENABLE_PROFILING) +endif() + include_directories(${MILVUS_ENGINE_INCLUDE}) include_directories(${MILVUS_ENGINE_SRC}) diff --git a/cpp/README.md b/cpp/README.md index 1b2f507db2..b46515a1cf 100644 --- a/cpp/README.md +++ b/cpp/README.md @@ -1,13 +1,13 @@ ### Compilation #### Step 1: install necessery tools - Install MySQL - centos7 : - yum install gfortran qt4 flex bison mysql-devel + yum install gfortran qt4 flex bison mysql-devel mysql ubuntu16.04 : - sudo apt-get install gfortran qt4-qmake flex bison libmysqlclient-dev + sudo apt-get install gfortran qt4-qmake flex bison libmysqlclient-dev mysql-client + + cd scripts && sudo ./requirements.sh If `libmysqlclient_r.so` does not exist after installing MySQL Development Files, you need to create a symbolic link: diff --git a/cpp/build.sh b/cpp/build.sh index 80be1d7ddb..edfe9305be 100755 --- a/cpp/build.sh +++ b/cpp/build.sh @@ -7,8 +7,10 @@ INSTALL_PREFIX=$(pwd)/milvus MAKE_CLEAN="OFF" BUILD_COVERAGE="OFF" DB_PATH="/opt/milvus" +PROFILING="OFF" +BUILD_FAISS_WITH_MKL="OFF" -while getopts "p:d:t:uhlrc" arg +while getopts "p:d:t:uhlrcgm" arg do case $arg in t) @@ -36,6 +38,12 @@ do c) BUILD_COVERAGE="ON" ;; + g) + PROFILING="ON" + ;; + m) + BUILD_FAISS_WITH_MKL="ON" + ;; h) # help echo " @@ -47,9 +55,11 @@ parameter: -l: build license version(default: OFF) -r: remove previous build directory(default: OFF) -c: code coverage(default: OFF) +-g: profiling(default: OFF) +-m: build faiss with MKL(default: OFF) usage: -./build.sh -t \${BUILD_TYPE} [-u] [-h] [-g] [-r] [-c] +./build.sh -t \${BUILD_TYPE} [-u] [-h] [-g] [-r] [-c] [-m] " exit 0 ;; @@ -77,6 +87,8 @@ if [[ ${MAKE_CLEAN} == "ON" ]]; then -DCMAKE_LICENSE_CHECK=${LICENSE_CHECK} \ -DBUILD_COVERAGE=${BUILD_COVERAGE} \ -DMILVUS_DB_PATH=${DB_PATH} \ + -DMILVUS_ENABLE_PROFILING=${PROFILING} \ + -DBUILD_FAISS_WITH_MKL=${BUILD_FAISS_WITH_MKL} \ $@ ../" echo ${CMAKE_CMD} diff --git a/cpp/cmake/DefineOptions.cmake b/cpp/cmake/DefineOptions.cmake index 147663d0db..cc9792391d 100644 --- a/cpp/cmake/DefineOptions.cmake +++ b/cpp/cmake/DefineOptions.cmake @@ -57,8 +57,6 @@ define_option(MILVUS_VERBOSE_THIRDPARTY_BUILD define_option(MILVUS_WITH_ARROW "Build with ARROW" OFF) -define_option(MILVUS_BOOST_USE_SHARED "Rely on boost shared libraries where relevant" OFF) - define_option(MILVUS_BOOST_VENDORED "Use vendored Boost instead of existing Boost. \ Note that this requires linking Boost statically" ON) @@ -111,6 +109,11 @@ define_option(MILVUS_WITH_ZSTD "Build with zstd compression" ${MILVUS_WITH_ZSTD_ define_option(MILVUS_WITH_AWS "Build with AWS SDK" ON) +if (MILVUS_ENABLE_PROFILING STREQUAL "ON") + define_option(MILVUS_WITH_LIBUNWIND "Build with libunwind" ON) + define_option(MILVUS_WITH_GPERFTOOLS "Build with gperftools" ON) +endif() + #---------------------------------------------------------------------- if(MSVC) set_option_category("MSVC") diff --git a/cpp/cmake/ThirdPartyPackages.cmake b/cpp/cmake/ThirdPartyPackages.cmake index 9e19b037f8..fed5dece88 100644 --- a/cpp/cmake/ThirdPartyPackages.cmake +++ b/cpp/cmake/ThirdPartyPackages.cmake @@ -5,7 +5,6 @@ # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at -# # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, @@ -37,7 +36,9 @@ set(MILVUS_THIRDPARTY_DEPENDENCIES yaml-cpp ZLIB ZSTD - AWS) + AWS + libunwind + gperftools) message(STATUS "Using ${MILVUS_DEPENDENCY_SOURCE} approach to find dependencies") @@ -89,6 +90,10 @@ macro(build_dependency DEPENDENCY_NAME) build_zstd() elseif("${DEPENDENCY_NAME}" STREQUAL "AWS") build_aws() + elseif("${DEPENDENCY_NAME}" STREQUAL "libunwind") + build_libunwind() + elseif("${DEPENDENCY_NAME}" STREQUAL "gperftools") + build_gperftools() else() message(FATAL_ERROR "Unknown thirdparty dependency to build: ${DEPENDENCY_NAME}") endif () @@ -96,12 +101,8 @@ endmacro() macro(resolve_dependency DEPENDENCY_NAME) if (${DEPENDENCY_NAME}_SOURCE STREQUAL "AUTO") - #message(STATUS "Finding ${DEPENDENCY_NAME} package") -# find_package(${DEPENDENCY_NAME} QUIET) -# if (NOT ${DEPENDENCY_NAME}_FOUND) - #message(STATUS "${DEPENDENCY_NAME} package not found") + #disable find_package for now build_dependency(${DEPENDENCY_NAME}) -# endif () elseif (${DEPENDENCY_NAME}_SOURCE STREQUAL "BUNDLED") build_dependency(${DEPENDENCY_NAME}) elseif (${DEPENDENCY_NAME}_SOURCE STREQUAL "SYSTEM") @@ -117,11 +118,9 @@ string(TOUPPER ${CMAKE_BUILD_TYPE} UPPERCASE_BUILD_TYPE) set(EP_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${CMAKE_CXX_FLAGS_${UPPERCASE_BUILD_TYPE}}") set(EP_C_FLAGS "${CMAKE_C_FLAGS} ${CMAKE_C_FLAGS_${UPPERCASE_BUILD_TYPE}}") -if(NOT MSVC) - # Set -fPIC on all external projects - set(EP_CXX_FLAGS "${EP_CXX_FLAGS} -fPIC") - set(EP_C_FLAGS "${EP_C_FLAGS} -fPIC") -endif() +# Set -fPIC on all external projects +set(EP_CXX_FLAGS "${EP_CXX_FLAGS} -fPIC") +set(EP_C_FLAGS "${EP_C_FLAGS} -fPIC") # CC/CXX environment variables are captured on the first invocation of the # builder (e.g make or ninja) instead of when CMake is invoked into to build @@ -159,20 +158,13 @@ endif() # Ensure that a default make is set if("${MAKE}" STREQUAL "") - if(NOT MSVC) - find_program(MAKE make) - endif() + find_program(MAKE make) endif() -set(MAKE_BUILD_ARGS "-j2") - -## Using make -j in sub-make is fragile -#if(${CMAKE_GENERATOR} MATCHES "Makefiles") -# set(MAKE_BUILD_ARGS "") -#else() -# # limit the maximum number of jobs for ninja -# set(MAKE_BUILD_ARGS "-j4") -#endif() +if (NOT DEFINED MAKE_BUILD_ARGS) + set(MAKE_BUILD_ARGS "-j8") +endif() +message(STATUS "Third Party MAKE_BUILD_ARGS = ${MAKE_BUILD_ARGS}") # ---------------------------------------------------------------------- # Find pthreads @@ -227,7 +219,7 @@ endif() if(DEFINED ENV{MILVUS_BZIP2_URL}) set(BZIP2_SOURCE_URL "$ENV{MILVUS_BZIP2_URL}") else() - set(BZIP2_SOURCE_URL "https://fossies.org/linux/misc/bzip2-${BZIP2_VERSION}.tar.gz") + set(BZIP2_SOURCE_URL "https://sourceware.org/pub/bzip2/bzip2-${BZIP2_VERSION}.tar.gz") endif() if(DEFINED ENV{MILVUS_EASYLOGGINGPP_URL}) @@ -285,7 +277,6 @@ if (DEFINED ENV{MILVUS_PROMETHEUS_URL}) set(PROMETHEUS_SOURCE_URL "$ENV{PROMETHEUS_OPENBLAS_URL}") else () set(PROMETHEUS_SOURCE_URL - #"https://github.com/JinHai-CN/prometheus-cpp/archive/${PROMETHEUS_VERSION}.tar.gz" https://github.com/jupp0r/prometheus-cpp.git) endif() @@ -347,6 +338,21 @@ if(DEFINED ENV{MILVUS_AWS_URL}) else() set(AWS_SOURCE_URL "https://github.com/aws/aws-sdk-cpp/archive/${AWS_VERSION}.tar.gz") endif() + +if(DEFINED ENV{MILVUS_LIBUNWIND_URL}) + set(LIBUNWIND_SOURCE_URL "$ENV{MILVUS_LIBUNWIND_URL}") +else() + set(LIBUNWIND_SOURCE_URL + "https://github.com/libunwind/libunwind/releases/download/v${LIBUNWIND_VERSION}/libunwind-${LIBUNWIND_VERSION}.tar.gz") +endif() + +if(DEFINED ENV{MILVUS_GPERFTOOLS_URL}) + set(GPERFTOOLS_SOURCE_URL "$ENV{MILVUS_GPERFTOOLS_URL}") +else() + set(GPERFTOOLS_SOURCE_URL + "https://github.com/gperftools/gperftools/releases/download/gperftools-${GPERFTOOLS_VERSION}/gperftools-${GPERFTOOLS_VERSION}.tar.gz") +endif() + # ---------------------------------------------------------------------- # ARROW @@ -354,19 +360,13 @@ macro(build_arrow) message(STATUS "Building Apache ARROW-${ARROW_VERSION} from source") set(ARROW_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/arrow_ep-prefix/src/arrow_ep/cpp") set(ARROW_STATIC_LIB_NAME arrow) -# set(ARROW_CUDA_STATIC_LIB_NAME arrow_cuda) + set(ARROW_STATIC_LIB "${ARROW_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}${ARROW_STATIC_LIB_NAME}${CMAKE_STATIC_LIBRARY_SUFFIX}" ) -# set(ARROW_CUDA_STATIC_LIB -# "${ARROW_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}${ARROW_CUDA_STATIC_LIB_NAME}${CMAKE_STATIC_LIBRARY_SUFFIX}" -# ) set(ARROW_INCLUDE_DIR "${ARROW_PREFIX}/include") - set(ARROW_CMAKE_ARGS ${EP_COMMON_CMAKE_ARGS} -# "-DARROW_THRIFT_URL=${THRIFT_SOURCE_URL}" - #"env ARROW_THRIFT_URL=${THRIFT_SOURCE_URL}" -DARROW_BUILD_STATIC=ON -DARROW_BUILD_SHARED=OFF -DARROW_PARQUET=ON @@ -375,8 +375,6 @@ macro(build_arrow) "-DCMAKE_LIBRARY_PATH=${CUDA_TOOLKIT_ROOT_DIR}/lib64/stubs" -DCMAKE_BUILD_TYPE=Release) -# set($ENV{ARROW_THRIFT_URL} ${THRIFT_SOURCE_URL}) - externalproject_add(arrow_ep GIT_REPOSITORY ${ARROW_SOURCE_URL} @@ -384,14 +382,8 @@ macro(build_arrow) ${ARROW_VERSION} GIT_SHALLOW TRUE -# SOURCE_DIR -# ${ARROW_PREFIX} -# BINARY_DIR -# ${ARROW_PREFIX} SOURCE_SUBDIR cpp -# COMMAND -# "export \"ARROW_THRIFT_URL=${THRIFT_SOURCE_URL}\"" ${EP_LOG_OPTIONS} CMAKE_ARGS ${ARROW_CMAKE_ARGS} @@ -400,21 +392,16 @@ macro(build_arrow) ${MAKE_BUILD_ARGS} INSTALL_COMMAND ${MAKE} install -# BUILD_IN_SOURCE -# 1 BUILD_BYPRODUCTS "${ARROW_STATIC_LIB}" -# "${ARROW_CUDA_STATIC_LIB}" ) -# ExternalProject_Add_StepDependencies(arrow_ep build thrift_ep) - file(MAKE_DIRECTORY "${ARROW_PREFIX}/include") add_library(arrow STATIC IMPORTED) set_target_properties(arrow PROPERTIES IMPORTED_LOCATION "${ARROW_STATIC_LIB}" INTERFACE_INCLUDE_DIRECTORIES "${ARROW_INCLUDE_DIR}") -# INTERFACE_LINK_LIBRARIES thrift) + add_dependencies(arrow arrow_ep) set(JEMALLOC_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/arrow_ep-prefix/src/arrow_ep-build/jemalloc_ep-prefix/src/jemalloc_ep") @@ -438,9 +425,6 @@ endif() # Add Boost dependencies (code adapted from Apache Kudu (incubating)) set(Boost_USE_MULTITHREADED ON) -if(MSVC AND MILVUS_USE_STATIC_CRT) - set(Boost_USE_STATIC_RUNTIME ON) -endif() set(Boost_ADDITIONAL_VERSIONS "1.70.0" "1.70" @@ -530,59 +514,8 @@ if(MILVUS_BOOST_VENDORED) add_dependencies(boost_filesystem_static boost_ep) add_dependencies(boost_serialization_static boost_ep) -else() - if(MSVC) - # disable autolinking in boost - add_definitions(-DBOOST_ALL_NO_LIB) - endif() - -# if(DEFINED ENV{BOOST_ROOT} OR DEFINED BOOST_ROOT) -# # In older versions of CMake (such as 3.2), the system paths for Boost will -# # be looked in first even if we set $BOOST_ROOT or pass -DBOOST_ROOT -# set(Boost_NO_SYSTEM_PATHS ON) -# endif() - - if(MILVUS_BOOST_USE_SHARED) - # Find shared Boost libraries. - set(Boost_USE_STATIC_LIBS OFF) - set(BUILD_SHARED_LIBS_KEEP ${BUILD_SHARED_LIBS}) - set(BUILD_SHARED_LIBS ON) - - if(MSVC) - # force all boost libraries to dynamic link - add_definitions(-DBOOST_ALL_DYN_LINK) - endif() - - if(MILVUS_BOOST_HEADER_ONLY) - find_package(Boost REQUIRED) - else() - find_package(Boost COMPONENTS serialization system filesystem REQUIRED) - set(BOOST_SYSTEM_LIBRARY Boost::system) - set(BOOST_FILESYSTEM_LIBRARY Boost::filesystem) - set(BOOST_SERIALIZATION_LIBRARY Boost::serialization) - set(MILVUS_BOOST_LIBS ${BOOST_SYSTEM_LIBRARY} ${BOOST_FILESYSTEM_LIBRARY}) - endif() - set(BUILD_SHARED_LIBS ${BUILD_SHARED_LIBS_KEEP}) - unset(BUILD_SHARED_LIBS_KEEP) - else() - # Find static boost headers and libs - # TODO Differentiate here between release and debug builds - set(Boost_USE_STATIC_LIBS ON) - if(MILVUS_BOOST_HEADER_ONLY) - find_package(Boost REQUIRED) - else() - find_package(Boost COMPONENTS serialization system filesystem REQUIRED) - set(BOOST_SYSTEM_LIBRARY Boost::system) - set(BOOST_FILESYSTEM_LIBRARY Boost::filesystem) - set(BOOST_SERIALIZATION_LIBRARY Boost::serialization) - set(MILVUS_BOOST_LIBS ${BOOST_SYSTEM_LIBRARY} ${BOOST_FILESYSTEM_LIBRARY}) - endif() - endif() endif() -#message(STATUS "Boost include dir: " ${Boost_INCLUDE_DIR}) -#message(STATUS "Boost libraries: " ${Boost_LIBRARIES}) - include_directories(SYSTEM ${Boost_INCLUDE_DIR}) link_directories(SYSTEM ${BOOST_LIB_DIR}) @@ -726,13 +659,6 @@ macro(build_openblas) add_dependencies(openblas openblas_ep) endmacro() -#if(MILVUS_WITH_OPENBLAS) -# resolve_dependency(OpenBLAS) -# -# get_target_property(OPENBLAS_INCLUDE_DIR openblas INTERFACE_INCLUDE_DIRECTORIES) -# include_directories(SYSTEM "${OPENBLAS_INCLUDE_DIR}") -#endif() - # ---------------------------------------------------------------------- # LAPACK @@ -770,16 +696,25 @@ macro(build_lapack) add_dependencies(lapack lapack_ep) endmacro() -#if(MILVUS_WITH_LAPACK) -# resolve_dependency(LAPACK) -# -# get_target_property(LAPACK_INCLUDE_DIR lapack INTERFACE_INCLUDE_DIRECTORIES) -# include_directories(SYSTEM "${LAPACK_INCLUDE_DIR}") -#endif() - # ---------------------------------------------------------------------- # FAISS +if(NOT DEFINED BUILD_FAISS_WITH_MKL) + set(BUILD_FAISS_WITH_MKL OFF) +endif() + +if(EXISTS "/proc/cpuinfo") + FILE(READ /proc/cpuinfo PROC_CPUINFO) + + SET(VENDOR_ID_RX "vendor_id[ \t]*:[ \t]*([a-zA-Z]+)\n") + STRING(REGEX MATCH "${VENDOR_ID_RX}" VENDOR_ID "${PROC_CPUINFO}") + STRING(REGEX REPLACE "${VENDOR_ID_RX}" "\\1" VENDOR_ID "${VENDOR_ID}") + + if(NOT ${VENDOR_ID} STREQUAL "GenuineIntel") + set(BUILD_FAISS_WITH_MKL OFF) + endif() +endif() + macro(build_faiss) message(STATUS "Building FAISS-${FAISS_VERSION} from source") set(FAISS_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/faiss_ep-prefix/src/faiss_ep") @@ -787,37 +722,40 @@ macro(build_faiss) set(FAISS_STATIC_LIB "${FAISS_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}faiss${CMAKE_STATIC_LIBRARY_SUFFIX}") -# add_custom_target(faiss_dependencies) -# add_dependencies(faiss_dependencies openblas_ep) -# add_dependencies(faiss_dependencies openblas) -# get_target_property(FAISS_OPENBLAS_LIB_DIR openblas IMPORTED_LOCATION) -# get_filename_component(FAISS_OPENBLAS_LIB "${FAISS_OPENBLAS_LIB_DIR}" DIRECTORY) - set(FAISS_CONFIGURE_ARGS "--prefix=${FAISS_PREFIX}" "CFLAGS=${EP_C_FLAGS}" "CXXFLAGS=${EP_CXX_FLAGS}" - "LDFLAGS=-L${OPENBLAS_PREFIX}/lib -L${LAPACK_PREFIX}/lib -lopenblas -llapack" --without-python) -# if(OPENBLAS_STATIC_LIB) -# set(OPENBLAS_LIBRARY ${OPENBLAS_STATIC_LIB}) -# else() -# set(OPENBLAS_LIBRARY ${OPENBLAS_SHARED_LIB}) -# endif() -# set(FAISS_DEPENDENCIES ${FAISS_DEPENDENCIES} ${OPENBLAS_LIBRARY}) + set(FAISS_CFLAGS ${EP_C_FLAGS}) + set(FAISS_CXXFLAGS ${EP_CXX_FLAGS}) + + if(${BUILD_FAISS_WITH_MKL} STREQUAL "ON") + message(STATUS "Build Faiss with MKL") + if(NOT DEFINED MKL_LIB_PATH) + set(MKL_LIB_PATH "/opt/intel/compilers_and_libraries_${MKL_VERSION}/linux/mkl/lib/intel64") + message(STATUS "MKL_LIB_PATH = ${MKL_LIB_PATH}") + endif() + + set(FAISS_CONFIGURE_ARGS ${FAISS_CONFIGURE_ARGS} + "CPPFLAGS=-DFINTEGER=long -DMKL_ILP64 -m64 -I${MKL_LIB_PATH}/../../include" + "LDFLAGS=-L${MKL_LIB_PATH}" + "LIBS=-Wl,--start-group ${MKL_LIB_PATH}/libmkl_intel_ilp64.a ${MKL_LIB_PATH}/libmkl_gnu_thread.a ${MKL_LIB_PATH}/libmkl_core.a -Wl,--end-group -lgomp -lpthread -lm -ldl") + + else() + message(STATUS "Build Faiss with OpenBlas/LAPACK") + set(FAISS_CONFIGURE_ARGS ${FAISS_CONFIGURE_ARGS} + "LDFLAGS=-L${OPENBLAS_PREFIX}/lib -L${LAPACK_PREFIX}/lib") + endif() if(${MILVUS_WITH_FAISS_GPU_VERSION} STREQUAL "ON") set(FAISS_CONFIGURE_ARGS ${FAISS_CONFIGURE_ARGS} "--with-cuda=${CUDA_TOOLKIT_ROOT_DIR}" -# "with_cuda_arch=\"-gencode=arch=compute_35,code=compute_35 \\ -# -gencode=arch=compute_52,code=compute_52 \\ -# -gencode=arch=compute_60,code=compute_60 \\ -# -gencode=arch=compute_61,code=compute_61\"" - "--with-cuda-arch=\"-gencode=arch=compute_35,code=compute_35\"" - "--with-cuda-arch=\"-gencode=arch=compute_52,code=compute_52\"" - "--with-cuda-arch=\"-gencode=arch=compute_60,code=compute_60\"" - "--with-cuda-arch=\"-gencode=arch=compute_61,code=compute_61\"" + "--with-cuda-arch=\"-gencode=arch=compute_35,code=sm_35\"" + "--with-cuda-arch=\"-gencode=arch=compute_52,code=sm_52\"" + "--with-cuda-arch=\"-gencode=arch=compute_60,code=sm_60\"" + "--with-cuda-arch=\"-gencode=arch=compute_61,code=sm_61\"" ) else() set(FAISS_CONFIGURE_ARGS ${FAISS_CONFIGURE_ARGS} --without-cuda) @@ -830,66 +768,67 @@ macro(build_faiss) CONFIGURE_COMMAND "./configure" ${FAISS_CONFIGURE_ARGS} -# BINARY_DIR -# ${FAISS_PREFIX} -# INSTALL_DIR -# ${FAISS_PREFIX} -# BUILD_COMMAND -# ${MAKE} ${MAKE_BUILD_ARGS} BUILD_COMMAND - ${MAKE} ${MAKE_BUILD_ARGS} all - COMMAND - cd gpu && ${MAKE} ${MAKE_BUILD_ARGS} + ${MAKE} ${MAKE_BUILD_ARGS} BUILD_IN_SOURCE 1 -# INSTALL_DIR -# ${FAISS_PREFIX} INSTALL_COMMAND ${MAKE} install - COMMAND - ln -s faiss_ep ../faiss BUILD_BYPRODUCTS ${FAISS_STATIC_LIB}) -# DEPENDS -# ${faiss_dependencies}) - - ExternalProject_Add_StepDependencies(faiss_ep build openblas_ep lapack_ep) + + if(${BUILD_FAISS_WITH_MKL} STREQUAL "OFF") + ExternalProject_Add_StepDependencies(faiss_ep build openblas_ep lapack_ep) + endif() file(MAKE_DIRECTORY "${FAISS_INCLUDE_DIR}") - add_library(faiss STATIC IMPORTED) - set_target_properties( - faiss - PROPERTIES IMPORTED_LOCATION "${FAISS_STATIC_LIB}" - INTERFACE_INCLUDE_DIRECTORIES "${FAISS_INCLUDE_DIR}" - INTERFACE_LINK_LIBRARIES "openblas;lapack" ) + add_library(faiss SHARED IMPORTED) + if(${BUILD_FAISS_WITH_MKL} STREQUAL "ON") + set(MKL_LIBS ${MKL_LIB_PATH}/libmkl_intel_ilp64.a + ${MKL_LIB_PATH}/libmkl_gnu_thread.a + ${MKL_LIB_PATH}/libmkl_core.a) + + set_target_properties( + faiss + PROPERTIES IMPORTED_LOCATION "${FAISS_STATIC_LIB}" + INTERFACE_INCLUDE_DIRECTORIES "${FAISS_INCLUDE_DIR}" + INTERFACE_LINK_LIBRARIES "${MKL_LIBS}" ) + else() + set_target_properties( + faiss + PROPERTIES IMPORTED_LOCATION "${FAISS_STATIC_LIB}" + INTERFACE_INCLUDE_DIRECTORIES "${FAISS_INCLUDE_DIR}" + INTERFACE_LINK_LIBRARIES "openblas;lapack" ) + endif() + add_dependencies(faiss faiss_ep) - #add_dependencies(faiss openblas_ep) - #add_dependencies(faiss lapack_ep) - #target_link_libraries(faiss ${OPENBLAS_PREFIX}/lib) - #target_link_libraries(faiss ${LAPACK_PREFIX}/lib) + + if(${BUILD_FAISS_WITH_MKL} STREQUAL "OFF") + add_dependencies(faiss openblas_ep) + add_dependencies(faiss lapack_ep) + endif() endmacro() if(MILVUS_WITH_FAISS) - resolve_dependency(OpenBLAS) - get_target_property(OPENBLAS_INCLUDE_DIR openblas INTERFACE_INCLUDE_DIRECTORIES) - include_directories(SYSTEM "${OPENBLAS_INCLUDE_DIR}") - link_directories(SYSTEM ${OPENBLAS_PREFIX}/lib) + if(${BUILD_FAISS_WITH_MKL} STREQUAL "OFF") + resolve_dependency(OpenBLAS) + get_target_property(OPENBLAS_INCLUDE_DIR openblas INTERFACE_INCLUDE_DIRECTORIES) + include_directories(SYSTEM "${OPENBLAS_INCLUDE_DIR}") + link_directories(SYSTEM ${OPENBLAS_PREFIX}/lib) - resolve_dependency(LAPACK) - get_target_property(LAPACK_INCLUDE_DIR lapack INTERFACE_INCLUDE_DIRECTORIES) - include_directories(SYSTEM "${LAPACK_INCLUDE_DIR}") - link_directories(SYSTEM "${LAPACK_PREFIX}/lib") + resolve_dependency(LAPACK) + get_target_property(LAPACK_INCLUDE_DIR lapack INTERFACE_INCLUDE_DIRECTORIES) + include_directories(SYSTEM "${LAPACK_INCLUDE_DIR}") + link_directories(SYSTEM "${LAPACK_PREFIX}/lib") + endif() resolve_dependency(FAISS) get_target_property(FAISS_INCLUDE_DIR faiss INTERFACE_INCLUDE_DIRECTORIES) include_directories(SYSTEM "${FAISS_INCLUDE_DIR}") - include_directories(SYSTEM "${CMAKE_CURRENT_BINARY_DIR}/faiss_ep-prefix/src/") - link_directories(SYSTEM ${FAISS_PREFIX}/) link_directories(SYSTEM ${FAISS_PREFIX}/lib/) - link_directories(SYSTEM ${FAISS_PREFIX}/gpu/) endif() # ---------------------------------------------------------------------- @@ -926,8 +865,6 @@ macro(build_gtest) set(GMOCK_STATIC_LIB "${GTEST_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}gmock${CMAKE_STATIC_LIBRARY_SUFFIX}" ) - - ExternalProject_Add(googletest_ep URL ${GTEST_SOURCE_URL} @@ -967,13 +904,11 @@ macro(build_gtest) endmacro() if (MILVUS_BUILD_TESTS) - #message(STATUS "Resolving gtest dependency") resolve_dependency(GTest) if(NOT GTEST_VENDORED) endif() - - # TODO: Don't use global includes but rather target_include_directories + get_target_property(GTEST_INCLUDE_DIR gtest INTERFACE_INCLUDE_DIRECTORIES) link_directories(SYSTEM "${GTEST_PREFIX}/lib") include_directories(SYSTEM ${GTEST_INCLUDE_DIR}) @@ -1011,32 +946,8 @@ macro(build_lz4) set(LZ4_BUILD_DIR "${CMAKE_CURRENT_BINARY_DIR}/lz4_ep-prefix/src/lz4_ep") set(LZ4_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/lz4_ep-prefix/") - if(MSVC) - if(MILVUS_USE_STATIC_CRT) - if(${UPPERCASE_BUILD_TYPE} STREQUAL "DEBUG") - set(LZ4_RUNTIME_LIBRARY_LINKAGE "/p:RuntimeLibrary=MultiThreadedDebug") - else() - set(LZ4_RUNTIME_LIBRARY_LINKAGE "/p:RuntimeLibrary=MultiThreaded") - endif() - endif() - set(LZ4_STATIC_LIB - "${LZ4_BUILD_DIR}/visual/VS2010/bin/x64_${CMAKE_BUILD_TYPE}/liblz4_static.lib") - set(LZ4_BUILD_COMMAND - BUILD_COMMAND - msbuild.exe - /m - /p:Configuration=${CMAKE_BUILD_TYPE} - /p:Platform=x64 - /p:PlatformToolset=v140 - ${LZ4_RUNTIME_LIBRARY_LINKAGE} - /t:Build - ${LZ4_BUILD_DIR}/visual/VS2010/lz4.sln) - else() - set(LZ4_STATIC_LIB "${LZ4_BUILD_DIR}/lib/liblz4.a") - #set(LZ4_BUILD_COMMAND BUILD_COMMAND ${CMAKE_SOURCE_DIR}/build-support/build-lz4-lib.sh - # "AR=${CMAKE_AR}") - set(LZ4_BUILD_COMMAND BUILD_COMMAND ${MAKE} ${MAKE_BUILD_ARGS} CFLAGS=${EP_C_FLAGS}) - endif() + set(LZ4_STATIC_LIB "${LZ4_BUILD_DIR}/lib/liblz4.a") + set(LZ4_BUILD_COMMAND BUILD_COMMAND ${MAKE} ${MAKE_BUILD_ARGS} CFLAGS=${EP_C_FLAGS}) # We need to copy the header in lib to directory outside of the build externalproject_add(lz4_ep @@ -1071,7 +982,6 @@ endmacro() if(MILVUS_WITH_LZ4) resolve_dependency(Lz4) - # TODO: Don't use global includes but rather target_include_directories get_target_property(LZ4_INCLUDE_DIR lz4 INTERFACE_INCLUDE_DIRECTORIES) link_directories(SYSTEM ${LZ4_BUILD_DIR}/lib/) include_directories(SYSTEM ${LZ4_INCLUDE_DIR}) @@ -1097,16 +1007,8 @@ macro(build_mysqlpp) externalproject_add(mysqlpp_ep URL ${MYSQLPP_SOURCE_URL} -# GIT_REPOSITORY -# ${MYSQLPP_SOURCE_URL} -# GIT_TAG -# ${MYSQLPP_VERSION} -# GIT_SHALLOW -# TRUE ${EP_LOG_OPTIONS} CONFIGURE_COMMAND -# "./bootstrap" -# COMMAND "./configure" ${MYSQLPP_CONFIGURE_ARGS} BUILD_COMMAND @@ -1167,10 +1069,6 @@ macro(build_prometheus) ${PROMETHEUS_VERSION} GIT_SHALLOW TRUE -# GIT_CONFIG -# recurse-submodules=true -# URL -# ${PROMETHEUS_SOURCE_URL} ${EP_LOG_OPTIONS} CMAKE_ARGS ${PROMETHEUS_CMAKE_ARGS} @@ -1214,21 +1112,15 @@ if(MILVUS_WITH_PROMETHEUS) resolve_dependency(Prometheus) - # TODO: Don't use global includes but rather target_include_directories - #get_target_property(PROMETHEUS-core_INCLUDE_DIRS prometheus-core INTERFACE_INCLUDE_DIRECTORIES) - - #get_target_property(PROMETHEUS_PUSH_INCLUDE_DIRS prometheus_push INTERFACE_INCLUDE_DIRECTORIES) link_directories(SYSTEM ${PROMETHEUS_PREFIX}/push/) include_directories(SYSTEM ${PROMETHEUS_PREFIX}/push/include) - #get_target_property(PROMETHEUS_PULL_INCLUDE_DIRS prometheus_pull INTERFACE_INCLUDE_DIRECTORIES) link_directories(SYSTEM ${PROMETHEUS_PREFIX}/pull/) include_directories(SYSTEM ${PROMETHEUS_PREFIX}/pull/include) link_directories(SYSTEM ${PROMETHEUS_PREFIX}/core/) include_directories(SYSTEM ${PROMETHEUS_PREFIX}/core/include) - #link_directories(${PROMETHEUS_PREFIX}/civetweb_ep-prefix/src/civetweb_ep) endif() # ---------------------------------------------------------------------- @@ -1276,8 +1168,6 @@ if(MILVUS_WITH_ROCKSDB) resolve_dependency(RocksDB) - # TODO: Don't use global includes but rather target_include_directories -# get_target_property(ROCKSDB_INCLUDE_DIRS rocksdb INTERFACE_INCLUDE_DIRECTORIES) link_directories(SYSTEM ${ROCKSDB_PREFIX}/lib/lib/) include_directories(SYSTEM ${ROCKSDB_INCLUDE_DIRS}) endif() @@ -1326,34 +1216,9 @@ macro(build_snappy) endmacro() if(MILVUS_WITH_SNAPPY) -# if(Snappy_SOURCE STREQUAL "AUTO") -# # Normally *Config.cmake files reside in /usr/lib/cmake but Snappy -# # errornously places them in ${CMAKE_ROOT}/Modules/ -# # This is fixed in 1.1.7 but fedora (30) still installs into the wrong -# # location. -# # https://bugzilla.redhat.com/show_bug.cgi?id=1679727 -# # https://src.fedoraproject.org/rpms/snappy/pull-request/1 -# find_package(Snappy QUIET HINTS "${CMAKE_ROOT}/Modules/") -# if(NOT Snappy_FOUND) -# find_package(SnappyAlt) -# endif() -# if(NOT Snappy_FOUND AND NOT SnappyAlt_FOUND) -# build_snappy() -# endif() -# elseif(Snappy_SOURCE STREQUAL "BUNDLED") -# build_snappy() -# elseif(Snappy_SOURCE STREQUAL "SYSTEM") -# # SnappyConfig.cmake is not installed on Ubuntu/Debian -# # TODO: Make a bug report upstream -# find_package(Snappy HINTS "${CMAKE_ROOT}/Modules/") -# if(NOT Snappy_FOUND) -# find_package(SnappyAlt REQUIRED) -# endif() -# endif() resolve_dependency(Snappy) - - # TODO: Don't use global includes but rather target_include_directories + get_target_property(SNAPPY_INCLUDE_DIRS snappy INTERFACE_INCLUDE_DIRECTORIES) link_directories(SYSTEM ${SNAPPY_PREFIX}/lib/) include_directories(SYSTEM ${SNAPPY_INCLUDE_DIRS}) @@ -1425,75 +1290,11 @@ macro(build_sqlite_orm) endif () - #set(SQLITE_ORM_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/sqlite_orm_ep-prefix/src/sqlite_orm_ep") - #set(SQLITE_ORM_INCLUDE_DIR "${SQLITE_ORM_PREFIX}/include/sqlite_orm") - -# set(SQLITE_ORM_STATIC_LIB -# "${SQLITE_ORM_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}sqlite_orm${CMAKE_STATIC_LIBRARY_SUFFIX}") -# -# set(SQLITE_ORM_CMAKE_CXX_FLAGS "${EP_CXX_FLAGS} -std=c++14") -# set(SQLITE_ORM_CMAKE_CXX_FLAGS_DEBUG "${EP_CXX_FLAGS} -std=c++14") -# -# set(SQLITE_ORM_CMAKE_ARGS -# ${EP_COMMON_CMAKE_ARGS} -# "-DCMAKE_INSTALL_PREFIX=${SQLITE_ORM_PREFIX}" -# #"LDFLAGS=-L${SQLITE_PREFIX}" -# #"-DCMAKE_PREFIX_PATH=${SQLITE_PREFIX}/include" -# "-DCMAKE_INCLUDE_PATH=${SQLITE_PREFIX}/include" -# "-DCMAKE_CXX_FLAGS=${SQLITE_ORM_CMAKE_CXX_FLAGS}" -# "-DCMAKE_CXX_FLAGS_DEBUG=${SQLITE_ORM_CMAKE_CXX_FLAGS}" -# -DSqliteOrm_BuildTests=off -# -DBUILD_TESTING=off) -# message(STATUS "SQLITE_INCLUDE: ${SQLITE_ORM_CMAKE_ARGS}") -# -# message(STATUS "SQLITE_ORM_CMAKE_CXX_FLAGS: ${SQLITE_ORM_CMAKE_CXX_FLAGS}") - -# externalproject_add(sqlite_orm_ep -# URL -# ${SQLITE_ORM_SOURCE_URL} -# PREFIX ${CMAKE_CURRENT_BINARY_DIR}/sqlite_orm_ep-prefix -# CONFIGURE_COMMAND -# "" -# BUILD_COMMAND -# "" -# INSTALL_COMMAND -# "" - #${EP_LOG_OPTIONS} - #${EP_LOG_OPTIONS} -# CMAKE_ARGS -# ${SQLITE_ORM_CMAKE_ARGS} -# BUILD_COMMAND -# ${MAKE} -# ${MAKE_BUILD_ARGS} -# #"LDFLAGS=-L${SQLITE_PREFIX}" -# BUILD_IN_SOURCE -# 1 -# BUILD_BYPRODUCTS -# "${SQLITE_ORM_STATIC_LIB}" -# ) -# ExternalProject_Add_StepDependencies(sqlite_orm_ep build sqlite_ep) - - #set(SQLITE_ORM_SQLITE_HEADER ${SQLITE_INCLUDE_DIR}/sqlite3.h) -# file(MAKE_DIRECTORY "${SQLITE_ORM_INCLUDE_DIR}") -# add_library(sqlite_orm STATIC IMPORTED) -## message(STATUS "SQLITE_INCLUDE_DIR: ${SQLITE_INCLUDE_DIR}") -# set_target_properties( -# sqlite_orm -# PROPERTIES -# IMPORTED_LOCATION "${SQLITE_ORM_STATIC_LIB}" -# INTERFACE_INCLUDE_DIRECTORIES "${SQLITE_ORM_INCLUDE_DIR};${SQLITE_INCLUDE_DIR}") -# target_include_directories(sqlite_orm INTERFACE ${SQLITE_PREFIX} ${SQLITE_INCLUDE_DIR}) -# target_link_libraries(sqlite_orm INTERFACE sqlite) -# -# add_dependencies(sqlite_orm sqlite_orm_ep) endmacro() if(MILVUS_WITH_SQLITE_ORM) resolve_dependency(SQLite_ORM) -# ExternalProject_Get_Property(sqlite_orm_ep source_dir) -# set(SQLITE_ORM_INCLUDE_DIR ${source_dir}/sqlite_orm_ep) include_directories(SYSTEM "${SQLITE_ORM_INCLUDE_DIR}") - #message(STATUS "SQLITE_ORM_INCLUDE_DIR: ${SQLITE_ORM_INCLUDE_DIR}") endif() # ---------------------------------------------------------------------- @@ -1533,18 +1334,7 @@ macro(build_thrift) endif() set(THRIFT_STATIC_LIB_NAME "${CMAKE_STATIC_LIBRARY_PREFIX}thrift") - if(MSVC) - if(MILVUS_USE_STATIC_CRT) - set(THRIFT_STATIC_LIB_NAME "${THRIFT_STATIC_LIB_NAME}") - set(THRIFT_CMAKE_ARGS ${THRIFT_CMAKE_ARGS} "-DWITH_MT=ON") - else() - set(THRIFT_STATIC_LIB_NAME "${THRIFT_STATIC_LIB_NAME}") - set(THRIFT_CMAKE_ARGS ${THRIFT_CMAKE_ARGS} "-DWITH_MT=OFF") - endif() - endif() - if(${UPPERCASE_BUILD_TYPE} STREQUAL "DEBUG") - set(THRIFT_STATIC_LIB_NAME "${THRIFT_STATIC_LIB_NAME}") - endif() + set(THRIFT_STATIC_LIB "${THRIFT_PREFIX}/lib/${THRIFT_STATIC_LIB_NAME}${CMAKE_STATIC_LIBRARY_SUFFIX}") @@ -1555,60 +1345,6 @@ macro(build_thrift) endif() set(THRIFT_DEPENDENCIES ${THRIFT_DEPENDENCIES} ${ZLIB_LIBRARY}) - if(MSVC) - set(WINFLEXBISON_VERSION 2.4.9) - set(WINFLEXBISON_PREFIX - "${CMAKE_CURRENT_BINARY_DIR}/winflexbison_ep/src/winflexbison_ep-install") - externalproject_add( - winflexbison_ep - URL - https://github.com/lexxmark/winflexbison/releases/download/v.${WINFLEXBISON_VERSION}/win_flex_bison-${WINFLEXBISON_VERSION}.zip - URL_HASH - MD5=a2e979ea9928fbf8567e995e9c0df765 - SOURCE_DIR - ${WINFLEXBISON_PREFIX} - CONFIGURE_COMMAND - "" - BUILD_COMMAND - "" - INSTALL_COMMAND - "" - ${EP_LOG_OPTIONS}) - set(THRIFT_DEPENDENCIES ${THRIFT_DEPENDENCIES} winflexbison_ep) - - set(THRIFT_CMAKE_ARGS - "-DFLEX_EXECUTABLE=${WINFLEXBISON_PREFIX}/win_flex.exe" - "-DBISON_EXECUTABLE=${WINFLEXBISON_PREFIX}/win_bison.exe" - "-DZLIB_INCLUDE_DIR=${ZLIB_INCLUDE_DIR}" - "-DWITH_SHARED_LIB=OFF" - "-DWITH_PLUGIN=OFF" - ${THRIFT_CMAKE_ARGS}) - elseif(APPLE) - # Some other process always resets BISON_EXECUTABLE to the system default, - # thus we use our own variable here. - if(NOT DEFINED THRIFT_BISON_EXECUTABLE) - find_package(BISON 2.5.1) - - # In the case where we cannot find a system-wide installation, look for - # homebrew and ask for its bison installation. - if(NOT BISON_FOUND) - find_program(BREW_BIN brew) - if(BREW_BIN) - execute_process(COMMAND ${BREW_BIN} --prefix bison - OUTPUT_VARIABLE BISON_PREFIX - OUTPUT_STRIP_TRAILING_WHITESPACE) - set(BISON_EXECUTABLE "${BISON_PREFIX}/bin/bison") - find_package(BISON 2.5.1) - set(THRIFT_BISON_EXECUTABLE "${BISON_EXECUTABLE}") - endif() - else() - set(THRIFT_BISON_EXECUTABLE "${BISON_EXECUTABLE}") - endif() - endif() - set(THRIFT_CMAKE_ARGS "-DBISON_EXECUTABLE=${THRIFT_BISON_EXECUTABLE}" - ${THRIFT_CMAKE_ARGS}) - endif() - externalproject_add(thrift_ep URL ${THRIFT_SOURCE_URL} @@ -1637,8 +1373,7 @@ endmacro() if(MILVUS_WITH_THRIFT) resolve_dependency(Thrift) - # TODO: Don't use global includes but rather target_include_directories -# MESSAGE(STATUS ${THRIFT_PREFIX}/lib/) + link_directories(SYSTEM ${THRIFT_PREFIX}/lib/) link_directories(SYSTEM ${CMAKE_CURRENT_BINARY_DIR}/thrift_ep-prefix/src/thrift_ep-build/lib) include_directories(SYSTEM ${THRIFT_INCLUDE_DIR}) @@ -1684,8 +1419,7 @@ endmacro() if(MILVUS_WITH_YAMLCPP) resolve_dependency(yaml-cpp) - - # TODO: Don't use global includes but rather target_include_directories + get_target_property(YAMLCPP_INCLUDE_DIR yaml-cpp INTERFACE_INCLUDE_DIRECTORIES) link_directories(SYSTEM ${YAMLCPP_PREFIX}/lib/) include_directories(SYSTEM ${YAMLCPP_INCLUDE_DIR}) @@ -1697,15 +1431,7 @@ endif() macro(build_zlib) message(STATUS "Building ZLIB-${ZLIB_VERSION} from source") set(ZLIB_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/zlib_ep-prefix/src/zlib_ep") - if(MSVC) - if(${UPPERCASE_BUILD_TYPE} STREQUAL "DEBUG") - set(ZLIB_STATIC_LIB_NAME zlibstaticd.lib) - else() - set(ZLIB_STATIC_LIB_NAME zlibstatic.lib) - endif() - else() - set(ZLIB_STATIC_LIB_NAME libz.a) - endif() + set(ZLIB_STATIC_LIB_NAME libz.a) set(ZLIB_STATIC_LIB "${ZLIB_PREFIX}/lib/${ZLIB_STATIC_LIB_NAME}") set(ZLIB_CMAKE_ARGS ${EP_COMMON_CMAKE_ARGS} "-DCMAKE_INSTALL_PREFIX=${ZLIB_PREFIX}" -DBUILD_SHARED_LIBS=OFF) @@ -1734,8 +1460,7 @@ endmacro() if(MILVUS_WITH_ZLIB) resolve_dependency(ZLIB) - - # TODO: Don't use global includes but rather target_include_directories + get_target_property(ZLIB_INCLUDE_DIR zlib INTERFACE_INCLUDE_DIRECTORIES) include_directories(SYSTEM ${ZLIB_INCLUDE_DIR}) endif() @@ -1757,22 +1482,15 @@ macro(build_zstd) -DZSTD_BUILD_STATIC=on -DZSTD_MULTITHREAD_SUPPORT=off) - if(MSVC) - set(ZSTD_STATIC_LIB "${ZSTD_PREFIX}/lib/zstd_static.lib") - if(MILVUS_USE_STATIC_CRT) - set(ZSTD_CMAKE_ARGS ${ZSTD_CMAKE_ARGS} "-DZSTD_USE_STATIC_RUNTIME=on") - endif() - else() - set(ZSTD_STATIC_LIB "${ZSTD_PREFIX}/lib/libzstd.a") - # Only pass our C flags on Unix as on MSVC it leads to a - # "incompatible command-line options" error - set(ZSTD_CMAKE_ARGS - ${ZSTD_CMAKE_ARGS} - -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} - -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} - -DCMAKE_C_FLAGS=${EP_C_FLAGS} - -DCMAKE_CXX_FLAGS=${EP_CXX_FLAGS}) - endif() + + set(ZSTD_STATIC_LIB "${ZSTD_PREFIX}/lib/libzstd.a") + + set(ZSTD_CMAKE_ARGS + ${ZSTD_CMAKE_ARGS} + -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} + -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} + -DCMAKE_C_FLAGS=${EP_C_FLAGS} + -DCMAKE_CXX_FLAGS=${EP_CXX_FLAGS}) if(CMAKE_VERSION VERSION_LESS 3.7) message(FATAL_ERROR "Building zstd using ExternalProject requires at least CMake 3.7") @@ -1806,8 +1524,7 @@ endmacro() if(MILVUS_WITH_ZSTD) resolve_dependency(ZSTD) - - # TODO: Don't use global includes but rather target_include_directories + get_target_property(ZSTD_INCLUDE_DIR zstd INTERFACE_INCLUDE_DIRECTORIES) link_directories(SYSTEM ${ZSTD_PREFIX}/lib) include_directories(SYSTEM ${ZSTD_INCLUDE_DIR}) @@ -1823,7 +1540,7 @@ macro(build_aws) ${EP_COMMON_TOOLCHAIN} "-DCMAKE_INSTALL_PREFIX=${AWS_PREFIX}" -DCMAKE_BUILD_TYPE=Release - -DCMAKE_INSTALL_LIBDIR=lib #${CMAKE_INSTALL_LIBDIR} + -DCMAKE_INSTALL_LIBDIR=lib -DBUILD_ONLY=s3 -DBUILD_SHARED_LIBS=off -DENABLE_TESTING=off @@ -1834,8 +1551,7 @@ macro(build_aws) "${AWS_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}aws-cpp-sdk-core${CMAKE_STATIC_LIBRARY_SUFFIX}") set(AWS_CPP_SDK_S3_STATIC_LIB "${AWS_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}aws-cpp-sdk-s3${CMAKE_STATIC_LIBRARY_SUFFIX}") - # Only pass our C flags on Unix as on MSVC it leads to a - # "incompatible command-line options" error + set(AWS_CMAKE_ARGS ${AWS_CMAKE_ARGS} -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} @@ -1843,10 +1559,6 @@ macro(build_aws) -DCMAKE_C_FLAGS=${EP_C_FLAGS} -DCMAKE_CXX_FLAGS=${EP_CXX_FLAGS}) - if(CMAKE_VERSION VERSION_LESS 3.7) - message(FATAL_ERROR "Building AWS using ExternalProject requires at least CMake 3.7") - endif() - externalproject_add(aws_ep ${EP_LOG_OPTIONS} CMAKE_ARGS @@ -1861,8 +1573,6 @@ macro(build_aws) BUILD_BYPRODUCTS "${AWS_CPP_SDK_S3_STATIC_LIB}" "${AWS_CPP_SDK_CORE_STATIC_LIB}") - - file(MAKE_DIRECTORY "${AWS_PREFIX}/include") add_library(aws-cpp-sdk-s3 STATIC IMPORTED) @@ -1885,8 +1595,7 @@ endmacro() if(MILVUS_WITH_AWS) resolve_dependency(AWS) - - # TODO: Don't use global includes but rather target_include_directories + link_directories(SYSTEM ${AWS_PREFIX}/lib) get_target_property(AWS_CPP_SDK_S3_INCLUDE_DIR aws-cpp-sdk-s3 INTERFACE_INCLUDE_DIRECTORIES) @@ -1896,3 +1605,91 @@ if(MILVUS_WITH_AWS) include_directories(SYSTEM ${AWS_CPP_SDK_CORE_INCLUDE_DIR}) endif() + +# ---------------------------------------------------------------------- +# libunwind + +macro(build_libunwind) + message(STATUS "Building libunwind-${LIBUNWIND_VERSION} from source") + set(LIBUNWIND_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/libunwind_ep-prefix/src/libunwind_ep/install") + set(LIBUNWIND_INCLUDE_DIR "${LIBUNWIND_PREFIX}/include") + set(LIBUNWIND_SHARED_LIB "${LIBUNWIND_PREFIX}/lib/libunwind${CMAKE_SHARED_LIBRARY_SUFFIX}") + set(LIBUNWIND_CONFIGURE_ARGS "--prefix=${LIBUNWIND_PREFIX}") + + externalproject_add(libunwind_ep + URL + ${LIBUNWIND_SOURCE_URL} + ${EP_LOG_OPTIONS} + CONFIGURE_COMMAND + "./configure" + ${LIBUNWIND_CONFIGURE_ARGS} + BUILD_COMMAND + ${MAKE} ${MAKE_BUILD_ARGS} + BUILD_IN_SOURCE + 1 + INSTALL_COMMAND + ${MAKE} install + BUILD_BYPRODUCTS + ${LIBUNWIND_SHARED_LIB}) + + file(MAKE_DIRECTORY "${LIBUNWIND_INCLUDE_DIR}") + + add_library(libunwind SHARED IMPORTED) + set_target_properties(libunwind + PROPERTIES IMPORTED_LOCATION "${LIBUNWIND_SHARED_LIB}" + INTERFACE_INCLUDE_DIRECTORIES "${LIBUNWIND_INCLUDE_DIR}") + + add_dependencies(libunwind libunwind_ep) +endmacro() + +if(MILVUS_WITH_LIBUNWIND) + resolve_dependency(libunwind) + + # TODO: Don't use global includes but rather target_include_directories + get_target_property(LIBUNWIND_INCLUDE_DIR libunwind INTERFACE_INCLUDE_DIRECTORIES) + include_directories(SYSTEM ${LIBUNWIND_INCLUDE_DIR}) +endif() + +# ---------------------------------------------------------------------- +# gperftools + +macro(build_gperftools) + message(STATUS "Building gperftools-${GPERFTOOLS_VERSION} from source") + set(GPERFTOOLS_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/gperftools_ep-prefix/src/gperftools_ep") + set(GPERFTOOLS_INCLUDE_DIR "${GPERFTOOLS_PREFIX}/include") + set(GPERFTOOLS_STATIC_LIB "${GPERFTOOLS_PREFIX}/lib/libprofiler${CMAKE_STATIC_LIBRARY_SUFFIX}") + set(GPERFTOOLS_CONFIGURE_ARGS "--prefix=${GPERFTOOLS_PREFIX}") + + externalproject_add(gperftools_ep + URL + ${GPERFTOOLS_SOURCE_URL} + ${EP_LOG_OPTIONS} + CONFIGURE_COMMAND + "./configure" + ${GPERFTOOLS_CONFIGURE_ARGS} + BUILD_COMMAND + ${MAKE} ${MAKE_BUILD_ARGS} + BUILD_IN_SOURCE + 1 + INSTALL_COMMAND + ${MAKE} install + BUILD_BYPRODUCTS + ${GPERFTOOLS_STATIC_LIB}) + + file(MAKE_DIRECTORY "${GPERFTOOLS_INCLUDE_DIR}") + + add_library(gperftools SHARED IMPORTED) + set_target_properties(gperftools + PROPERTIES IMPORTED_LOCATION "${GPERFTOOLS_STATIC_LIB}" + INTERFACE_INCLUDE_DIRECTORIES "${GPERFTOOLS_INCLUDE_DIR}") + + add_dependencies(gperftools gperftools_ep) +endmacro() + +if(MILVUS_WITH_GPERFTOOLS) + resolve_dependency(gperftools) + + # TODO: Don't use global includes but rather target_include_directories + get_target_property(GPERFTOOLS_INCLUDE_DIR gperftools INTERFACE_INCLUDE_DIRECTORIES) + include_directories(SYSTEM ${GPERFTOOLS_INCLUDE_DIR}) +endif() diff --git a/cpp/conf/log_config.template b/cpp/conf/log_config.template index f4f3d3684c..cc33899589 100644 --- a/cpp/conf/log_config.template +++ b/cpp/conf/log_config.template @@ -6,7 +6,7 @@ TO_STANDARD_OUTPUT = false SUBSECOND_PRECISION = 3 PERFORMANCE_TRACKING = false - MAX_LOG_FILE_SIZE = 2097152 ## Throw log files away after 2MB + MAX_LOG_FILE_SIZE = 209715200 ## Throw log files away after 200MB * DEBUG: FILENAME = "@MILVUS_DB_PATH@/logs/milvus-%datetime{%H:%m}-debug.log" ENABLED = true diff --git a/cpp/conf/server_config.template b/cpp/conf/server_config.template index 2dd8daa0f1..b079d603d2 100644 --- a/cpp/conf/server_config.template +++ b/cpp/conf/server_config.template @@ -34,6 +34,8 @@ license_config: # license configure cache_config: # cache configure cpu_cache_capacity: 16 # how many memory are used as cache, unit: GB, range: 0 ~ less than total memory + cache_free_percent: 0.85 # how much memory should be free when cache is full, range: greater than zero ~ 1.0 + insert_cache_immediately: false # insert data will be load into cache immediately for hot query engine_config: nprobe: 10 diff --git a/cpp/scripts/requirements.sh b/cpp/scripts/requirements.sh new file mode 100755 index 0000000000..5f8b74ad2c --- /dev/null +++ b/cpp/scripts/requirements.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash + +wget -P /tmp https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2019.PUB +apt-key add /tmp/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2019.PUB + +sh -c 'echo deb https://apt.repos.intel.com/mkl all main > /etc/apt/sources.list.d/intel-mkl.list' +apt -y update && apt-get -y install intel-mkl-gnu-2019.4-243 intel-mkl-core-2019.4-243 + +#sh -c 'echo export LD_LIBRARY_PATH=/opt/intel/compilers_and_libraries_2019.4.243/linux/mkl/lib/intel64:\$LD_LIBRARY_PATH > /etc/profile.d/mkl.sh' +#source /etc/profile diff --git a/cpp/src/CMakeLists.txt b/cpp/src/CMakeLists.txt index 84b225efbe..5e4f940ff6 100644 --- a/cpp/src/CMakeLists.txt +++ b/cpp/src/CMakeLists.txt @@ -53,7 +53,6 @@ set(engine_files ${db_files} ${db_scheduler_files} ${wrapper_files} -# metrics/Metrics.cpp ${metrics_files} ) @@ -71,6 +70,10 @@ include_directories(/usr/include/mysql) include_directories(grpc/gen-status) include_directories(grpc/gen-milvus) +if (MILVUS_ENABLE_PROFILING STREQUAL "ON") + SET(PROFILER_LIB profiler) +endif() + set(third_party_libs easyloggingpp sqlite @@ -79,10 +82,7 @@ set(third_party_libs grpc++ grpcpp_channelz yaml-cpp - libgpufaiss.a faiss - lapack - openblas prometheus-cpp-push prometheus-cpp-pull prometheus-cpp-core @@ -95,11 +95,22 @@ set(third_party_libs zlib zstd mysqlpp + ${PROFILER_LIB} ${CUDA_TOOLKIT_ROOT_DIR}/lib64/stubs/libnvidia-ml.so ) + if (MEGASEARCH_WITH_ARROW STREQUAL "ON") set(third_party_libs ${third_party_libs} arrow) -endif() +endif() +if(${BUILD_FAISS_WITH_MKL} STREQUAL "ON") + set(third_party_libs ${third_party_libs} + ${MKL_LIBS} + ${MKL_LIBS}) +else() + set(third_party_libs ${third_party_libs} + lapack + openblas) +endif() if (GPU_VERSION STREQUAL "ON") link_directories("${CUDA_TOOLKIT_ROOT_DIR}/lib64") @@ -198,7 +209,7 @@ install(FILES ${CMAKE_BINARY_DIR}/mysqlpp_ep-prefix/src/mysqlpp_ep/lib/${CMAKE_SHARED_LIBRARY_PREFIX}mysqlpp${CMAKE_SHARED_LIBRARY_SUFFIX} ${CMAKE_BINARY_DIR}/mysqlpp_ep-prefix/src/mysqlpp_ep/lib/${CMAKE_SHARED_LIBRARY_PREFIX}mysqlpp${CMAKE_SHARED_LIBRARY_SUFFIX}.3 ${CMAKE_BINARY_DIR}/mysqlpp_ep-prefix/src/mysqlpp_ep/lib/${CMAKE_SHARED_LIBRARY_PREFIX}mysqlpp${CMAKE_SHARED_LIBRARY_SUFFIX}.3.2.4 - DESTINATION lib) #need to copy libmysqlpp.so + DESTINATION lib) #add_subdirectory(sdk) add_subdirectory(grpcsdk) diff --git a/cpp/src/cache/Cache.cpp b/cpp/src/cache/Cache.cpp index 4d09acc9a5..0cc804ac5f 100644 --- a/cpp/src/cache/Cache.cpp +++ b/cpp/src/cache/Cache.cpp @@ -13,9 +13,12 @@ namespace zilliz { namespace milvus { namespace cache { +constexpr double DEFAULT_THRESHHOLD_PERCENT = 0.85; + Cache::Cache(int64_t capacity, uint64_t cache_max_count) : usage_(0), capacity_(capacity), + freemem_percent_(DEFAULT_THRESHHOLD_PERCENT), lru_(cache_max_count) { // AGENT_LOG_DEBUG << "Construct Cache with capacity " << std::to_string(mem_capacity) } @@ -64,15 +67,14 @@ void Cache::insert(const std::string& key, const DataObjPtr& data_ptr) { usage_ += data_ptr->size(); } -// AGENT_LOG_DEBUG << "Insert into LRU(" << (capacity_ > 0 ? std::to_string(usage_ * 100 / capacity_) : "Nan") -// << "%, +" << data_ptr->size() << ", " << usage_ << ", " << lru_.size() << "):" -// << " " << key; + SERVER_LOG_DEBUG << "Insert " << key << " size:" << data_ptr->size() + << " bytes into cache, usage: " << usage_ << " bytes"; } if (usage_ > capacity_) { -// AGENT_LOG_TRACE << "Current usage " << usage_ -// << " exceeds cache capacity " << capacity_ -// << ", start free memory"; + SERVER_LOG_DEBUG << "Current usage " << usage_ + << " exceeds cache capacity " << capacity_ + << ", start free memory"; free_memory(); } } @@ -86,12 +88,9 @@ void Cache::erase(const std::string& key) { const CacheObjPtr& obj_ptr = lru_.get(key); const DataObjPtr& data_ptr = obj_ptr->data_; usage_ -= data_ptr->size(); -// AGENT_LOG_DEBUG << "Erase from LRU(" << (capacity_ > 0 ? std::to_string(usage_*100/capacity_) : "Nan") -// << "%, -" << data_ptr->size() << ", " << usage_ << ", " << lru_.size() << "): " -// << (data_ptr->flags().get_flag(DataObjAttr::kPinned) ? "Pinned " : "") -// << (data_ptr->flags().get_flag(DataObjAttr::kValid) ? "Valid " : "") -// << "(ref:" << obj_ptr->ref_ << ") " -// << key; + + SERVER_LOG_DEBUG << "Erase " << key << " from cache"; + lru_.erase(key); } @@ -99,7 +98,7 @@ void Cache::clear() { std::lock_guard lock(mutex_); lru_.clear(); usage_ = 0; -// AGENT_LOG_DEBUG << "Clear LRU !"; + SERVER_LOG_DEBUG << "Clear cache !"; } #if 0 /* caiyd 20190221, need more testing before enable */ @@ -162,7 +161,7 @@ void Cache::restore_from_file(const std::string& key, const CacheObjPtr& obj_ptr void Cache::free_memory() { if (usage_ <= capacity_) return; - int64_t threshhold = capacity_ * THRESHHOLD_PERCENT; + int64_t threshhold = capacity_ * freemem_percent_; int64_t delta_size = usage_ - threshhold; std::set key_array; @@ -183,7 +182,7 @@ void Cache::free_memory() { } } -// AGENT_LOG_DEBUG << "to be released memory size: " << released_size; + SERVER_LOG_DEBUG << "to be released memory size: " << released_size; for (auto& key : key_array) { erase(key); @@ -193,28 +192,15 @@ void Cache::free_memory() { } void Cache::print() { - int64_t still_pinned_count = 0; - int64_t total_pinned_size = 0; - int64_t total_valid_empty_size = 0; + size_t cache_count = 0; { std::lock_guard lock(mutex_); - - for (auto it = lru_.begin(); it != lru_.end(); ++it) { - auto& obj_ptr = it->second; - const auto& data_ptr = obj_ptr->data_; - if (data_ptr != nullptr) { - total_pinned_size += data_ptr->size(); - ++still_pinned_count; - } else { - total_valid_empty_size += data_ptr->size(); - } - } + cache_count = lru_.size(); } - SERVER_LOG_DEBUG << "[Still Pinned count]: " << still_pinned_count; - SERVER_LOG_DEBUG << "[Pinned Memory total size(byte)]: " << total_pinned_size; - SERVER_LOG_DEBUG << "[valid_empty total size(byte)]: " << total_valid_empty_size; - SERVER_LOG_DEBUG << "[free memory size(byte)]: " << capacity_ - total_pinned_size - total_valid_empty_size; + SERVER_LOG_DEBUG << "[Cache item count]: " << cache_count; + SERVER_LOG_DEBUG << "[Cache usage]: " << usage_ << " bytes"; + SERVER_LOG_DEBUG << "[Cache capacity]: " << capacity_ << " bytes"; } } // cache diff --git a/cpp/src/cache/Cache.h b/cpp/src/cache/Cache.h index 80f3d0d713..4d6f32b9eb 100644 --- a/cpp/src/cache/Cache.h +++ b/cpp/src/cache/Cache.h @@ -18,7 +18,6 @@ namespace milvus { namespace cache { const std::string SWAP_DIR = ".CACHE"; -const float THRESHHOLD_PERCENT = 0.75; class Cache { private: @@ -45,6 +44,9 @@ public: int64_t capacity() const { return capacity_; } //unit: BYTE void set_capacity(int64_t capacity); //unit: BYTE + double freemem_percent() const { return freemem_percent_; }; + void set_freemem_percent(double percent) { freemem_percent_ = percent; } + size_t size() const; bool exists(const std::string& key); DataObjPtr get(const std::string& key); @@ -57,6 +59,7 @@ public: private: int64_t usage_; int64_t capacity_; + double freemem_percent_; LRU lru_; mutable std::mutex mutex_; diff --git a/cpp/src/cache/CpuCacheMgr.cpp b/cpp/src/cache/CpuCacheMgr.cpp index 36da562b83..a90f8537b4 100644 --- a/cpp/src/cache/CpuCacheMgr.cpp +++ b/cpp/src/cache/CpuCacheMgr.cpp @@ -6,6 +6,7 @@ #include "CpuCacheMgr.h" #include "server/ServerConfig.h" +#include "utils/Log.h" namespace zilliz { namespace milvus { @@ -16,6 +17,14 @@ CpuCacheMgr::CpuCacheMgr() { int64_t cap = config.GetInt64Value(server::CONFIG_CPU_CACHE_CAPACITY, 16); cap *= 1024*1024*1024; cache_ = std::make_shared(cap, 1UL<<32); + + double free_percent = config.GetDoubleValue(server::CACHE_FREE_PERCENT, 0.85); + if(free_percent > 0.0 && free_percent <= 1.0) { + cache_->set_freemem_percent(free_percent); + } else { + SERVER_LOG_ERROR << "Invalid cache_free_percent: " << free_percent << + ", defaultly set to " << cache_->freemem_percent(); + } } } diff --git a/cpp/src/db/DBImpl.cpp b/cpp/src/db/DBImpl.cpp index 55fc9cea5a..ccafcbeed9 100644 --- a/cpp/src/db/DBImpl.cpp +++ b/cpp/src/db/DBImpl.cpp @@ -89,8 +89,11 @@ DBImpl::DBImpl(const Options& options) meta_ptr_ = DBMetaImplFactory::Build(options.meta, options.mode); mem_mgr_ = MemManagerFactory::Build(meta_ptr_, options_); if (options.mode != Options::MODE::READ_ONLY) { + ENGINE_LOG_INFO << "StartTimerTasks"; StartTimerTasks(); } + + } Status DBImpl::CreateTable(meta::TableSchema& table_schema) { @@ -206,9 +209,10 @@ Status DBImpl::Query(const std::string& table_id, const std::vector Status DBImpl::QueryAsync(const std::string& table_id, const meta::TableFilesSchema& files, uint64_t k, uint64_t nq, const float* vectors, const meta::DatesT& dates, QueryResults& results) { + server::TimeRecorder rc(""); //step 1: get files to search - ENGINE_LOG_DEBUG << "Search DateT Size=" << files.size(); + ENGINE_LOG_DEBUG << "Engine query begin, index file count:" << files.size() << " date range count:" << dates.size(); SearchContextPtr context = std::make_shared(k, nq, vectors); for (auto &file : files) { TableFileSchemaPtr file_ptr = std::make_shared(file); @@ -221,8 +225,31 @@ Status DBImpl::QueryAsync(const std::string& table_id, const meta::TableFilesSch context->WaitResult(); - //step 3: construct results + //step 3: print time cost information + double load_cost = context->LoadCost(); + double search_cost = context->SearchCost(); + double reduce_cost = context->ReduceCost(); + std::string load_info = server::TimeRecorder::GetTimeSpanStr(load_cost); + std::string search_info = server::TimeRecorder::GetTimeSpanStr(search_cost); + std::string reduce_info = server::TimeRecorder::GetTimeSpanStr(reduce_cost); + if(search_cost > 0.0 || reduce_cost > 0.0) { + double total_cost = load_cost + search_cost + reduce_cost; + double load_percent = load_cost/total_cost; + double search_percent = search_cost/total_cost; + double reduce_percent = reduce_cost/total_cost; + + ENGINE_LOG_DEBUG << "Engine load index totally cost:" << load_info << " percent: " << load_percent*100 << "%"; + ENGINE_LOG_DEBUG << "Engine search index totally cost:" << search_info << " percent: " << search_percent*100 << "%"; + ENGINE_LOG_DEBUG << "Engine reduce topk totally cost:" << reduce_info << " percent: " << reduce_percent*100 << "%"; + } else { + ENGINE_LOG_DEBUG << "Engine load cost:" << load_info + << " search cost: " << search_info + << " reduce cost: " << reduce_info; + } + + //step 4: construct results results = context->GetResult(); + rc.ElapseFromBegin("Engine query totally cost"); return Status::OK(); } @@ -235,7 +262,6 @@ void DBImpl::BackgroundTimerTask() { Status status; server::SystemInfo::GetInstance().Init(); while (true) { - if (!bg_error_.ok()) break; if (shutting_down_.load(std::memory_order_acquire)){ for(auto& iter : compact_thread_results_) { iter.wait(); @@ -357,10 +383,11 @@ Status DBImpl::MergeFiles(const std::string& table_id, const meta::DateT& date, updated.push_back(table_file); status = meta_ptr_->UpdateTableFiles(updated); ENGINE_LOG_DEBUG << "New merged file " << table_file.file_id_ << - " of size=" << index->PhysicalSize()/(1024*1024) << " M"; + " of size " << index->PhysicalSize() << " bytes"; - //current disable this line to avoid memory - //index->Cache(); + if(options_.insert_cache_immediately_) { + index->Cache(); + } return status; } @@ -390,15 +417,11 @@ Status DBImpl::BackgroundMergeFiles(const std::string& table_id) { } void DBImpl::BackgroundCompaction(std::set table_ids) { -// static int b_count = 0; -// b_count++; -// std::cout << "BackgroundCompaction: " << b_count << std::endl; - Status status; for (auto& table_id : table_ids) { status = BackgroundMergeFiles(table_id); if (!status.ok()) { - bg_error_ = status; + ENGINE_LOG_ERROR << "Merge files for table " << table_id << " failed: " << status.ToString(); return; } } @@ -408,7 +431,6 @@ void DBImpl::BackgroundCompaction(std::set table_ids) { int ttl = 1; if (options_.mode == Options::MODE::CLUSTER) { ttl = meta::D_SEC; -// ENGINE_LOG_DEBUG << "Server mode is cluster. Clean up files with ttl = " << std::to_string(ttl) << "seconds."; } meta_ptr_->CleanUpFilesWithTTL(ttl); } @@ -460,7 +482,7 @@ Status DBImpl::BuildIndex(const meta::TableFileSchema& file) { try { //step 1: load index - to_index->Load(); + to_index->Load(options_.insert_cache_immediately_); //step 2: create table file meta::TableFileSchema table_file; @@ -501,11 +523,12 @@ Status DBImpl::BuildIndex(const meta::TableFileSchema& file) { meta_ptr_->UpdateTableFiles(update_files); ENGINE_LOG_DEBUG << "New index file " << table_file.file_id_ << " of size " - << index->PhysicalSize()/(1024*1024) << " M" + << index->PhysicalSize() << " bytes" << " from file " << to_remove.file_id_; - //current disable this line to avoid memory - //index->Cache(); + if(options_.insert_cache_immediately_) { + index->Cache(); + } } catch (std::exception& ex) { std::string msg = "Build index encounter exception" + std::string(ex.what()); @@ -541,10 +564,9 @@ void DBImpl::BackgroundBuildIndex() { meta_ptr_->FilesToIndex(to_index_files); Status status; for (auto& file : to_index_files) { - /* ENGINE_LOG_DEBUG << "Buiding index for " << file.location; */ status = BuildIndex(file); if (!status.ok()) { - bg_error_ = status; + ENGINE_LOG_ERROR << "Building index for " << file.id_ << " failed: " << status.ToString(); return; } @@ -552,7 +574,6 @@ void DBImpl::BackgroundBuildIndex() { break; } } - /* ENGINE_LOG_DEBUG << "All Buiding index Done"; */ } Status DBImpl::DropAll() { diff --git a/cpp/src/db/DBImpl.h b/cpp/src/db/DBImpl.h index 1309154a3c..73d28779b7 100644 --- a/cpp/src/db/DBImpl.h +++ b/cpp/src/db/DBImpl.h @@ -118,10 +118,8 @@ class DBImpl : public DB { BuildIndex(const meta::TableFileSchema &); private: - const Options options_; - Status bg_error_; std::atomic shutting_down_; std::thread bg_timer_thread_; diff --git a/cpp/src/db/DBMetaImpl.cpp b/cpp/src/db/DBMetaImpl.cpp index 99ae3b6711..56f741c4dc 100644 --- a/cpp/src/db/DBMetaImpl.cpp +++ b/cpp/src/db/DBMetaImpl.cpp @@ -291,6 +291,8 @@ Status DBMetaImpl::HasNonIndexFiles(const std::string& table_id, bool& has) { try { auto selected = ConnectorPtr->select(columns(&TableFileSchema::id_), where((c(&TableFileSchema::file_type_) == (int) TableFileSchema::RAW + or + c(&TableFileSchema::file_type_) == (int) TableFileSchema::NEW or c(&TableFileSchema::file_type_) == (int) TableFileSchema::TO_INDEX) and c(&TableFileSchema::table_id_) == table_id @@ -863,7 +865,7 @@ Status DBMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) { table_file.date_ = std::get<3>(file); utils::DeleteTableFilePath(options_, table_file); - ENGINE_LOG_DEBUG << "Removing deleted id =" << table_file.id_ << " location = " << table_file.location_ << std::endl; + ENGINE_LOG_DEBUG << "Removing file id:" << table_file.id_ << " location:" << table_file.location_; ConnectorPtr->remove(table_file.id_); } diff --git a/cpp/src/db/ExecutionEngine.h b/cpp/src/db/ExecutionEngine.h index cd9e071b9c..d08b9f582a 100644 --- a/cpp/src/db/ExecutionEngine.h +++ b/cpp/src/db/ExecutionEngine.h @@ -39,7 +39,7 @@ public: virtual Status Serialize() = 0; - virtual Status Load() = 0; + virtual Status Load(bool to_cache = true) = 0; virtual Status Merge(const std::string& location) = 0; diff --git a/cpp/src/db/FaissExecutionEngine.cpp b/cpp/src/db/FaissExecutionEngine.cpp index e32363dd1d..fc798afb9c 100644 --- a/cpp/src/db/FaissExecutionEngine.cpp +++ b/cpp/src/db/FaissExecutionEngine.cpp @@ -79,18 +79,17 @@ Status FaissExecutionEngine::Serialize() { return Status::OK(); } -Status FaissExecutionEngine::Load() { +Status FaissExecutionEngine::Load(bool to_cache) { auto index = zilliz::milvus::cache::CpuCacheMgr::GetInstance()->GetIndex(location_); - bool to_cache = false; + bool already_in_cache = (index != nullptr); auto start_time = METRICS_NOW_TIME; if (!index) { index = read_index(location_); - to_cache = true; ENGINE_LOG_DEBUG << "Disk io from: " << location_; } pIndex_ = index->data(); - if (to_cache) { + if (!already_in_cache && to_cache) { Cache(); auto end_time = METRICS_NOW_TIME; auto total_time = METRICS_MICROSECONDS(start_time, end_time); @@ -98,7 +97,6 @@ Status FaissExecutionEngine::Load() { server::Metrics::GetInstance().FaissDiskLoadDurationSecondsHistogramObserve(total_time); double total_size = (pIndex_->d) * (pIndex_->ntotal) * 4; - server::Metrics::GetInstance().FaissDiskLoadSizeBytesHistogramObserve(total_size); // server::Metrics::GetInstance().FaissDiskLoadIOSpeedHistogramObserve(total_size/double(total_time)); server::Metrics::GetInstance().FaissDiskLoadIOSpeedGaugeSet(total_size/double(total_time)); @@ -151,10 +149,11 @@ Status FaissExecutionEngine::Search(long n, std::shared_ptr ivf_index = std::dynamic_pointer_cast(pIndex_); if(ivf_index) { - ENGINE_LOG_DEBUG << "Index type: IVFFLAT nProbe: " << nprobe_; + ENGINE_LOG_DEBUG << "Searching index type: " << build_index_type_ << " nProbe: " << nprobe_; ivf_index->nprobe = nprobe_; ivf_index->search(n, data, k, distances, labels); } else { + ENGINE_LOG_DEBUG << "Searching raw file"; pIndex_->search(n, data, k, distances, labels); } diff --git a/cpp/src/db/FaissExecutionEngine.h b/cpp/src/db/FaissExecutionEngine.h index 7005061ff6..c1baf48be3 100644 --- a/cpp/src/db/FaissExecutionEngine.h +++ b/cpp/src/db/FaissExecutionEngine.h @@ -44,7 +44,7 @@ public: Status Serialize() override; - Status Load() override; + Status Load(bool to_cache) override; Status Merge(const std::string& location) override; diff --git a/cpp/src/db/MemManager.cpp b/cpp/src/db/MemManager.cpp index 366188b547..dc6c9681b0 100644 --- a/cpp/src/db/MemManager.cpp +++ b/cpp/src/db/MemManager.cpp @@ -83,11 +83,12 @@ Status MemVectors::Serialize(std::string &table_id) { auto status = meta_->UpdateTableFile(schema_); - LOG(DEBUG) << "New " << ((schema_.file_type_ == meta::TableFileSchema::RAW) ? "raw" : "to_index") - << " file " << schema_.file_id_ << " of size " << (double) (active_engine_->Size()) / (double) meta::M - << " M"; + ENGINE_LOG_DEBUG << "New " << ((schema_.file_type_ == meta::TableFileSchema::RAW) ? "raw" : "to_index") + << " file " << schema_.file_id_ << " of size " << active_engine_->Size() << " bytes"; - active_engine_->Cache(); + if(options_.insert_cache_immediately_) { + active_engine_->Cache(); + } return status; } diff --git a/cpp/src/db/MemTableFile.cpp b/cpp/src/db/MemTableFile.cpp index 649a680cf3..f36223f097 100644 --- a/cpp/src/db/MemTableFile.cpp +++ b/cpp/src/db/MemTableFile.cpp @@ -95,10 +95,12 @@ Status MemTableFile::Serialize() { auto status = meta_->UpdateTableFile(table_file_schema_); - LOG(DEBUG) << "New " << ((table_file_schema_.file_type_ == meta::TableFileSchema::RAW) ? "raw" : "to_index") - << " file " << table_file_schema_.file_id_ << " of size " << (double) size / (double) M << " M"; + ENGINE_LOG_DEBUG << "New " << ((table_file_schema_.file_type_ == meta::TableFileSchema::RAW) ? "raw" : "to_index") + << " file " << table_file_schema_.file_id_ << " of size " << size << " bytes"; - execution_engine_->Cache(); + if(options_.insert_cache_immediately_) { + execution_engine_->Cache(); + } return status; } diff --git a/cpp/src/db/MySQLMetaImpl.cpp b/cpp/src/db/MySQLMetaImpl.cpp index dc22c0307c..14879d81fe 100644 --- a/cpp/src/db/MySQLMetaImpl.cpp +++ b/cpp/src/db/MySQLMetaImpl.cpp @@ -382,7 +382,49 @@ Status MySQLMetaImpl::CreateTable(TableSchema &table_schema) { } Status MySQLMetaImpl::HasNonIndexFiles(const std::string &table_id, bool &has) { - // TODO + + has = false; + + try { + + StoreQueryResult res; + + { + ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); + + if (connectionPtr == nullptr) { + return Status::Error("Failed to connect to database server"); + } + + + Query hasNonIndexFilesQuery = connectionPtr->query(); + //since table_id is a unique column we just need to check whether it exists or not + hasNonIndexFilesQuery << "SELECT EXISTS " << + "(SELECT 1 FROM TableFiles " << + "WHERE table_id = " << quote << table_id << " AND " << + "(file_type = " << std::to_string(TableFileSchema::RAW) << " OR " << + "file_type = " << std::to_string(TableFileSchema::NEW) << " OR " << + "file_type = " << std::to_string(TableFileSchema::TO_INDEX) << ")) " << + "AS " << quote << "check" << ";"; + + ENGINE_LOG_DEBUG << "MySQLMetaImpl::HasNonIndexFiles: " << hasNonIndexFilesQuery.str(); + + res = hasNonIndexFilesQuery.store(); + } //Scoped Connection + + int check = res[0]["check"]; + has = (check == 1); + + } catch (const BadQuery &er) { + // Handle any query errors + ENGINE_LOG_ERROR << "QUERY ERROR WHEN CHECKING IF NON INDEX FILES EXISTS" << ": " << er.what(); + return Status::DBTransactionError("QUERY ERROR WHEN CHECKING IF NON INDEX FILES EXISTS", er.what()); + } catch (const Exception &er) { + // Catch-all for any other MySQL++ exceptions + ENGINE_LOG_ERROR << "GENERAL ERROR WHEN CHECKING IF NON INDEX FILES EXISTS" << ": " << er.what(); + return Status::DBTransactionError("GENERAL ERROR WHEN CHECKING IF NON INDEX FILES EXISTS", er.what()); + } + return Status::OK(); } @@ -1378,11 +1420,17 @@ Status MySQLMetaImpl::UpdateTableFilesToIndex(const std::string &table_id) { Query updateTableFilesToIndexQuery = connectionPtr->query(); updateTableFilesToIndexQuery << "UPDATE TableFiles " << - "SET file_type = " << std::to_string(TableFileSchema::TO_INDEX) << " " << - "WHERE table_id = " << quote << table_id << " AND " << - "file_type = " << std::to_string(TableFileSchema::RAW) << ";"; + "SET file_type = " << std::to_string(TableFileSchema::TO_INDEX) << " " << + "WHERE table_id = " << quote << table_id << " AND " << + "file_type = " << std::to_string(TableFileSchema::RAW) << ";"; - ENGINE_LOG_DEBUG << "MySQLMetaImpl::UpdateTableFile: " << updateTableFilesToIndexQuery.str(); + ENGINE_LOG_DEBUG << "MySQLMetaImpl::UpdateTableFilesToIndex: " << updateTableFilesToIndexQuery.str(); + + if (!updateTableFilesToIndexQuery.exec()) { + ENGINE_LOG_ERROR << "QUERY ERROR WHEN UPDATING TABLE FILE"; + return Status::DBTransactionError("QUERY ERROR WHEN UPDATING TABLE FILE", + updateTableFilesToIndexQuery.error()); + } } catch (const BadQuery &er) { // Handle any query errors @@ -1530,8 +1578,7 @@ Status MySQLMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) { utils::DeleteTableFilePath(options_, table_file); - ENGINE_LOG_DEBUG << "Removing deleted id =" << table_file.id_ << " location = " - << table_file.location_ << std::endl; + ENGINE_LOG_DEBUG << "Removing file id:" << table_file.id_ << " location:" << table_file.location_; idsToDelete.emplace_back(std::to_string(table_file.id_)); } diff --git a/cpp/src/db/Options.h b/cpp/src/db/Options.h index efc5ee3604..2f7869d6e2 100644 --- a/cpp/src/db/Options.h +++ b/cpp/src/db/Options.h @@ -63,7 +63,9 @@ struct Options { size_t index_trigger_size = ONE_GB; //unit: byte DBMetaOptions meta; int mode = MODE::SINGLE; + size_t insert_buffer_size = 4 * ONE_GB; + bool insert_cache_immediately_ = false; }; // Options diff --git a/cpp/src/db/scheduler/TaskDispatchStrategy.cpp b/cpp/src/db/scheduler/TaskDispatchStrategy.cpp index 985f86cb09..73c46942c8 100644 --- a/cpp/src/db/scheduler/TaskDispatchStrategy.cpp +++ b/cpp/src/db/scheduler/TaskDispatchStrategy.cpp @@ -32,7 +32,7 @@ public: IndexLoadTaskPtr loader = std::static_pointer_cast(task); if(index_files.find(loader->file_->id_) != index_files.end()){ - ENGINE_LOG_INFO << "Append SearchContext to exist IndexLoaderContext"; + ENGINE_LOG_DEBUG << "Append SearchContext to exist IndexLoaderContext"; index_files.erase(loader->file_->id_); loader->search_contexts_.push_back(context); } @@ -40,7 +40,7 @@ public: //index_files still contains some index files, create new loader for(auto& pair : index_files) { - ENGINE_LOG_INFO << "Create new IndexLoaderContext for: " << pair.second->location_; + ENGINE_LOG_DEBUG << "Create new IndexLoaderContext for: " << pair.second->location_; IndexLoadTaskPtr new_loader = std::make_shared(); new_loader->search_contexts_.push_back(context); new_loader->file_ = pair.second; diff --git a/cpp/src/db/scheduler/context/SearchContext.cpp b/cpp/src/db/scheduler/context/SearchContext.cpp index 493005f05a..599686b5f4 100644 --- a/cpp/src/db/scheduler/context/SearchContext.cpp +++ b/cpp/src/db/scheduler/context/SearchContext.cpp @@ -31,7 +31,7 @@ SearchContext::AddIndexFile(TableFileSchemaPtr& index_file) { return false; } - SERVER_LOG_INFO << "SearchContext " << identity_ << " add index file: " << index_file->id_; + SERVER_LOG_DEBUG << "SearchContext " << identity_ << " add index file: " << index_file->id_; map_index_files_[index_file->id_] = index_file; return true; @@ -42,7 +42,7 @@ SearchContext::IndexSearchDone(size_t index_id) { std::unique_lock lock(mtx_); map_index_files_.erase(index_id); done_cond_.notify_all(); - SERVER_LOG_INFO << "SearchContext " << identity_ << " finish index file: " << index_id; + SERVER_LOG_DEBUG << "SearchContext " << identity_ << " finish index file: " << index_id; } void diff --git a/cpp/src/db/scheduler/context/SearchContext.h b/cpp/src/db/scheduler/context/SearchContext.h index e81622eb32..3cea23d96d 100644 --- a/cpp/src/db/scheduler/context/SearchContext.h +++ b/cpp/src/db/scheduler/context/SearchContext.h @@ -37,9 +37,19 @@ public: const ResultSet& GetResult() const { return result_; } ResultSet& GetResult() { return result_; } + std::string Identity() const { return identity_; } + void IndexSearchDone(size_t index_id); void WaitResult(); + void AccumLoadCost(double span) { time_cost_load_ += span; } + void AccumSearchCost(double span) { time_cost_search_ += span; } + void AccumReduceCost(double span) { time_cost_reduce_ += span; } + + double LoadCost() const { return time_cost_load_; } + double SearchCost() const { return time_cost_search_; } + double ReduceCost() const { return time_cost_reduce_; } + private: uint64_t topk_ = 0; uint64_t nq_ = 0; @@ -52,6 +62,10 @@ private: std::condition_variable done_cond_; std::string identity_; //for debug + + double time_cost_load_ = 0.0; //time cost for load all index files, unit: us + double time_cost_search_ = 0.0; //time cost for entire search, unit: us + double time_cost_reduce_ = 0.0; //time cost for entire reduce, unit: us }; using SearchContextPtr = std::shared_ptr; diff --git a/cpp/src/db/scheduler/task/IndexLoadTask.cpp b/cpp/src/db/scheduler/task/IndexLoadTask.cpp index 91f0e26577..f3aacf8352 100644 --- a/cpp/src/db/scheduler/task/IndexLoadTask.cpp +++ b/cpp/src/db/scheduler/task/IndexLoadTask.cpp @@ -41,20 +41,21 @@ IndexLoadTask::IndexLoadTask() } std::shared_ptr IndexLoadTask::Execute() { - ENGINE_LOG_INFO << "Loading index(" << file_->id_ << ") from location: " << file_->location_; - - server::TimeRecorder rc("Load index"); + server::TimeRecorder rc(""); //step 1: load index ExecutionEnginePtr index_ptr = EngineFactory::Build(file_->dimension_, file_->location_, (EngineType)file_->engine_type_); index_ptr->Load(); - rc.Record("load index file to memory"); - size_t file_size = index_ptr->PhysicalSize(); - LOG(DEBUG) << "Index file type " << file_->file_type_ << " Of Size: " - << file_size/(1024*1024) << " M"; + + std::string info = "Load file id:" + std::to_string(file_->id_) + " file type:" + std::to_string(file_->file_type_) + + " size:" + std::to_string(file_size) + " bytes from location: " + file_->location_ + " totally cost"; + double span = rc.ElapseFromBegin(info); + for(auto& context : search_contexts_) { + context->AccumLoadCost(span); + } CollectFileMetrics(file_->file_type_, file_size); diff --git a/cpp/src/db/scheduler/task/SearchTask.cpp b/cpp/src/db/scheduler/task/SearchTask.cpp index 5606ba2c84..1c0776ee89 100644 --- a/cpp/src/db/scheduler/task/SearchTask.cpp +++ b/cpp/src/db/scheduler/task/SearchTask.cpp @@ -51,10 +51,10 @@ std::shared_ptr SearchTask::Execute() { return nullptr; } - SERVER_LOG_INFO << "Searching in index(" << index_id_<< ") with " + SERVER_LOG_DEBUG << "Searching in file id:" << index_id_<< " with " << search_contexts_.size() << " tasks"; - server::TimeRecorder rc("DoSearch index(" + std::to_string(index_id_) + ")"); + server::TimeRecorder rc("DoSearch file id:" + std::to_string(index_id_)); auto start_time = METRICS_NOW_TIME; @@ -71,17 +71,19 @@ std::shared_ptr SearchTask::Execute() { index_engine_->Search(context->nq(), context->vectors(), inner_k, output_distence.data(), output_ids.data()); - rc.Record("do search"); + double span = rc.RecordSection("do search for context:" + context->Identity()); + context->AccumSearchCost(span); //step 3: cluster result SearchContext::ResultSet result_set; auto spec_k = index_engine_->Count() < context->topk() ? index_engine_->Count() : context->topk(); SearchTask::ClusterResult(output_ids, output_distence, context->nq(), spec_k, result_set); - rc.Record("cluster result"); //step 4: pick up topk result SearchTask::TopkResult(result_set, inner_k, metric_l2, context->GetResult()); - rc.Record("reduce topk"); + + span = rc.RecordSection("reduce topk for context:" + context->Identity()); + context->AccumReduceCost(span); } catch (std::exception& ex) { SERVER_LOG_ERROR << "SearchTask encounter exception: " << ex.what(); @@ -97,7 +99,7 @@ std::shared_ptr SearchTask::Execute() { auto total_time = METRICS_MICROSECONDS(start_time, end_time); CollectDurationMetrics(index_type_, total_time); - rc.Elapse("totally cost"); + rc.ElapseFromBegin("totally cost"); return nullptr; } diff --git a/cpp/src/sdk/examples/simple/src/ClientTest.cpp b/cpp/src/sdk/examples/simple/src/ClientTest.cpp index 5caab0961d..a17e4e5703 100644 --- a/cpp/src/sdk/examples/simple/src/ClientTest.cpp +++ b/cpp/src/sdk/examples/simple/src/ClientTest.cpp @@ -150,7 +150,7 @@ namespace { std::cout << "The top 1 result is wrong: " << result_id << " vs. " << search_id << std::endl; } else { - std::cout << "Check result sucessfully" << std::endl; + std::cout << "No." << index-1 << " Check result successfully" << std::endl; } } BLOCK_SPLITER @@ -236,6 +236,7 @@ ClientTest::Test(const std::string& address, const std::string& port) { std::vector> search_record_array; {//add vectors for (int i = 0; i < ADD_VECTOR_LOOP; i++) {//add vectors + TimeRecorder recorder("Add vector No." + std::to_string(i)); std::vector record_array; int64_t begin_index = i * BATCH_ROW_COUNT; BuildVectors(begin_index, begin_index + BATCH_ROW_COUNT, record_array); @@ -257,6 +258,7 @@ ClientTest::Test(const std::string& address, const std::string& port) { } {//wait unit build index finish + TimeRecorder recorder("Build index"); std::cout << "Wait until build all index done" << std::endl; Status stat = conn->BuildIndex(TABLE_NAME); std::cout << "BuildIndex function call status: " << stat.ToString() << std::endl; diff --git a/cpp/src/sdk/src/client/ClientProxy.cpp b/cpp/src/sdk/src/client/ClientProxy.cpp index b4524ba7dd..7cd0b8e65c 100644 --- a/cpp/src/sdk/src/client/ClientProxy.cpp +++ b/cpp/src/sdk/src/client/ClientProxy.cpp @@ -210,17 +210,25 @@ ClientProxy::SearchVector(const std::string &table_name, } //step 3: search vectors - std::vector result_array; - ClientPtr()->interface()->SearchVector(result_array, table_name, thrift_records, thrift_ranges, topk); + std::vector result_array; + ClientPtr()->interface()->SearchVector2(result_array, table_name, thrift_records, thrift_ranges, topk); //step 4: convert result array for(auto& thrift_topk_result : result_array) { TopKQueryResult result; - for(auto& thrift_query_result : thrift_topk_result.query_result_arrays) { + size_t id_count = thrift_topk_result.id_array.size()/sizeof(int64_t); + size_t dist_count = thrift_topk_result.distance_array.size()/ sizeof(double); + if(id_count != dist_count) { + return Status(StatusCode::UnknownError, "illegal result"); + } + + int64_t* id_ptr = (int64_t*)thrift_topk_result.id_array.data(); + double* dist_ptr = (double*)thrift_topk_result.distance_array.data(); + for(size_t i = 0; i < id_count; i++) { QueryResult query_result; - query_result.id = thrift_query_result.id; - query_result.distance = thrift_query_result.distance; + query_result.id = id_ptr[i]; + query_result.distance = dist_ptr[i]; result.query_result_arrays.emplace_back(query_result); } diff --git a/cpp/src/server/DBWrapper.cpp b/cpp/src/server/DBWrapper.cpp index 6c67176571..e908f62d7c 100644 --- a/cpp/src/server/DBWrapper.cpp +++ b/cpp/src/server/DBWrapper.cpp @@ -16,19 +16,19 @@ namespace server { DBWrapper::DBWrapper() { zilliz::milvus::engine::Options opt; - ConfigNode& config = ServerConfig::GetInstance().GetConfig(CONFIG_DB); - opt.meta.backend_uri = config.GetValue(CONFIG_DB_URL); - std::string db_path = config.GetValue(CONFIG_DB_PATH); + ConfigNode& db_config = ServerConfig::GetInstance().GetConfig(CONFIG_DB); + opt.meta.backend_uri = db_config.GetValue(CONFIG_DB_URL); + std::string db_path = db_config.GetValue(CONFIG_DB_PATH); opt.meta.path = db_path + "/db"; - std::string db_slave_path = config.GetValue(CONFIG_DB_SLAVE_PATH); + std::string db_slave_path = db_config.GetValue(CONFIG_DB_SLAVE_PATH); StringHelpFunctions::SplitStringByDelimeter(db_slave_path, ";", opt.meta.slave_paths); - int64_t index_size = config.GetInt64Value(CONFIG_DB_INDEX_TRIGGER_SIZE); + int64_t index_size = db_config.GetInt64Value(CONFIG_DB_INDEX_TRIGGER_SIZE); if(index_size > 0) {//ensure larger than zero, unit is MB opt.index_trigger_size = (size_t)index_size * engine::ONE_MB; } - int64_t insert_buffer_size = config.GetInt64Value(CONFIG_DB_INSERT_BUFFER_SIZE, 4); + int64_t insert_buffer_size = db_config.GetInt64Value(CONFIG_DB_INSERT_BUFFER_SIZE, 4); if (insert_buffer_size >= 1) { opt.insert_buffer_size = insert_buffer_size * engine::ONE_GB; } @@ -37,6 +37,9 @@ DBWrapper::DBWrapper() { kill(0, SIGUSR1); } + ConfigNode& cache_config = ServerConfig::GetInstance().GetConfig(CONFIG_CACHE); + opt.insert_cache_immediately_ = cache_config.GetBoolValue(CONFIG_INSERT_CACHE_IMMEDIATELY, false); + ConfigNode& serverConfig = ServerConfig::GetInstance().GetConfig(CONFIG_SERVER); std::string mode = serverConfig.GetValue(CONFIG_CLUSTER_MODE, "single"); if (mode == "single") { @@ -55,8 +58,8 @@ DBWrapper::DBWrapper() { //set archive config engine::ArchiveConf::CriteriaT criterial; - int64_t disk = config.GetInt64Value(CONFIG_DB_ARCHIVE_DISK, 0); - int64_t days = config.GetInt64Value(CONFIG_DB_ARCHIVE_DAYS, 0); + int64_t disk = db_config.GetInt64Value(CONFIG_DB_ARCHIVE_DISK, 0); + int64_t days = db_config.GetInt64Value(CONFIG_DB_ARCHIVE_DAYS, 0); if(disk > 0) { criterial[engine::ARCHIVE_CONF_DISK] = disk; } diff --git a/cpp/src/server/MilvusServer.cpp b/cpp/src/server/MilvusServer.cpp index 92e06d0ccd..2f68c4b189 100644 --- a/cpp/src/server/MilvusServer.cpp +++ b/cpp/src/server/MilvusServer.cpp @@ -76,7 +76,7 @@ MilvusServer::StartService() { return; } - stdcxx::shared_ptr threadManager(ThreadManager::newSimpleThreadManager()); + stdcxx::shared_ptr threadManager(ThreadManager::newSimpleThreadManager(16)); stdcxx::shared_ptr threadFactory(new PosixThreadFactory()); threadManager->threadFactory(threadFactory); threadManager->start(); diff --git a/cpp/src/server/RequestHandler.cpp b/cpp/src/server/RequestHandler.cpp index 5ff6b1784a..5074043148 100644 --- a/cpp/src/server/RequestHandler.cpp +++ b/cpp/src/server/RequestHandler.cpp @@ -60,11 +60,22 @@ RequestHandler::SearchVector(std::vector &_return, const std::vector &query_range_array, const int64_t topk) { // SERVER_LOG_DEBUG << "Entering RequestHandler::SearchVector"; - BaseTaskPtr task_ptr = SearchVectorTask::Create(table_name, std::vector(), query_record_array, + BaseTaskPtr task_ptr = SearchVectorTask1::Create(table_name, std::vector(), query_record_array, query_range_array, topk, _return); RequestScheduler::ExecTask(task_ptr); } +void +RequestHandler::SearchVector2(std::vector & _return, + const std::string& table_name, + const std::vector & query_record_array, + const std::vector & query_range_array, + const int64_t topk) { + BaseTaskPtr task_ptr = SearchVectorTask2::Create(table_name, std::vector(), query_record_array, + query_range_array, topk, _return); + RequestScheduler::ExecTask(task_ptr); +} + void RequestHandler::SearchVectorInFiles(std::vector<::milvus::thrift::TopKQueryResult> &_return, const std::string& table_name, @@ -73,7 +84,7 @@ RequestHandler::SearchVectorInFiles(std::vector<::milvus::thrift::TopKQueryResul const std::vector<::milvus::thrift::Range> &query_range_array, const int64_t topk) { // SERVER_LOG_DEBUG << "Entering RequestHandler::SearchVectorInFiles. file_id_array size = " << std::to_string(file_id_array.size()); - BaseTaskPtr task_ptr = SearchVectorTask::Create(table_name, file_id_array, query_record_array, + BaseTaskPtr task_ptr = SearchVectorTask1::Create(table_name, file_id_array, query_record_array, query_range_array, topk, _return); RequestScheduler::ExecTask(task_ptr); } diff --git a/cpp/src/server/RequestHandler.h b/cpp/src/server/RequestHandler.h index aeda4e5ed6..a79830b512 100644 --- a/cpp/src/server/RequestHandler.h +++ b/cpp/src/server/RequestHandler.h @@ -106,6 +106,29 @@ public: const std::vector<::milvus::thrift::Range> & query_range_array, const int64_t topk); + /** + * @brief Query vector + * + * This method is used to query vector in table. + * + * @param table_name, table_name is queried. + * @param query_record_array, all vector are going to be queried. + * @param query_range_array, optional ranges for conditional search. If not specified, search whole table + * @param topk, how many similarity vectors will be searched. + * + * @return query binary result array. + * + * @param table_name + * @param query_record_array + * @param query_range_array + * @param topk + */ + void SearchVector2(std::vector<::milvus::thrift::TopKQueryBinResult> & _return, + const std::string& table_name, + const std::vector<::milvus::thrift::RowRecord> & query_record_array, + const std::vector<::milvus::thrift::Range> & query_range_array, + const int64_t topk); + /** * @brief Internal use query interface * diff --git a/cpp/src/server/RequestTask.cpp b/cpp/src/server/RequestTask.cpp index af6264da90..d6c1a2fb9c 100644 --- a/cpp/src/server/RequestTask.cpp +++ b/cpp/src/server/RequestTask.cpp @@ -12,6 +12,10 @@ #include "DBWrapper.h" #include "version.h" +#ifdef MILVUS_ENABLE_PROFILING +#include "gperftools/profiler.h" +#endif + namespace zilliz { namespace milvus { namespace server { @@ -127,6 +131,18 @@ namespace { } } } + + std::string + GetCurrTimeStr() { + char tm_buf[20] = {0}; + time_t tt; + time(&tt); + tt = tt + 8 * 60 * 60; + tm* t = gmtime(&tt); + sprintf(tm_buf, "%4d%02d%02d_%02d%02d%02d", (t->tm_year+1900), (t->tm_mon+1), (t->tm_mday), + (t->tm_hour), (t->tm_min), (t->tm_sec)); + return tm_buf; + } } //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -178,7 +194,7 @@ ServerError CreateTableTask::OnExecute() { return SetError(SERVER_UNEXPECTED_ERROR, ex.what()); } - rc.Record("done"); + rc.ElapseFromBegin("totally cost"); return SERVER_SUCCESS; } @@ -223,7 +239,7 @@ ServerError DescribeTableTask::OnExecute() { return SetError(SERVER_UNEXPECTED_ERROR, ex.what()); } - rc.Record("done"); + rc.ElapseFromBegin("totally cost"); return SERVER_SUCCESS; } @@ -261,7 +277,7 @@ ServerError BuildIndexTask::OnExecute() { return SetError(SERVER_BUILD_INDEX_ERROR, "Engine failed: " + stat.ToString()); } - rc.Elapse("totally cost"); + rc.ElapseFromBegin("totally cost"); } catch (std::exception& ex) { return SetError(SERVER_UNEXPECTED_ERROR, ex.what()); } @@ -298,7 +314,7 @@ ServerError HasTableTask::OnExecute() { return SetError(DB_META_TRANSACTION_FAILED, "Engine failed: " + stat.ToString()); } - rc.Elapse("totally cost"); + rc.ElapseFromBegin("totally cost"); } catch (std::exception& ex) { return SetError(SERVER_UNEXPECTED_ERROR, ex.what()); } @@ -340,8 +356,6 @@ ServerError DeleteTableTask::OnExecute() { } } - rc.Record("check validation"); - //step 3: delete table std::vector dates; stat = DBWrapper::DB()->DeleteTable(table_name_, dates); @@ -349,8 +363,7 @@ ServerError DeleteTableTask::OnExecute() { return SetError(DB_META_TRANSACTION_FAILED, "Engine failed: " + stat.ToString()); } - rc.Record("deleta table"); - rc.Elapse("total cost"); + rc.ElapseFromBegin("totally cost"); } catch (std::exception& ex) { return SetError(SERVER_UNEXPECTED_ERROR, ex.what()); } @@ -428,7 +441,13 @@ ServerError AddVectorTask::OnExecute() { } } - rc.Record("check validation"); + rc.RecordSection("check validation"); + +#ifdef MILVUS_ENABLE_PROFILING + std::string fname = "/tmp/insert_" + std::to_string(this->record_array_.size()) + + "_" + GetCurrTimeStr() + ".profiling"; + ProfilerStart(fname.c_str()); +#endif //step 3: prepare float data std::vector vec_f; @@ -439,12 +458,11 @@ ServerError AddVectorTask::OnExecute() { return SetError(error_code, error_msg); } - rc.Record("prepare vectors data"); + rc.RecordSection("prepare vectors data"); //step 4: insert vectors uint64_t vec_count = (uint64_t)record_array_.size(); stat = DBWrapper::DB()->InsertVectors(table_name_, vec_count, vec_f.data(), record_ids_); - rc.Record("add vectors to engine"); if(!stat.ok()) { return SetError(SERVER_CACHE_ERROR, "Cache error: " + stat.ToString()); } @@ -455,8 +473,12 @@ ServerError AddVectorTask::OnExecute() { return SetError(SERVER_ILLEGAL_VECTOR_ID, msg); } - rc.Record("do insert"); - rc.Elapse("total cost"); +#ifdef MILVUS_ENABLE_PROFILING + ProfilerStop(); +#endif + + rc.RecordSection("add vectors to engine"); + rc.ElapseFromBegin("totally cost"); } catch (std::exception& ex) { return SetError(SERVER_UNEXPECTED_ERROR, ex.what()); @@ -466,33 +488,21 @@ ServerError AddVectorTask::OnExecute() { } //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -SearchVectorTask::SearchVectorTask(const std::string &table_name, - const std::vector& file_id_array, - const std::vector &query_record_array, - const std::vector &query_range_array, - const int64_t top_k, - std::vector &result_array) +SearchVectorTaskBase::SearchVectorTaskBase(const std::string &table_name, + const std::vector& file_id_array, + const std::vector &query_record_array, + const std::vector &query_range_array, + const int64_t top_k) : BaseTask(DQL_TASK_GROUP), table_name_(table_name), file_id_array_(file_id_array), record_array_(query_record_array), range_array_(query_range_array), - top_k_(top_k), - result_array_(result_array) { + top_k_(top_k) { } -BaseTaskPtr SearchVectorTask::Create(const std::string& table_name, - const std::vector& file_id_array, - const std::vector & query_record_array, - const std::vector & query_range_array, - const int64_t top_k, - std::vector& result_array) { - return std::shared_ptr(new SearchVectorTask(table_name, file_id_array, - query_record_array, query_range_array, top_k, result_array)); -} - -ServerError SearchVectorTask::OnExecute() { +ServerError SearchVectorTaskBase::OnExecute() { try { TimeRecorder rc("SearchVectorTask"); @@ -503,7 +513,7 @@ ServerError SearchVectorTask::OnExecute() { return SetError(res, "Invalid table name: " + table_name_); } - if(top_k_ <= 0) { + if(top_k_ <= 0 || top_k_ > 1024) { return SetError(SERVER_INVALID_TOPK, "Invalid topk: " + std::to_string(top_k_)); } if(record_array_.empty()) { @@ -531,7 +541,14 @@ ServerError SearchVectorTask::OnExecute() { return SetError(error_code, error_msg); } - rc.Record("check validation"); + double span_check = rc.RecordSection("check validation"); + +#ifdef MILVUS_ENABLE_PROFILING + std::string fname = "/tmp/search_nq_" + std::to_string(this->record_array_.size()) + + "_top_" + std::to_string(this->top_k_) + "_" + + GetCurrTimeStr() + ".profiling"; + ProfilerStart(fname.c_str()); +#endif //step 3: prepare float data std::vector vec_f; @@ -540,7 +557,7 @@ ServerError SearchVectorTask::OnExecute() { return SetError(error_code, error_msg); } - rc.Record("prepare vector data"); + double span_prepare = rc.RecordSection("prepare vector data"); //step 4: search vectors engine::QueryResults results; @@ -552,7 +569,7 @@ ServerError SearchVectorTask::OnExecute() { stat = DBWrapper::DB()->Query(table_name_, file_id_array_, (size_t) top_k_, record_count, vec_f.data(), dates, results); } - rc.Record("search vectors from engine"); + double span_search = rc.RecordSection("search vectors from engine"); if(!stat.ok()) { return SetError(DB_META_TRANSACTION_FAILED, "Engine failed: " + stat.ToString()); } @@ -567,26 +584,22 @@ ServerError SearchVectorTask::OnExecute() { return SetError(SERVER_ILLEGAL_SEARCH_RESULT, msg); } - rc.Record("do search"); - //step 5: construct result array - for(uint64_t i = 0; i < record_count; i++) { - auto& result = results[i]; - const auto& record = record_array_[i]; + ConstructResult(results); - thrift::TopKQueryResult thrift_topk_result; - for(auto& pair : result) { - thrift::QueryResult thrift_result; - thrift_result.__set_id(pair.first); - thrift_result.__set_distance(pair.second); +#ifdef MILVUS_ENABLE_PROFILING + ProfilerStop(); +#endif - thrift_topk_result.query_result_arrays.emplace_back(thrift_result); - } + double span_result = rc.RecordSection("construct result"); + rc.ElapseFromBegin("totally cost"); - result_array_.emplace_back(thrift_topk_result); - } - rc.Record("construct result"); - rc.Elapse("total cost"); + //step 6: print time cost percent + double total_cost = span_check + span_prepare + span_search + span_result; + SERVER_LOG_DEBUG << "SearchVectorTask: " << "check validation(" << (span_check/total_cost)*100.0 << "%)" + << " prepare data(" << (span_prepare/total_cost)*100.0 << "%)" + << " search(" << (span_search/total_cost)*100.0 << "%)" + << " construct result(" << (span_result/total_cost)*100.0 << "%)"; } catch (std::exception& ex) { return SetError(SERVER_UNEXPECTED_ERROR, ex.what()); @@ -595,6 +608,100 @@ ServerError SearchVectorTask::OnExecute() { return SERVER_SUCCESS; } +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +SearchVectorTask1::SearchVectorTask1(const std::string &table_name, + const std::vector& file_id_array, + const std::vector &query_record_array, + const std::vector &query_range_array, + const int64_t top_k, + std::vector &result_array) + : SearchVectorTaskBase(table_name, file_id_array, query_record_array, query_range_array, top_k), + result_array_(result_array) { + +} + +BaseTaskPtr SearchVectorTask1::Create(const std::string& table_name, + const std::vector& file_id_array, + const std::vector & query_record_array, + const std::vector & query_range_array, + const int64_t top_k, + std::vector& result_array) { + return std::shared_ptr(new SearchVectorTask1(table_name, file_id_array, + query_record_array, query_range_array, top_k, result_array)); +} + +ServerError SearchVectorTask1::ConstructResult(engine::QueryResults& results) { + for(uint64_t i = 0; i < results.size(); i++) { + auto& result = results[i]; + const auto& record = record_array_[i]; + + thrift::TopKQueryResult thrift_topk_result; + for(auto& pair : result) { + thrift::QueryResult thrift_result; + thrift_result.__set_id(pair.first); + thrift_result.__set_distance(pair.second); + + thrift_topk_result.query_result_arrays.emplace_back(thrift_result); + } + + result_array_.emplace_back(thrift_topk_result); + } + + return SERVER_SUCCESS; +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +SearchVectorTask2::SearchVectorTask2(const std::string &table_name, + const std::vector& file_id_array, + const std::vector &query_record_array, + const std::vector &query_range_array, + const int64_t top_k, + std::vector &result_array) + : SearchVectorTaskBase(table_name, file_id_array, query_record_array, query_range_array, top_k), + result_array_(result_array) { + +} + +BaseTaskPtr SearchVectorTask2::Create(const std::string& table_name, + const std::vector& file_id_array, + const std::vector & query_record_array, + const std::vector & query_range_array, + const int64_t top_k, + std::vector& result_array) { + return std::shared_ptr(new SearchVectorTask2(table_name, file_id_array, + query_record_array, query_range_array, top_k, result_array)); +} + +ServerError SearchVectorTask2::ConstructResult(engine::QueryResults& results) { + for(size_t i = 0; i < results.size(); i++) { + auto& result = results[i]; + + thrift::TopKQueryBinResult thrift_topk_result; + if(result.empty()) { + result_array_.emplace_back(thrift_topk_result); + continue; + } + + std::string str_ids, str_distances; + str_ids.resize(sizeof(engine::IDNumber)*result.size()); + str_distances.resize(sizeof(double)*result.size()); + + engine::IDNumber* ids_ptr = (engine::IDNumber*)str_ids.data(); + double* distance_ptr = (double*)str_distances.data(); + for(size_t k = 0; k < result.size(); k++) { + auto& pair = result[k]; + ids_ptr[k] = pair.first; + distance_ptr[k] = pair.second; + } + + thrift_topk_result.__set_id_array(str_ids); + thrift_topk_result.__set_distance_array(str_distances); + result_array_.emplace_back(thrift_topk_result); + } + + return SERVER_SUCCESS; +} + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// GetTableRowCountTask::GetTableRowCountTask(const std::string& table_name, int64_t& row_count) : BaseTask(DDL_DML_TASK_GROUP), @@ -627,7 +734,7 @@ ServerError GetTableRowCountTask::OnExecute() { row_count_ = (int64_t) row_count; - rc.Elapse("total cost"); + rc.ElapseFromBegin("totally cost"); } catch (std::exception& ex) { return SetError(SERVER_UNEXPECTED_ERROR, ex.what()); diff --git a/cpp/src/server/RequestTask.h b/cpp/src/server/RequestTask.h index 37c04272fa..4a95ef5b61 100644 --- a/cpp/src/server/RequestTask.h +++ b/cpp/src/server/RequestTask.h @@ -129,7 +129,28 @@ private: }; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -class SearchVectorTask : public BaseTask { +class SearchVectorTaskBase : public BaseTask { +protected: + SearchVectorTaskBase(const std::string& table_name, + const std::vector& file_id_array, + const std::vector<::milvus::thrift::RowRecord> & query_record_array, + const std::vector<::milvus::thrift::Range> & query_range_array, + const int64_t top_k); + + ServerError OnExecute() override; + + virtual ServerError ConstructResult(engine::QueryResults& results) = 0; + +protected: + std::string table_name_; + std::vector file_id_array_; + int64_t top_k_; + const std::vector<::milvus::thrift::RowRecord>& record_array_; + const std::vector<::milvus::thrift::Range>& range_array_; +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +class SearchVectorTask1 : public SearchVectorTaskBase { public: static BaseTaskPtr Create(const std::string& table_name, const std::vector& file_id_array, @@ -139,24 +160,43 @@ public: std::vector<::milvus::thrift::TopKQueryResult>& result_array); protected: - SearchVectorTask(const std::string& table_name, - const std::vector& file_id_array, - const std::vector<::milvus::thrift::RowRecord> & query_record_array, - const std::vector<::milvus::thrift::Range> & query_range_array, - const int64_t top_k, + SearchVectorTask1(const std::string& table_name, + const std::vector& file_id_array, + const std::vector<::milvus::thrift::RowRecord> & query_record_array, + const std::vector<::milvus::thrift::Range> & query_range_array, + const int64_t top_k, std::vector<::milvus::thrift::TopKQueryResult>& result_array); - ServerError OnExecute() override; + ServerError ConstructResult(engine::QueryResults& results) override; private: - std::string table_name_; - std::vector file_id_array_; - int64_t top_k_; - const std::vector<::milvus::thrift::RowRecord>& record_array_; - const std::vector<::milvus::thrift::Range>& range_array_; std::vector<::milvus::thrift::TopKQueryResult>& result_array_; }; +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +class SearchVectorTask2 : public SearchVectorTaskBase { +public: + static BaseTaskPtr Create(const std::string& table_name, + const std::vector& file_id_array, + const std::vector<::milvus::thrift::RowRecord> & query_record_array, + const std::vector<::milvus::thrift::Range> & query_range_array, + const int64_t top_k, + std::vector<::milvus::thrift::TopKQueryBinResult>& result_array); + +protected: + SearchVectorTask2(const std::string& table_name, + const std::vector& file_id_array, + const std::vector<::milvus::thrift::RowRecord> & query_record_array, + const std::vector<::milvus::thrift::Range> & query_range_array, + const int64_t top_k, + std::vector<::milvus::thrift::TopKQueryBinResult>& result_array); + + ServerError ConstructResult(engine::QueryResults& results) override; + +private: + std::vector<::milvus::thrift::TopKQueryBinResult>& result_array_; +}; + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// class GetTableRowCountTask : public BaseTask { public: diff --git a/cpp/src/server/ServerConfig.h b/cpp/src/server/ServerConfig.h index bc202adcf6..12d1e8b889 100644 --- a/cpp/src/server/ServerConfig.h +++ b/cpp/src/server/ServerConfig.h @@ -34,6 +34,8 @@ static const std::string CONFIG_LOG = "log_config"; static const std::string CONFIG_CACHE = "cache_config"; static const std::string CONFIG_CPU_CACHE_CAPACITY = "cpu_cache_capacity"; static const std::string CONFIG_GPU_CACHE_CAPACITY = "gpu_cache_capacity"; +static const std::string CACHE_FREE_PERCENT = "cache_free_percent"; +static const std::string CONFIG_INSERT_CACHE_IMMEDIATELY = "insert_cache_immediately"; static const std::string CONFIG_LICENSE = "license_config"; static const std::string CONFIG_LICENSE_PATH = "license_path"; diff --git a/cpp/src/thrift/gen-cpp/MilvusService.cpp b/cpp/src/thrift/gen-cpp/MilvusService.cpp index fb66b0ecbe..3420b92b5d 100644 --- a/cpp/src/thrift/gen-cpp/MilvusService.cpp +++ b/cpp/src/thrift/gen-cpp/MilvusService.cpp @@ -814,14 +814,14 @@ uint32_t MilvusService_AddVector_args::read(::apache::thrift::protocol::TProtoco if (ftype == ::apache::thrift::protocol::T_LIST) { { this->record_array.clear(); - uint32_t _size19; - ::apache::thrift::protocol::TType _etype22; - xfer += iprot->readListBegin(_etype22, _size19); - this->record_array.resize(_size19); - uint32_t _i23; - for (_i23 = 0; _i23 < _size19; ++_i23) + uint32_t _size21; + ::apache::thrift::protocol::TType _etype24; + xfer += iprot->readListBegin(_etype24, _size21); + this->record_array.resize(_size21); + uint32_t _i25; + for (_i25 = 0; _i25 < _size21; ++_i25) { - xfer += this->record_array[_i23].read(iprot); + xfer += this->record_array[_i25].read(iprot); } xfer += iprot->readListEnd(); } @@ -854,10 +854,10 @@ uint32_t MilvusService_AddVector_args::write(::apache::thrift::protocol::TProtoc xfer += oprot->writeFieldBegin("record_array", ::apache::thrift::protocol::T_LIST, 3); { xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRUCT, static_cast(this->record_array.size())); - std::vector ::const_iterator _iter24; - for (_iter24 = this->record_array.begin(); _iter24 != this->record_array.end(); ++_iter24) + std::vector ::const_iterator _iter26; + for (_iter26 = this->record_array.begin(); _iter26 != this->record_array.end(); ++_iter26) { - xfer += (*_iter24).write(oprot); + xfer += (*_iter26).write(oprot); } xfer += oprot->writeListEnd(); } @@ -885,10 +885,10 @@ uint32_t MilvusService_AddVector_pargs::write(::apache::thrift::protocol::TProto xfer += oprot->writeFieldBegin("record_array", ::apache::thrift::protocol::T_LIST, 3); { xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRUCT, static_cast((*(this->record_array)).size())); - std::vector ::const_iterator _iter25; - for (_iter25 = (*(this->record_array)).begin(); _iter25 != (*(this->record_array)).end(); ++_iter25) + std::vector ::const_iterator _iter27; + for (_iter27 = (*(this->record_array)).begin(); _iter27 != (*(this->record_array)).end(); ++_iter27) { - xfer += (*_iter25).write(oprot); + xfer += (*_iter27).write(oprot); } xfer += oprot->writeListEnd(); } @@ -929,14 +929,14 @@ uint32_t MilvusService_AddVector_result::read(::apache::thrift::protocol::TProto if (ftype == ::apache::thrift::protocol::T_LIST) { { this->success.clear(); - uint32_t _size26; - ::apache::thrift::protocol::TType _etype29; - xfer += iprot->readListBegin(_etype29, _size26); - this->success.resize(_size26); - uint32_t _i30; - for (_i30 = 0; _i30 < _size26; ++_i30) + uint32_t _size28; + ::apache::thrift::protocol::TType _etype31; + xfer += iprot->readListBegin(_etype31, _size28); + this->success.resize(_size28); + uint32_t _i32; + for (_i32 = 0; _i32 < _size28; ++_i32) { - xfer += iprot->readI64(this->success[_i30]); + xfer += iprot->readI64(this->success[_i32]); } xfer += iprot->readListEnd(); } @@ -975,10 +975,10 @@ uint32_t MilvusService_AddVector_result::write(::apache::thrift::protocol::TProt xfer += oprot->writeFieldBegin("success", ::apache::thrift::protocol::T_LIST, 0); { xfer += oprot->writeListBegin(::apache::thrift::protocol::T_I64, static_cast(this->success.size())); - std::vector ::const_iterator _iter31; - for (_iter31 = this->success.begin(); _iter31 != this->success.end(); ++_iter31) + std::vector ::const_iterator _iter33; + for (_iter33 = this->success.begin(); _iter33 != this->success.end(); ++_iter33) { - xfer += oprot->writeI64((*_iter31)); + xfer += oprot->writeI64((*_iter33)); } xfer += oprot->writeListEnd(); } @@ -1023,14 +1023,14 @@ uint32_t MilvusService_AddVector_presult::read(::apache::thrift::protocol::TProt if (ftype == ::apache::thrift::protocol::T_LIST) { { (*(this->success)).clear(); - uint32_t _size32; - ::apache::thrift::protocol::TType _etype35; - xfer += iprot->readListBegin(_etype35, _size32); - (*(this->success)).resize(_size32); - uint32_t _i36; - for (_i36 = 0; _i36 < _size32; ++_i36) + uint32_t _size34; + ::apache::thrift::protocol::TType _etype37; + xfer += iprot->readListBegin(_etype37, _size34); + (*(this->success)).resize(_size34); + uint32_t _i38; + for (_i38 = 0; _i38 < _size34; ++_i38) { - xfer += iprot->readI64((*(this->success))[_i36]); + xfer += iprot->readI64((*(this->success))[_i38]); } xfer += iprot->readListEnd(); } @@ -1097,14 +1097,14 @@ uint32_t MilvusService_SearchVector_args::read(::apache::thrift::protocol::TProt if (ftype == ::apache::thrift::protocol::T_LIST) { { this->query_record_array.clear(); - uint32_t _size37; - ::apache::thrift::protocol::TType _etype40; - xfer += iprot->readListBegin(_etype40, _size37); - this->query_record_array.resize(_size37); - uint32_t _i41; - for (_i41 = 0; _i41 < _size37; ++_i41) + uint32_t _size39; + ::apache::thrift::protocol::TType _etype42; + xfer += iprot->readListBegin(_etype42, _size39); + this->query_record_array.resize(_size39); + uint32_t _i43; + for (_i43 = 0; _i43 < _size39; ++_i43) { - xfer += this->query_record_array[_i41].read(iprot); + xfer += this->query_record_array[_i43].read(iprot); } xfer += iprot->readListEnd(); } @@ -1117,14 +1117,14 @@ uint32_t MilvusService_SearchVector_args::read(::apache::thrift::protocol::TProt if (ftype == ::apache::thrift::protocol::T_LIST) { { this->query_range_array.clear(); - uint32_t _size42; - ::apache::thrift::protocol::TType _etype45; - xfer += iprot->readListBegin(_etype45, _size42); - this->query_range_array.resize(_size42); - uint32_t _i46; - for (_i46 = 0; _i46 < _size42; ++_i46) + uint32_t _size44; + ::apache::thrift::protocol::TType _etype47; + xfer += iprot->readListBegin(_etype47, _size44); + this->query_range_array.resize(_size44); + uint32_t _i48; + for (_i48 = 0; _i48 < _size44; ++_i48) { - xfer += this->query_range_array[_i46].read(iprot); + xfer += this->query_range_array[_i48].read(iprot); } xfer += iprot->readListEnd(); } @@ -1165,10 +1165,10 @@ uint32_t MilvusService_SearchVector_args::write(::apache::thrift::protocol::TPro xfer += oprot->writeFieldBegin("query_record_array", ::apache::thrift::protocol::T_LIST, 3); { xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRUCT, static_cast(this->query_record_array.size())); - std::vector ::const_iterator _iter47; - for (_iter47 = this->query_record_array.begin(); _iter47 != this->query_record_array.end(); ++_iter47) + std::vector ::const_iterator _iter49; + for (_iter49 = this->query_record_array.begin(); _iter49 != this->query_record_array.end(); ++_iter49) { - xfer += (*_iter47).write(oprot); + xfer += (*_iter49).write(oprot); } xfer += oprot->writeListEnd(); } @@ -1177,10 +1177,10 @@ uint32_t MilvusService_SearchVector_args::write(::apache::thrift::protocol::TPro xfer += oprot->writeFieldBegin("query_range_array", ::apache::thrift::protocol::T_LIST, 4); { xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRUCT, static_cast(this->query_range_array.size())); - std::vector ::const_iterator _iter48; - for (_iter48 = this->query_range_array.begin(); _iter48 != this->query_range_array.end(); ++_iter48) + std::vector ::const_iterator _iter50; + for (_iter50 = this->query_range_array.begin(); _iter50 != this->query_range_array.end(); ++_iter50) { - xfer += (*_iter48).write(oprot); + xfer += (*_iter50).write(oprot); } xfer += oprot->writeListEnd(); } @@ -1212,10 +1212,10 @@ uint32_t MilvusService_SearchVector_pargs::write(::apache::thrift::protocol::TPr xfer += oprot->writeFieldBegin("query_record_array", ::apache::thrift::protocol::T_LIST, 3); { xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRUCT, static_cast((*(this->query_record_array)).size())); - std::vector ::const_iterator _iter49; - for (_iter49 = (*(this->query_record_array)).begin(); _iter49 != (*(this->query_record_array)).end(); ++_iter49) + std::vector ::const_iterator _iter51; + for (_iter51 = (*(this->query_record_array)).begin(); _iter51 != (*(this->query_record_array)).end(); ++_iter51) { - xfer += (*_iter49).write(oprot); + xfer += (*_iter51).write(oprot); } xfer += oprot->writeListEnd(); } @@ -1224,10 +1224,10 @@ uint32_t MilvusService_SearchVector_pargs::write(::apache::thrift::protocol::TPr xfer += oprot->writeFieldBegin("query_range_array", ::apache::thrift::protocol::T_LIST, 4); { xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRUCT, static_cast((*(this->query_range_array)).size())); - std::vector ::const_iterator _iter50; - for (_iter50 = (*(this->query_range_array)).begin(); _iter50 != (*(this->query_range_array)).end(); ++_iter50) + std::vector ::const_iterator _iter52; + for (_iter52 = (*(this->query_range_array)).begin(); _iter52 != (*(this->query_range_array)).end(); ++_iter52) { - xfer += (*_iter50).write(oprot); + xfer += (*_iter52).write(oprot); } xfer += oprot->writeListEnd(); } @@ -1272,14 +1272,14 @@ uint32_t MilvusService_SearchVector_result::read(::apache::thrift::protocol::TPr if (ftype == ::apache::thrift::protocol::T_LIST) { { this->success.clear(); - uint32_t _size51; - ::apache::thrift::protocol::TType _etype54; - xfer += iprot->readListBegin(_etype54, _size51); - this->success.resize(_size51); - uint32_t _i55; - for (_i55 = 0; _i55 < _size51; ++_i55) + uint32_t _size53; + ::apache::thrift::protocol::TType _etype56; + xfer += iprot->readListBegin(_etype56, _size53); + this->success.resize(_size53); + uint32_t _i57; + for (_i57 = 0; _i57 < _size53; ++_i57) { - xfer += this->success[_i55].read(iprot); + xfer += this->success[_i57].read(iprot); } xfer += iprot->readListEnd(); } @@ -1318,10 +1318,10 @@ uint32_t MilvusService_SearchVector_result::write(::apache::thrift::protocol::TP xfer += oprot->writeFieldBegin("success", ::apache::thrift::protocol::T_LIST, 0); { xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRUCT, static_cast(this->success.size())); - std::vector ::const_iterator _iter56; - for (_iter56 = this->success.begin(); _iter56 != this->success.end(); ++_iter56) + std::vector ::const_iterator _iter58; + for (_iter58 = this->success.begin(); _iter58 != this->success.end(); ++_iter58) { - xfer += (*_iter56).write(oprot); + xfer += (*_iter58).write(oprot); } xfer += oprot->writeListEnd(); } @@ -1366,14 +1366,357 @@ uint32_t MilvusService_SearchVector_presult::read(::apache::thrift::protocol::TP if (ftype == ::apache::thrift::protocol::T_LIST) { { (*(this->success)).clear(); - uint32_t _size57; - ::apache::thrift::protocol::TType _etype60; - xfer += iprot->readListBegin(_etype60, _size57); - (*(this->success)).resize(_size57); - uint32_t _i61; - for (_i61 = 0; _i61 < _size57; ++_i61) + uint32_t _size59; + ::apache::thrift::protocol::TType _etype62; + xfer += iprot->readListBegin(_etype62, _size59); + (*(this->success)).resize(_size59); + uint32_t _i63; + for (_i63 = 0; _i63 < _size59; ++_i63) { - xfer += (*(this->success))[_i61].read(iprot); + xfer += (*(this->success))[_i63].read(iprot); + } + xfer += iprot->readListEnd(); + } + this->__isset.success = true; + } else { + xfer += iprot->skip(ftype); + } + break; + case 1: + if (ftype == ::apache::thrift::protocol::T_STRUCT) { + xfer += this->e.read(iprot); + this->__isset.e = true; + } else { + xfer += iprot->skip(ftype); + } + break; + default: + xfer += iprot->skip(ftype); + break; + } + xfer += iprot->readFieldEnd(); + } + + xfer += iprot->readStructEnd(); + + return xfer; +} + + +MilvusService_SearchVector2_args::~MilvusService_SearchVector2_args() throw() { +} + + +uint32_t MilvusService_SearchVector2_args::read(::apache::thrift::protocol::TProtocol* iprot) { + + ::apache::thrift::protocol::TInputRecursionTracker tracker(*iprot); + uint32_t xfer = 0; + std::string fname; + ::apache::thrift::protocol::TType ftype; + int16_t fid; + + xfer += iprot->readStructBegin(fname); + + using ::apache::thrift::protocol::TProtocolException; + + + while (true) + { + xfer += iprot->readFieldBegin(fname, ftype, fid); + if (ftype == ::apache::thrift::protocol::T_STOP) { + break; + } + switch (fid) + { + case 2: + if (ftype == ::apache::thrift::protocol::T_STRING) { + xfer += iprot->readString(this->table_name); + this->__isset.table_name = true; + } else { + xfer += iprot->skip(ftype); + } + break; + case 3: + if (ftype == ::apache::thrift::protocol::T_LIST) { + { + this->query_record_array.clear(); + uint32_t _size64; + ::apache::thrift::protocol::TType _etype67; + xfer += iprot->readListBegin(_etype67, _size64); + this->query_record_array.resize(_size64); + uint32_t _i68; + for (_i68 = 0; _i68 < _size64; ++_i68) + { + xfer += this->query_record_array[_i68].read(iprot); + } + xfer += iprot->readListEnd(); + } + this->__isset.query_record_array = true; + } else { + xfer += iprot->skip(ftype); + } + break; + case 4: + if (ftype == ::apache::thrift::protocol::T_LIST) { + { + this->query_range_array.clear(); + uint32_t _size69; + ::apache::thrift::protocol::TType _etype72; + xfer += iprot->readListBegin(_etype72, _size69); + this->query_range_array.resize(_size69); + uint32_t _i73; + for (_i73 = 0; _i73 < _size69; ++_i73) + { + xfer += this->query_range_array[_i73].read(iprot); + } + xfer += iprot->readListEnd(); + } + this->__isset.query_range_array = true; + } else { + xfer += iprot->skip(ftype); + } + break; + case 5: + if (ftype == ::apache::thrift::protocol::T_I64) { + xfer += iprot->readI64(this->topk); + this->__isset.topk = true; + } else { + xfer += iprot->skip(ftype); + } + break; + default: + xfer += iprot->skip(ftype); + break; + } + xfer += iprot->readFieldEnd(); + } + + xfer += iprot->readStructEnd(); + + return xfer; +} + +uint32_t MilvusService_SearchVector2_args::write(::apache::thrift::protocol::TProtocol* oprot) const { + uint32_t xfer = 0; + ::apache::thrift::protocol::TOutputRecursionTracker tracker(*oprot); + xfer += oprot->writeStructBegin("MilvusService_SearchVector2_args"); + + xfer += oprot->writeFieldBegin("table_name", ::apache::thrift::protocol::T_STRING, 2); + xfer += oprot->writeString(this->table_name); + xfer += oprot->writeFieldEnd(); + + xfer += oprot->writeFieldBegin("query_record_array", ::apache::thrift::protocol::T_LIST, 3); + { + xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRUCT, static_cast(this->query_record_array.size())); + std::vector ::const_iterator _iter74; + for (_iter74 = this->query_record_array.begin(); _iter74 != this->query_record_array.end(); ++_iter74) + { + xfer += (*_iter74).write(oprot); + } + xfer += oprot->writeListEnd(); + } + xfer += oprot->writeFieldEnd(); + + xfer += oprot->writeFieldBegin("query_range_array", ::apache::thrift::protocol::T_LIST, 4); + { + xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRUCT, static_cast(this->query_range_array.size())); + std::vector ::const_iterator _iter75; + for (_iter75 = this->query_range_array.begin(); _iter75 != this->query_range_array.end(); ++_iter75) + { + xfer += (*_iter75).write(oprot); + } + xfer += oprot->writeListEnd(); + } + xfer += oprot->writeFieldEnd(); + + xfer += oprot->writeFieldBegin("topk", ::apache::thrift::protocol::T_I64, 5); + xfer += oprot->writeI64(this->topk); + xfer += oprot->writeFieldEnd(); + + xfer += oprot->writeFieldStop(); + xfer += oprot->writeStructEnd(); + return xfer; +} + + +MilvusService_SearchVector2_pargs::~MilvusService_SearchVector2_pargs() throw() { +} + + +uint32_t MilvusService_SearchVector2_pargs::write(::apache::thrift::protocol::TProtocol* oprot) const { + uint32_t xfer = 0; + ::apache::thrift::protocol::TOutputRecursionTracker tracker(*oprot); + xfer += oprot->writeStructBegin("MilvusService_SearchVector2_pargs"); + + xfer += oprot->writeFieldBegin("table_name", ::apache::thrift::protocol::T_STRING, 2); + xfer += oprot->writeString((*(this->table_name))); + xfer += oprot->writeFieldEnd(); + + xfer += oprot->writeFieldBegin("query_record_array", ::apache::thrift::protocol::T_LIST, 3); + { + xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRUCT, static_cast((*(this->query_record_array)).size())); + std::vector ::const_iterator _iter76; + for (_iter76 = (*(this->query_record_array)).begin(); _iter76 != (*(this->query_record_array)).end(); ++_iter76) + { + xfer += (*_iter76).write(oprot); + } + xfer += oprot->writeListEnd(); + } + xfer += oprot->writeFieldEnd(); + + xfer += oprot->writeFieldBegin("query_range_array", ::apache::thrift::protocol::T_LIST, 4); + { + xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRUCT, static_cast((*(this->query_range_array)).size())); + std::vector ::const_iterator _iter77; + for (_iter77 = (*(this->query_range_array)).begin(); _iter77 != (*(this->query_range_array)).end(); ++_iter77) + { + xfer += (*_iter77).write(oprot); + } + xfer += oprot->writeListEnd(); + } + xfer += oprot->writeFieldEnd(); + + xfer += oprot->writeFieldBegin("topk", ::apache::thrift::protocol::T_I64, 5); + xfer += oprot->writeI64((*(this->topk))); + xfer += oprot->writeFieldEnd(); + + xfer += oprot->writeFieldStop(); + xfer += oprot->writeStructEnd(); + return xfer; +} + + +MilvusService_SearchVector2_result::~MilvusService_SearchVector2_result() throw() { +} + + +uint32_t MilvusService_SearchVector2_result::read(::apache::thrift::protocol::TProtocol* iprot) { + + ::apache::thrift::protocol::TInputRecursionTracker tracker(*iprot); + uint32_t xfer = 0; + std::string fname; + ::apache::thrift::protocol::TType ftype; + int16_t fid; + + xfer += iprot->readStructBegin(fname); + + using ::apache::thrift::protocol::TProtocolException; + + + while (true) + { + xfer += iprot->readFieldBegin(fname, ftype, fid); + if (ftype == ::apache::thrift::protocol::T_STOP) { + break; + } + switch (fid) + { + case 0: + if (ftype == ::apache::thrift::protocol::T_LIST) { + { + this->success.clear(); + uint32_t _size78; + ::apache::thrift::protocol::TType _etype81; + xfer += iprot->readListBegin(_etype81, _size78); + this->success.resize(_size78); + uint32_t _i82; + for (_i82 = 0; _i82 < _size78; ++_i82) + { + xfer += this->success[_i82].read(iprot); + } + xfer += iprot->readListEnd(); + } + this->__isset.success = true; + } else { + xfer += iprot->skip(ftype); + } + break; + case 1: + if (ftype == ::apache::thrift::protocol::T_STRUCT) { + xfer += this->e.read(iprot); + this->__isset.e = true; + } else { + xfer += iprot->skip(ftype); + } + break; + default: + xfer += iprot->skip(ftype); + break; + } + xfer += iprot->readFieldEnd(); + } + + xfer += iprot->readStructEnd(); + + return xfer; +} + +uint32_t MilvusService_SearchVector2_result::write(::apache::thrift::protocol::TProtocol* oprot) const { + + uint32_t xfer = 0; + + xfer += oprot->writeStructBegin("MilvusService_SearchVector2_result"); + + if (this->__isset.success) { + xfer += oprot->writeFieldBegin("success", ::apache::thrift::protocol::T_LIST, 0); + { + xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRUCT, static_cast(this->success.size())); + std::vector ::const_iterator _iter83; + for (_iter83 = this->success.begin(); _iter83 != this->success.end(); ++_iter83) + { + xfer += (*_iter83).write(oprot); + } + xfer += oprot->writeListEnd(); + } + xfer += oprot->writeFieldEnd(); + } else if (this->__isset.e) { + xfer += oprot->writeFieldBegin("e", ::apache::thrift::protocol::T_STRUCT, 1); + xfer += this->e.write(oprot); + xfer += oprot->writeFieldEnd(); + } + xfer += oprot->writeFieldStop(); + xfer += oprot->writeStructEnd(); + return xfer; +} + + +MilvusService_SearchVector2_presult::~MilvusService_SearchVector2_presult() throw() { +} + + +uint32_t MilvusService_SearchVector2_presult::read(::apache::thrift::protocol::TProtocol* iprot) { + + ::apache::thrift::protocol::TInputRecursionTracker tracker(*iprot); + uint32_t xfer = 0; + std::string fname; + ::apache::thrift::protocol::TType ftype; + int16_t fid; + + xfer += iprot->readStructBegin(fname); + + using ::apache::thrift::protocol::TProtocolException; + + + while (true) + { + xfer += iprot->readFieldBegin(fname, ftype, fid); + if (ftype == ::apache::thrift::protocol::T_STOP) { + break; + } + switch (fid) + { + case 0: + if (ftype == ::apache::thrift::protocol::T_LIST) { + { + (*(this->success)).clear(); + uint32_t _size84; + ::apache::thrift::protocol::TType _etype87; + xfer += iprot->readListBegin(_etype87, _size84); + (*(this->success)).resize(_size84); + uint32_t _i88; + for (_i88 = 0; _i88 < _size84; ++_i88) + { + xfer += (*(this->success))[_i88].read(iprot); } xfer += iprot->readListEnd(); } @@ -1440,14 +1783,14 @@ uint32_t MilvusService_SearchVectorInFiles_args::read(::apache::thrift::protocol if (ftype == ::apache::thrift::protocol::T_LIST) { { this->file_id_array.clear(); - uint32_t _size62; - ::apache::thrift::protocol::TType _etype65; - xfer += iprot->readListBegin(_etype65, _size62); - this->file_id_array.resize(_size62); - uint32_t _i66; - for (_i66 = 0; _i66 < _size62; ++_i66) + uint32_t _size89; + ::apache::thrift::protocol::TType _etype92; + xfer += iprot->readListBegin(_etype92, _size89); + this->file_id_array.resize(_size89); + uint32_t _i93; + for (_i93 = 0; _i93 < _size89; ++_i93) { - xfer += iprot->readString(this->file_id_array[_i66]); + xfer += iprot->readString(this->file_id_array[_i93]); } xfer += iprot->readListEnd(); } @@ -1460,14 +1803,14 @@ uint32_t MilvusService_SearchVectorInFiles_args::read(::apache::thrift::protocol if (ftype == ::apache::thrift::protocol::T_LIST) { { this->query_record_array.clear(); - uint32_t _size67; - ::apache::thrift::protocol::TType _etype70; - xfer += iprot->readListBegin(_etype70, _size67); - this->query_record_array.resize(_size67); - uint32_t _i71; - for (_i71 = 0; _i71 < _size67; ++_i71) + uint32_t _size94; + ::apache::thrift::protocol::TType _etype97; + xfer += iprot->readListBegin(_etype97, _size94); + this->query_record_array.resize(_size94); + uint32_t _i98; + for (_i98 = 0; _i98 < _size94; ++_i98) { - xfer += this->query_record_array[_i71].read(iprot); + xfer += this->query_record_array[_i98].read(iprot); } xfer += iprot->readListEnd(); } @@ -1480,14 +1823,14 @@ uint32_t MilvusService_SearchVectorInFiles_args::read(::apache::thrift::protocol if (ftype == ::apache::thrift::protocol::T_LIST) { { this->query_range_array.clear(); - uint32_t _size72; - ::apache::thrift::protocol::TType _etype75; - xfer += iprot->readListBegin(_etype75, _size72); - this->query_range_array.resize(_size72); - uint32_t _i76; - for (_i76 = 0; _i76 < _size72; ++_i76) + uint32_t _size99; + ::apache::thrift::protocol::TType _etype102; + xfer += iprot->readListBegin(_etype102, _size99); + this->query_range_array.resize(_size99); + uint32_t _i103; + for (_i103 = 0; _i103 < _size99; ++_i103) { - xfer += this->query_range_array[_i76].read(iprot); + xfer += this->query_range_array[_i103].read(iprot); } xfer += iprot->readListEnd(); } @@ -1528,10 +1871,10 @@ uint32_t MilvusService_SearchVectorInFiles_args::write(::apache::thrift::protoco xfer += oprot->writeFieldBegin("file_id_array", ::apache::thrift::protocol::T_LIST, 3); { xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRING, static_cast(this->file_id_array.size())); - std::vector ::const_iterator _iter77; - for (_iter77 = this->file_id_array.begin(); _iter77 != this->file_id_array.end(); ++_iter77) + std::vector ::const_iterator _iter104; + for (_iter104 = this->file_id_array.begin(); _iter104 != this->file_id_array.end(); ++_iter104) { - xfer += oprot->writeString((*_iter77)); + xfer += oprot->writeString((*_iter104)); } xfer += oprot->writeListEnd(); } @@ -1540,10 +1883,10 @@ uint32_t MilvusService_SearchVectorInFiles_args::write(::apache::thrift::protoco xfer += oprot->writeFieldBegin("query_record_array", ::apache::thrift::protocol::T_LIST, 4); { xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRUCT, static_cast(this->query_record_array.size())); - std::vector ::const_iterator _iter78; - for (_iter78 = this->query_record_array.begin(); _iter78 != this->query_record_array.end(); ++_iter78) + std::vector ::const_iterator _iter105; + for (_iter105 = this->query_record_array.begin(); _iter105 != this->query_record_array.end(); ++_iter105) { - xfer += (*_iter78).write(oprot); + xfer += (*_iter105).write(oprot); } xfer += oprot->writeListEnd(); } @@ -1552,10 +1895,10 @@ uint32_t MilvusService_SearchVectorInFiles_args::write(::apache::thrift::protoco xfer += oprot->writeFieldBegin("query_range_array", ::apache::thrift::protocol::T_LIST, 5); { xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRUCT, static_cast(this->query_range_array.size())); - std::vector ::const_iterator _iter79; - for (_iter79 = this->query_range_array.begin(); _iter79 != this->query_range_array.end(); ++_iter79) + std::vector ::const_iterator _iter106; + for (_iter106 = this->query_range_array.begin(); _iter106 != this->query_range_array.end(); ++_iter106) { - xfer += (*_iter79).write(oprot); + xfer += (*_iter106).write(oprot); } xfer += oprot->writeListEnd(); } @@ -1587,10 +1930,10 @@ uint32_t MilvusService_SearchVectorInFiles_pargs::write(::apache::thrift::protoc xfer += oprot->writeFieldBegin("file_id_array", ::apache::thrift::protocol::T_LIST, 3); { xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRING, static_cast((*(this->file_id_array)).size())); - std::vector ::const_iterator _iter80; - for (_iter80 = (*(this->file_id_array)).begin(); _iter80 != (*(this->file_id_array)).end(); ++_iter80) + std::vector ::const_iterator _iter107; + for (_iter107 = (*(this->file_id_array)).begin(); _iter107 != (*(this->file_id_array)).end(); ++_iter107) { - xfer += oprot->writeString((*_iter80)); + xfer += oprot->writeString((*_iter107)); } xfer += oprot->writeListEnd(); } @@ -1599,10 +1942,10 @@ uint32_t MilvusService_SearchVectorInFiles_pargs::write(::apache::thrift::protoc xfer += oprot->writeFieldBegin("query_record_array", ::apache::thrift::protocol::T_LIST, 4); { xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRUCT, static_cast((*(this->query_record_array)).size())); - std::vector ::const_iterator _iter81; - for (_iter81 = (*(this->query_record_array)).begin(); _iter81 != (*(this->query_record_array)).end(); ++_iter81) + std::vector ::const_iterator _iter108; + for (_iter108 = (*(this->query_record_array)).begin(); _iter108 != (*(this->query_record_array)).end(); ++_iter108) { - xfer += (*_iter81).write(oprot); + xfer += (*_iter108).write(oprot); } xfer += oprot->writeListEnd(); } @@ -1611,10 +1954,10 @@ uint32_t MilvusService_SearchVectorInFiles_pargs::write(::apache::thrift::protoc xfer += oprot->writeFieldBegin("query_range_array", ::apache::thrift::protocol::T_LIST, 5); { xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRUCT, static_cast((*(this->query_range_array)).size())); - std::vector ::const_iterator _iter82; - for (_iter82 = (*(this->query_range_array)).begin(); _iter82 != (*(this->query_range_array)).end(); ++_iter82) + std::vector ::const_iterator _iter109; + for (_iter109 = (*(this->query_range_array)).begin(); _iter109 != (*(this->query_range_array)).end(); ++_iter109) { - xfer += (*_iter82).write(oprot); + xfer += (*_iter109).write(oprot); } xfer += oprot->writeListEnd(); } @@ -1659,14 +2002,14 @@ uint32_t MilvusService_SearchVectorInFiles_result::read(::apache::thrift::protoc if (ftype == ::apache::thrift::protocol::T_LIST) { { this->success.clear(); - uint32_t _size83; - ::apache::thrift::protocol::TType _etype86; - xfer += iprot->readListBegin(_etype86, _size83); - this->success.resize(_size83); - uint32_t _i87; - for (_i87 = 0; _i87 < _size83; ++_i87) + uint32_t _size110; + ::apache::thrift::protocol::TType _etype113; + xfer += iprot->readListBegin(_etype113, _size110); + this->success.resize(_size110); + uint32_t _i114; + for (_i114 = 0; _i114 < _size110; ++_i114) { - xfer += this->success[_i87].read(iprot); + xfer += this->success[_i114].read(iprot); } xfer += iprot->readListEnd(); } @@ -1705,10 +2048,10 @@ uint32_t MilvusService_SearchVectorInFiles_result::write(::apache::thrift::proto xfer += oprot->writeFieldBegin("success", ::apache::thrift::protocol::T_LIST, 0); { xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRUCT, static_cast(this->success.size())); - std::vector ::const_iterator _iter88; - for (_iter88 = this->success.begin(); _iter88 != this->success.end(); ++_iter88) + std::vector ::const_iterator _iter115; + for (_iter115 = this->success.begin(); _iter115 != this->success.end(); ++_iter115) { - xfer += (*_iter88).write(oprot); + xfer += (*_iter115).write(oprot); } xfer += oprot->writeListEnd(); } @@ -1753,14 +2096,14 @@ uint32_t MilvusService_SearchVectorInFiles_presult::read(::apache::thrift::proto if (ftype == ::apache::thrift::protocol::T_LIST) { { (*(this->success)).clear(); - uint32_t _size89; - ::apache::thrift::protocol::TType _etype92; - xfer += iprot->readListBegin(_etype92, _size89); - (*(this->success)).resize(_size89); - uint32_t _i93; - for (_i93 = 0; _i93 < _size89; ++_i93) + uint32_t _size116; + ::apache::thrift::protocol::TType _etype119; + xfer += iprot->readListBegin(_etype119, _size116); + (*(this->success)).resize(_size116); + uint32_t _i120; + for (_i120 = 0; _i120 < _size116; ++_i120) { - xfer += (*(this->success))[_i93].read(iprot); + xfer += (*(this->success))[_i120].read(iprot); } xfer += iprot->readListEnd(); } @@ -2291,14 +2634,14 @@ uint32_t MilvusService_ShowTables_result::read(::apache::thrift::protocol::TProt if (ftype == ::apache::thrift::protocol::T_LIST) { { this->success.clear(); - uint32_t _size94; - ::apache::thrift::protocol::TType _etype97; - xfer += iprot->readListBegin(_etype97, _size94); - this->success.resize(_size94); - uint32_t _i98; - for (_i98 = 0; _i98 < _size94; ++_i98) + uint32_t _size121; + ::apache::thrift::protocol::TType _etype124; + xfer += iprot->readListBegin(_etype124, _size121); + this->success.resize(_size121); + uint32_t _i125; + for (_i125 = 0; _i125 < _size121; ++_i125) { - xfer += iprot->readString(this->success[_i98]); + xfer += iprot->readString(this->success[_i125]); } xfer += iprot->readListEnd(); } @@ -2337,10 +2680,10 @@ uint32_t MilvusService_ShowTables_result::write(::apache::thrift::protocol::TPro xfer += oprot->writeFieldBegin("success", ::apache::thrift::protocol::T_LIST, 0); { xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRING, static_cast(this->success.size())); - std::vector ::const_iterator _iter99; - for (_iter99 = this->success.begin(); _iter99 != this->success.end(); ++_iter99) + std::vector ::const_iterator _iter126; + for (_iter126 = this->success.begin(); _iter126 != this->success.end(); ++_iter126) { - xfer += oprot->writeString((*_iter99)); + xfer += oprot->writeString((*_iter126)); } xfer += oprot->writeListEnd(); } @@ -2385,14 +2728,14 @@ uint32_t MilvusService_ShowTables_presult::read(::apache::thrift::protocol::TPro if (ftype == ::apache::thrift::protocol::T_LIST) { { (*(this->success)).clear(); - uint32_t _size100; - ::apache::thrift::protocol::TType _etype103; - xfer += iprot->readListBegin(_etype103, _size100); - (*(this->success)).resize(_size100); - uint32_t _i104; - for (_i104 = 0; _i104 < _size100; ++_i104) + uint32_t _size127; + ::apache::thrift::protocol::TType _etype130; + xfer += iprot->readListBegin(_etype130, _size127); + (*(this->success)).resize(_size127); + uint32_t _i131; + for (_i131 = 0; _i131 < _size127; ++_i131) { - xfer += iprot->readString((*(this->success))[_i104]); + xfer += iprot->readString((*(this->success))[_i131]); } xfer += iprot->readListEnd(); } @@ -2983,6 +3326,70 @@ void MilvusServiceClient::recv_SearchVector(std::vector & _retu throw ::apache::thrift::TApplicationException(::apache::thrift::TApplicationException::MISSING_RESULT, "SearchVector failed: unknown result"); } +void MilvusServiceClient::SearchVector2(std::vector & _return, const std::string& table_name, const std::vector & query_record_array, const std::vector & query_range_array, const int64_t topk) +{ + send_SearchVector2(table_name, query_record_array, query_range_array, topk); + recv_SearchVector2(_return); +} + +void MilvusServiceClient::send_SearchVector2(const std::string& table_name, const std::vector & query_record_array, const std::vector & query_range_array, const int64_t topk) +{ + int32_t cseqid = 0; + oprot_->writeMessageBegin("SearchVector2", ::apache::thrift::protocol::T_CALL, cseqid); + + MilvusService_SearchVector2_pargs args; + args.table_name = &table_name; + args.query_record_array = &query_record_array; + args.query_range_array = &query_range_array; + args.topk = &topk; + args.write(oprot_); + + oprot_->writeMessageEnd(); + oprot_->getTransport()->writeEnd(); + oprot_->getTransport()->flush(); +} + +void MilvusServiceClient::recv_SearchVector2(std::vector & _return) +{ + + int32_t rseqid = 0; + std::string fname; + ::apache::thrift::protocol::TMessageType mtype; + + iprot_->readMessageBegin(fname, mtype, rseqid); + if (mtype == ::apache::thrift::protocol::T_EXCEPTION) { + ::apache::thrift::TApplicationException x; + x.read(iprot_); + iprot_->readMessageEnd(); + iprot_->getTransport()->readEnd(); + throw x; + } + if (mtype != ::apache::thrift::protocol::T_REPLY) { + iprot_->skip(::apache::thrift::protocol::T_STRUCT); + iprot_->readMessageEnd(); + iprot_->getTransport()->readEnd(); + } + if (fname.compare("SearchVector2") != 0) { + iprot_->skip(::apache::thrift::protocol::T_STRUCT); + iprot_->readMessageEnd(); + iprot_->getTransport()->readEnd(); + } + MilvusService_SearchVector2_presult result; + result.success = &_return; + result.read(iprot_); + iprot_->readMessageEnd(); + iprot_->getTransport()->readEnd(); + + if (result.__isset.success) { + // _return pointer has now been filled + return; + } + if (result.__isset.e) { + throw result.e; + } + throw ::apache::thrift::TApplicationException(::apache::thrift::TApplicationException::MISSING_RESULT, "SearchVector2 failed: unknown result"); +} + void MilvusServiceClient::SearchVectorInFiles(std::vector & _return, const std::string& table_name, const std::vector & file_id_array, const std::vector & query_record_array, const std::vector & query_range_array, const int64_t topk) { send_SearchVectorInFiles(table_name, file_id_array, query_record_array, query_range_array, topk); @@ -3649,6 +4056,63 @@ void MilvusServiceProcessor::process_SearchVector(int32_t seqid, ::apache::thrif } } +void MilvusServiceProcessor::process_SearchVector2(int32_t seqid, ::apache::thrift::protocol::TProtocol* iprot, ::apache::thrift::protocol::TProtocol* oprot, void* callContext) +{ + void* ctx = NULL; + if (this->eventHandler_.get() != NULL) { + ctx = this->eventHandler_->getContext("MilvusService.SearchVector2", callContext); + } + ::apache::thrift::TProcessorContextFreer freer(this->eventHandler_.get(), ctx, "MilvusService.SearchVector2"); + + if (this->eventHandler_.get() != NULL) { + this->eventHandler_->preRead(ctx, "MilvusService.SearchVector2"); + } + + MilvusService_SearchVector2_args args; + args.read(iprot); + iprot->readMessageEnd(); + uint32_t bytes = iprot->getTransport()->readEnd(); + + if (this->eventHandler_.get() != NULL) { + this->eventHandler_->postRead(ctx, "MilvusService.SearchVector2", bytes); + } + + MilvusService_SearchVector2_result result; + try { + iface_->SearchVector2(result.success, args.table_name, args.query_record_array, args.query_range_array, args.topk); + result.__isset.success = true; + } catch (Exception &e) { + result.e = e; + result.__isset.e = true; + } catch (const std::exception& e) { + if (this->eventHandler_.get() != NULL) { + this->eventHandler_->handlerError(ctx, "MilvusService.SearchVector2"); + } + + ::apache::thrift::TApplicationException x(e.what()); + oprot->writeMessageBegin("SearchVector2", ::apache::thrift::protocol::T_EXCEPTION, seqid); + x.write(oprot); + oprot->writeMessageEnd(); + oprot->getTransport()->writeEnd(); + oprot->getTransport()->flush(); + return; + } + + if (this->eventHandler_.get() != NULL) { + this->eventHandler_->preWrite(ctx, "MilvusService.SearchVector2"); + } + + oprot->writeMessageBegin("SearchVector2", ::apache::thrift::protocol::T_REPLY, seqid); + result.write(oprot); + oprot->writeMessageEnd(); + bytes = oprot->getTransport()->writeEnd(); + oprot->getTransport()->flush(); + + if (this->eventHandler_.get() != NULL) { + this->eventHandler_->postWrite(ctx, "MilvusService.SearchVector2", bytes); + } +} + void MilvusServiceProcessor::process_SearchVectorInFiles(int32_t seqid, ::apache::thrift::protocol::TProtocol* iprot, ::apache::thrift::protocol::TProtocol* oprot, void* callContext) { void* ctx = NULL; @@ -4455,6 +4919,97 @@ void MilvusServiceConcurrentClient::recv_SearchVector(std::vector & _return, const std::string& table_name, const std::vector & query_record_array, const std::vector & query_range_array, const int64_t topk) +{ + int32_t seqid = send_SearchVector2(table_name, query_record_array, query_range_array, topk); + recv_SearchVector2(_return, seqid); +} + +int32_t MilvusServiceConcurrentClient::send_SearchVector2(const std::string& table_name, const std::vector & query_record_array, const std::vector & query_range_array, const int64_t topk) +{ + int32_t cseqid = this->sync_.generateSeqId(); + ::apache::thrift::async::TConcurrentSendSentry sentry(&this->sync_); + oprot_->writeMessageBegin("SearchVector2", ::apache::thrift::protocol::T_CALL, cseqid); + + MilvusService_SearchVector2_pargs args; + args.table_name = &table_name; + args.query_record_array = &query_record_array; + args.query_range_array = &query_range_array; + args.topk = &topk; + args.write(oprot_); + + oprot_->writeMessageEnd(); + oprot_->getTransport()->writeEnd(); + oprot_->getTransport()->flush(); + + sentry.commit(); + return cseqid; +} + +void MilvusServiceConcurrentClient::recv_SearchVector2(std::vector & _return, const int32_t seqid) +{ + + int32_t rseqid = 0; + std::string fname; + ::apache::thrift::protocol::TMessageType mtype; + + // the read mutex gets dropped and reacquired as part of waitForWork() + // The destructor of this sentry wakes up other clients + ::apache::thrift::async::TConcurrentRecvSentry sentry(&this->sync_, seqid); + + while(true) { + if(!this->sync_.getPending(fname, mtype, rseqid)) { + iprot_->readMessageBegin(fname, mtype, rseqid); + } + if(seqid == rseqid) { + if (mtype == ::apache::thrift::protocol::T_EXCEPTION) { + ::apache::thrift::TApplicationException x; + x.read(iprot_); + iprot_->readMessageEnd(); + iprot_->getTransport()->readEnd(); + sentry.commit(); + throw x; + } + if (mtype != ::apache::thrift::protocol::T_REPLY) { + iprot_->skip(::apache::thrift::protocol::T_STRUCT); + iprot_->readMessageEnd(); + iprot_->getTransport()->readEnd(); + } + if (fname.compare("SearchVector2") != 0) { + iprot_->skip(::apache::thrift::protocol::T_STRUCT); + iprot_->readMessageEnd(); + iprot_->getTransport()->readEnd(); + + // in a bad state, don't commit + using ::apache::thrift::protocol::TProtocolException; + throw TProtocolException(TProtocolException::INVALID_DATA); + } + MilvusService_SearchVector2_presult result; + result.success = &_return; + result.read(iprot_); + iprot_->readMessageEnd(); + iprot_->getTransport()->readEnd(); + + if (result.__isset.success) { + // _return pointer has now been filled + sentry.commit(); + return; + } + if (result.__isset.e) { + sentry.commit(); + throw result.e; + } + // in a bad state, don't commit + throw ::apache::thrift::TApplicationException(::apache::thrift::TApplicationException::MISSING_RESULT, "SearchVector2 failed: unknown result"); + } + // seqid != rseqid + this->sync_.updatePending(fname, mtype, rseqid); + + // this will temporarily unlock the readMutex, and let other clients get work done + this->sync_.waitForWork(seqid); + } // end while(true) +} + void MilvusServiceConcurrentClient::SearchVectorInFiles(std::vector & _return, const std::string& table_name, const std::vector & file_id_array, const std::vector & query_record_array, const std::vector & query_range_array, const int64_t topk) { int32_t seqid = send_SearchVectorInFiles(table_name, file_id_array, query_record_array, query_range_array, topk); diff --git a/cpp/src/thrift/gen-cpp/MilvusService.h b/cpp/src/thrift/gen-cpp/MilvusService.h index 5b4204e6b8..c18a33009a 100644 --- a/cpp/src/thrift/gen-cpp/MilvusService.h +++ b/cpp/src/thrift/gen-cpp/MilvusService.h @@ -104,6 +104,25 @@ class MilvusServiceIf { */ virtual void SearchVector(std::vector & _return, const std::string& table_name, const std::vector & query_record_array, const std::vector & query_range_array, const int64_t topk) = 0; + /** + * @brief Query vector + * + * This method is used to query vector in table. + * + * @param table_name, table_name is queried. + * @param query_record_array, all vector are going to be queried. + * @param query_range_array, optional ranges for conditional search. If not specified, search whole table + * @param topk, how many similarity vectors will be searched. + * + * @return query binary result array. + * + * @param table_name + * @param query_record_array + * @param query_range_array + * @param topk + */ + virtual void SearchVector2(std::vector & _return, const std::string& table_name, const std::vector & query_record_array, const std::vector & query_range_array, const int64_t topk) = 0; + /** * @brief Internal use query interface * @@ -218,6 +237,9 @@ class MilvusServiceNull : virtual public MilvusServiceIf { void SearchVector(std::vector & /* _return */, const std::string& /* table_name */, const std::vector & /* query_record_array */, const std::vector & /* query_range_array */, const int64_t /* topk */) { return; } + void SearchVector2(std::vector & /* _return */, const std::string& /* table_name */, const std::vector & /* query_record_array */, const std::vector & /* query_range_array */, const int64_t /* topk */) { + return; + } void SearchVectorInFiles(std::vector & /* _return */, const std::string& /* table_name */, const std::vector & /* file_id_array */, const std::vector & /* query_record_array */, const std::vector & /* query_range_array */, const int64_t /* topk */) { return; } @@ -912,6 +934,139 @@ class MilvusService_SearchVector_presult { }; +typedef struct _MilvusService_SearchVector2_args__isset { + _MilvusService_SearchVector2_args__isset() : table_name(false), query_record_array(false), query_range_array(false), topk(false) {} + bool table_name :1; + bool query_record_array :1; + bool query_range_array :1; + bool topk :1; +} _MilvusService_SearchVector2_args__isset; + +class MilvusService_SearchVector2_args { + public: + + MilvusService_SearchVector2_args(const MilvusService_SearchVector2_args&); + MilvusService_SearchVector2_args& operator=(const MilvusService_SearchVector2_args&); + MilvusService_SearchVector2_args() : table_name(), topk(0) { + } + + virtual ~MilvusService_SearchVector2_args() throw(); + std::string table_name; + std::vector query_record_array; + std::vector query_range_array; + int64_t topk; + + _MilvusService_SearchVector2_args__isset __isset; + + void __set_table_name(const std::string& val); + + void __set_query_record_array(const std::vector & val); + + void __set_query_range_array(const std::vector & val); + + void __set_topk(const int64_t val); + + bool operator == (const MilvusService_SearchVector2_args & rhs) const + { + if (!(table_name == rhs.table_name)) + return false; + if (!(query_record_array == rhs.query_record_array)) + return false; + if (!(query_range_array == rhs.query_range_array)) + return false; + if (!(topk == rhs.topk)) + return false; + return true; + } + bool operator != (const MilvusService_SearchVector2_args &rhs) const { + return !(*this == rhs); + } + + bool operator < (const MilvusService_SearchVector2_args & ) const; + + uint32_t read(::apache::thrift::protocol::TProtocol* iprot); + uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const; + +}; + + +class MilvusService_SearchVector2_pargs { + public: + + + virtual ~MilvusService_SearchVector2_pargs() throw(); + const std::string* table_name; + const std::vector * query_record_array; + const std::vector * query_range_array; + const int64_t* topk; + + uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const; + +}; + +typedef struct _MilvusService_SearchVector2_result__isset { + _MilvusService_SearchVector2_result__isset() : success(false), e(false) {} + bool success :1; + bool e :1; +} _MilvusService_SearchVector2_result__isset; + +class MilvusService_SearchVector2_result { + public: + + MilvusService_SearchVector2_result(const MilvusService_SearchVector2_result&); + MilvusService_SearchVector2_result& operator=(const MilvusService_SearchVector2_result&); + MilvusService_SearchVector2_result() { + } + + virtual ~MilvusService_SearchVector2_result() throw(); + std::vector success; + Exception e; + + _MilvusService_SearchVector2_result__isset __isset; + + void __set_success(const std::vector & val); + + void __set_e(const Exception& val); + + bool operator == (const MilvusService_SearchVector2_result & rhs) const + { + if (!(success == rhs.success)) + return false; + if (!(e == rhs.e)) + return false; + return true; + } + bool operator != (const MilvusService_SearchVector2_result &rhs) const { + return !(*this == rhs); + } + + bool operator < (const MilvusService_SearchVector2_result & ) const; + + uint32_t read(::apache::thrift::protocol::TProtocol* iprot); + uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const; + +}; + +typedef struct _MilvusService_SearchVector2_presult__isset { + _MilvusService_SearchVector2_presult__isset() : success(false), e(false) {} + bool success :1; + bool e :1; +} _MilvusService_SearchVector2_presult__isset; + +class MilvusService_SearchVector2_presult { + public: + + + virtual ~MilvusService_SearchVector2_presult() throw(); + std::vector * success; + Exception e; + + _MilvusService_SearchVector2_presult__isset __isset; + + uint32_t read(::apache::thrift::protocol::TProtocol* iprot); + +}; + typedef struct _MilvusService_SearchVectorInFiles_args__isset { _MilvusService_SearchVectorInFiles_args__isset() : table_name(false), file_id_array(false), query_record_array(false), query_range_array(false), topk(false) {} bool table_name :1; @@ -1531,6 +1686,9 @@ class MilvusServiceClient : virtual public MilvusServiceIf { void SearchVector(std::vector & _return, const std::string& table_name, const std::vector & query_record_array, const std::vector & query_range_array, const int64_t topk); void send_SearchVector(const std::string& table_name, const std::vector & query_record_array, const std::vector & query_range_array, const int64_t topk); void recv_SearchVector(std::vector & _return); + void SearchVector2(std::vector & _return, const std::string& table_name, const std::vector & query_record_array, const std::vector & query_range_array, const int64_t topk); + void send_SearchVector2(const std::string& table_name, const std::vector & query_record_array, const std::vector & query_range_array, const int64_t topk); + void recv_SearchVector2(std::vector & _return); void SearchVectorInFiles(std::vector & _return, const std::string& table_name, const std::vector & file_id_array, const std::vector & query_record_array, const std::vector & query_range_array, const int64_t topk); void send_SearchVectorInFiles(const std::string& table_name, const std::vector & file_id_array, const std::vector & query_record_array, const std::vector & query_range_array, const int64_t topk); void recv_SearchVectorInFiles(std::vector & _return); @@ -1567,6 +1725,7 @@ class MilvusServiceProcessor : public ::apache::thrift::TDispatchProcessor { void process_BuildIndex(int32_t seqid, ::apache::thrift::protocol::TProtocol* iprot, ::apache::thrift::protocol::TProtocol* oprot, void* callContext); void process_AddVector(int32_t seqid, ::apache::thrift::protocol::TProtocol* iprot, ::apache::thrift::protocol::TProtocol* oprot, void* callContext); void process_SearchVector(int32_t seqid, ::apache::thrift::protocol::TProtocol* iprot, ::apache::thrift::protocol::TProtocol* oprot, void* callContext); + void process_SearchVector2(int32_t seqid, ::apache::thrift::protocol::TProtocol* iprot, ::apache::thrift::protocol::TProtocol* oprot, void* callContext); void process_SearchVectorInFiles(int32_t seqid, ::apache::thrift::protocol::TProtocol* iprot, ::apache::thrift::protocol::TProtocol* oprot, void* callContext); void process_DescribeTable(int32_t seqid, ::apache::thrift::protocol::TProtocol* iprot, ::apache::thrift::protocol::TProtocol* oprot, void* callContext); void process_GetTableRowCount(int32_t seqid, ::apache::thrift::protocol::TProtocol* iprot, ::apache::thrift::protocol::TProtocol* oprot, void* callContext); @@ -1581,6 +1740,7 @@ class MilvusServiceProcessor : public ::apache::thrift::TDispatchProcessor { processMap_["BuildIndex"] = &MilvusServiceProcessor::process_BuildIndex; processMap_["AddVector"] = &MilvusServiceProcessor::process_AddVector; processMap_["SearchVector"] = &MilvusServiceProcessor::process_SearchVector; + processMap_["SearchVector2"] = &MilvusServiceProcessor::process_SearchVector2; processMap_["SearchVectorInFiles"] = &MilvusServiceProcessor::process_SearchVectorInFiles; processMap_["DescribeTable"] = &MilvusServiceProcessor::process_DescribeTable; processMap_["GetTableRowCount"] = &MilvusServiceProcessor::process_GetTableRowCount; @@ -1670,6 +1830,16 @@ class MilvusServiceMultiface : virtual public MilvusServiceIf { return; } + void SearchVector2(std::vector & _return, const std::string& table_name, const std::vector & query_record_array, const std::vector & query_range_array, const int64_t topk) { + size_t sz = ifaces_.size(); + size_t i = 0; + for (; i < (sz - 1); ++i) { + ifaces_[i]->SearchVector2(_return, table_name, query_record_array, query_range_array, topk); + } + ifaces_[i]->SearchVector2(_return, table_name, query_record_array, query_range_array, topk); + return; + } + void SearchVectorInFiles(std::vector & _return, const std::string& table_name, const std::vector & file_id_array, const std::vector & query_record_array, const std::vector & query_range_array, const int64_t topk) { size_t sz = ifaces_.size(); size_t i = 0; @@ -1767,6 +1937,9 @@ class MilvusServiceConcurrentClient : virtual public MilvusServiceIf { void SearchVector(std::vector & _return, const std::string& table_name, const std::vector & query_record_array, const std::vector & query_range_array, const int64_t topk); int32_t send_SearchVector(const std::string& table_name, const std::vector & query_record_array, const std::vector & query_range_array, const int64_t topk); void recv_SearchVector(std::vector & _return, const int32_t seqid); + void SearchVector2(std::vector & _return, const std::string& table_name, const std::vector & query_record_array, const std::vector & query_range_array, const int64_t topk); + int32_t send_SearchVector2(const std::string& table_name, const std::vector & query_record_array, const std::vector & query_range_array, const int64_t topk); + void recv_SearchVector2(std::vector & _return, const int32_t seqid); void SearchVectorInFiles(std::vector & _return, const std::string& table_name, const std::vector & file_id_array, const std::vector & query_record_array, const std::vector & query_range_array, const int64_t topk); int32_t send_SearchVectorInFiles(const std::string& table_name, const std::vector & file_id_array, const std::vector & query_record_array, const std::vector & query_range_array, const int64_t topk); void recv_SearchVectorInFiles(std::vector & _return, const int32_t seqid); diff --git a/cpp/src/thrift/gen-cpp/MilvusService_server.skeleton.cpp b/cpp/src/thrift/gen-cpp/MilvusService_server.skeleton.cpp index f22cfc2894..efa8e95034 100644 --- a/cpp/src/thrift/gen-cpp/MilvusService_server.skeleton.cpp +++ b/cpp/src/thrift/gen-cpp/MilvusService_server.skeleton.cpp @@ -120,6 +120,28 @@ class MilvusServiceHandler : virtual public MilvusServiceIf { printf("SearchVector\n"); } + /** + * @brief Query vector + * + * This method is used to query vector in table. + * + * @param table_name, table_name is queried. + * @param query_record_array, all vector are going to be queried. + * @param query_range_array, optional ranges for conditional search. If not specified, search whole table + * @param topk, how many similarity vectors will be searched. + * + * @return query binary result array. + * + * @param table_name + * @param query_record_array + * @param query_range_array + * @param topk + */ + void SearchVector2(std::vector & _return, const std::string& table_name, const std::vector & query_record_array, const std::vector & query_range_array, const int64_t topk) { + // Your implementation goes here + printf("SearchVector2\n"); + } + /** * @brief Internal use query interface * diff --git a/cpp/src/thrift/gen-cpp/milvus_types.cpp b/cpp/src/thrift/gen-cpp/milvus_types.cpp index 06923c3a1a..a1ef24de52 100644 --- a/cpp/src/thrift/gen-cpp/milvus_types.cpp +++ b/cpp/src/thrift/gen-cpp/milvus_types.cpp @@ -781,4 +781,119 @@ void TopKQueryResult::printTo(std::ostream& out) const { out << ")"; } + +TopKQueryBinResult::~TopKQueryBinResult() throw() { +} + + +void TopKQueryBinResult::__set_id_array(const std::string& val) { + this->id_array = val; +} + +void TopKQueryBinResult::__set_distance_array(const std::string& val) { + this->distance_array = val; +} +std::ostream& operator<<(std::ostream& out, const TopKQueryBinResult& obj) +{ + obj.printTo(out); + return out; +} + + +uint32_t TopKQueryBinResult::read(::apache::thrift::protocol::TProtocol* iprot) { + + ::apache::thrift::protocol::TInputRecursionTracker tracker(*iprot); + uint32_t xfer = 0; + std::string fname; + ::apache::thrift::protocol::TType ftype; + int16_t fid; + + xfer += iprot->readStructBegin(fname); + + using ::apache::thrift::protocol::TProtocolException; + + bool isset_id_array = false; + bool isset_distance_array = false; + + while (true) + { + xfer += iprot->readFieldBegin(fname, ftype, fid); + if (ftype == ::apache::thrift::protocol::T_STOP) { + break; + } + switch (fid) + { + case 1: + if (ftype == ::apache::thrift::protocol::T_STRING) { + xfer += iprot->readBinary(this->id_array); + isset_id_array = true; + } else { + xfer += iprot->skip(ftype); + } + break; + case 2: + if (ftype == ::apache::thrift::protocol::T_STRING) { + xfer += iprot->readBinary(this->distance_array); + isset_distance_array = true; + } else { + xfer += iprot->skip(ftype); + } + break; + default: + xfer += iprot->skip(ftype); + break; + } + xfer += iprot->readFieldEnd(); + } + + xfer += iprot->readStructEnd(); + + if (!isset_id_array) + throw TProtocolException(TProtocolException::INVALID_DATA); + if (!isset_distance_array) + throw TProtocolException(TProtocolException::INVALID_DATA); + return xfer; +} + +uint32_t TopKQueryBinResult::write(::apache::thrift::protocol::TProtocol* oprot) const { + uint32_t xfer = 0; + ::apache::thrift::protocol::TOutputRecursionTracker tracker(*oprot); + xfer += oprot->writeStructBegin("TopKQueryBinResult"); + + xfer += oprot->writeFieldBegin("id_array", ::apache::thrift::protocol::T_STRING, 1); + xfer += oprot->writeBinary(this->id_array); + xfer += oprot->writeFieldEnd(); + + xfer += oprot->writeFieldBegin("distance_array", ::apache::thrift::protocol::T_STRING, 2); + xfer += oprot->writeBinary(this->distance_array); + xfer += oprot->writeFieldEnd(); + + xfer += oprot->writeFieldStop(); + xfer += oprot->writeStructEnd(); + return xfer; +} + +void swap(TopKQueryBinResult &a, TopKQueryBinResult &b) { + using ::std::swap; + swap(a.id_array, b.id_array); + swap(a.distance_array, b.distance_array); +} + +TopKQueryBinResult::TopKQueryBinResult(const TopKQueryBinResult& other19) { + id_array = other19.id_array; + distance_array = other19.distance_array; +} +TopKQueryBinResult& TopKQueryBinResult::operator=(const TopKQueryBinResult& other20) { + id_array = other20.id_array; + distance_array = other20.distance_array; + return *this; +} +void TopKQueryBinResult::printTo(std::ostream& out) const { + using ::apache::thrift::to_string; + out << "TopKQueryBinResult("; + out << "id_array=" << to_string(id_array); + out << ", " << "distance_array=" << to_string(distance_array); + out << ")"; +} + }} // namespace diff --git a/cpp/src/thrift/gen-cpp/milvus_types.h b/cpp/src/thrift/gen-cpp/milvus_types.h index 63e313d197..a75d883074 100644 --- a/cpp/src/thrift/gen-cpp/milvus_types.h +++ b/cpp/src/thrift/gen-cpp/milvus_types.h @@ -63,6 +63,8 @@ class QueryResult; class TopKQueryResult; +class TopKQueryBinResult; + typedef struct _Exception__isset { _Exception__isset() : code(false), reason(false) {} bool code :1; @@ -346,6 +348,47 @@ void swap(TopKQueryResult &a, TopKQueryResult &b); std::ostream& operator<<(std::ostream& out, const TopKQueryResult& obj); + +class TopKQueryBinResult : public virtual ::apache::thrift::TBase { + public: + + TopKQueryBinResult(const TopKQueryBinResult&); + TopKQueryBinResult& operator=(const TopKQueryBinResult&); + TopKQueryBinResult() : id_array(), distance_array() { + } + + virtual ~TopKQueryBinResult() throw(); + std::string id_array; + std::string distance_array; + + void __set_id_array(const std::string& val); + + void __set_distance_array(const std::string& val); + + bool operator == (const TopKQueryBinResult & rhs) const + { + if (!(id_array == rhs.id_array)) + return false; + if (!(distance_array == rhs.distance_array)) + return false; + return true; + } + bool operator != (const TopKQueryBinResult &rhs) const { + return !(*this == rhs); + } + + bool operator < (const TopKQueryBinResult & ) const; + + uint32_t read(::apache::thrift::protocol::TProtocol* iprot); + uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const; + + virtual void printTo(std::ostream& out) const; +}; + +void swap(TopKQueryBinResult &a, TopKQueryBinResult &b); + +std::ostream& operator<<(std::ostream& out, const TopKQueryBinResult& obj); + }} // namespace #endif diff --git a/cpp/src/thrift/milvus.thrift b/cpp/src/thrift/milvus.thrift index 001b27b04b..64f25ae387 100644 --- a/cpp/src/thrift/milvus.thrift +++ b/cpp/src/thrift/milvus.thrift @@ -84,6 +84,14 @@ struct TopKQueryResult { 1: list query_result_arrays; ///< TopK query result } +/** + * @brief TopK query binary result + */ +struct TopKQueryBinResult { + 1: required binary id_array; ///< id array, interger array + 2: required binary distance_array; ///< distance array, double array +} + service MilvusService { /** * @brief Create table method @@ -158,6 +166,23 @@ service MilvusService { 4: list query_range_array, 5: i64 topk) throws(1: Exception e); + /** + * @brief Query vector + * + * This method is used to query vector in table. + * + * @param table_name, table_name is queried. + * @param query_record_array, all vector are going to be queried. + * @param query_range_array, optional ranges for conditional search. If not specified, search whole table + * @param topk, how many similarity vectors will be searched. + * + * @return query binary result array. + */ + list SearchVector2(2: string table_name, + 3: list query_record_array, + 4: list query_range_array, + 5: i64 topk) throws(1: Exception e); + /** * @brief Internal use query interface * diff --git a/cpp/src/utils/TimeRecorder.cpp b/cpp/src/utils/TimeRecorder.cpp index fb3cb421a1..ee33680f54 100644 --- a/cpp/src/utils/TimeRecorder.cpp +++ b/cpp/src/utils/TimeRecorder.cpp @@ -12,131 +12,81 @@ namespace milvus { namespace server { TimeRecorder::TimeRecorder(const std::string &header, - TimeRecorder::TimeDisplayUnit unit, int64_t log_level) : - header_(header), - time_unit_(unit), - log_level_(log_level) { + header_(header), + log_level_(log_level) { start_ = last_ = stdclock::now(); - span_ = 0.0; +} + +TimeRecorder::~TimeRecorder() { } std::string -TimeRecorder::GetTimeSpanStr(TimeRecorder::TimeDisplayUnit &unit, double span) const { - std::string spanStr; - std::string unitStr; +TimeRecorder::GetTimeSpanStr(double span) { + std::string str_sec = std::to_string(span * 0.000001) + ((span > 1000000) ? " seconds" : " second"); + std::string str_ms = std::to_string(span * 0.001) + " ms"; - switch (unit) { - case TimeRecorder::eTimeAutoUnit: { - if (span >= 1000000) { - int64_t t = (int64_t) span; - int64_t hour, minute; - double second; - hour = t / 1000000 / 3600; - t -= hour * 3600 * 1000000; - minute = t / 1000000 / 60; - t -= minute * 60 * 1000000; - second = t * 0.000001; - spanStr += (hour < 10 ? "0" : "") + std::to_string(hour) + ":"; - spanStr += (minute < 10 ? "0" : "") + std::to_string(minute) + ":"; - spanStr += (second < 10 ? "0" : "") + std::to_string(second); - unitStr = ""; - } else if (span >= 1000) { - spanStr = std::to_string(span * 0.001); - unitStr = " ms"; - } else { - spanStr = std::to_string(span); - unitStr = " us"; - } - } - break; - case TimeRecorder::eTimeHourUnit: - spanStr = std::to_string((span * 0.000001) / 3600); - unitStr = " hour"; - break; - case TimeRecorder::eTimeMinuteUnit: - spanStr = std::to_string((span * 0.000001) / 60); - unitStr = " min"; - break; - case TimeRecorder::eTimeSecondUnit: - spanStr = std::to_string(span * 0.000001); - unitStr = " sec"; - break; - case TimeRecorder::eTimeMilliSecUnit: - spanStr = std::to_string(span * 0.001); - unitStr = " ms"; - break; - case TimeRecorder::eTimeMicroSecUnit: - default: - spanStr = std::to_string(span); - unitStr = " us"; - break; - } - - return spanStr + unitStr; + return str_sec + " [" + str_ms + "]"; } void TimeRecorder::PrintTimeRecord(const std::string &msg, double span) { - std::string strLog; - if (!header_.empty()) strLog += header_ + ": "; - strLog += msg; - strLog += " ("; - strLog += GetTimeSpanStr(time_unit_, span); - strLog += ")"; + std::string str_log; + if (!header_.empty()) str_log += header_ + ": "; + str_log += msg; + str_log += " ("; + str_log += TimeRecorder::GetTimeSpanStr(span); + str_log += ")"; switch (log_level_) { case 0: { - SERVER_LOG_TRACE << strLog; + SERVER_LOG_TRACE << str_log; break; } case 1: { - SERVER_LOG_DEBUG << strLog; + SERVER_LOG_DEBUG << str_log; break; } case 2: { - SERVER_LOG_INFO << strLog; + SERVER_LOG_INFO << str_log; break; } case 3: { - SERVER_LOG_WARNING << strLog; + SERVER_LOG_WARNING << str_log; break; } case 4: { - SERVER_LOG_ERROR << strLog; + SERVER_LOG_ERROR << str_log; break; } case 5: { - SERVER_LOG_FATAL << strLog; + SERVER_LOG_FATAL << str_log; break; } default: { - SERVER_LOG_INFO << strLog; + SERVER_LOG_INFO << str_log; break; } } } -void -TimeRecorder::Record(const std::string &msg) { +double +TimeRecorder::RecordSection(const std::string &msg) { stdclock::time_point curr = stdclock::now(); - span_ = (std::chrono::duration(curr - last_)).count(); + double span = (std::chrono::duration(curr - last_)).count(); last_ = curr; - PrintTimeRecord(msg, span_); -} - -void -TimeRecorder::Elapse(const std::string &msg) { - stdclock::time_point curr = stdclock::now(); - span_ = (std::chrono::duration(curr - start_)).count(); - - PrintTimeRecord(msg, span_); + PrintTimeRecord(msg, span); + return span; } double -TimeRecorder::Span() { - return span_; +TimeRecorder::ElapseFromBegin(const std::string &msg) { + stdclock::time_point curr = stdclock::now(); + double span = (std::chrono::duration(curr - start_)).count(); + + PrintTimeRecord(msg, span); + return span; } } diff --git a/cpp/src/utils/TimeRecorder.h b/cpp/src/utils/TimeRecorder.h index 9b960dc0ea..1656e0d77f 100644 --- a/cpp/src/utils/TimeRecorder.h +++ b/cpp/src/utils/TimeRecorder.h @@ -17,36 +17,24 @@ class TimeRecorder { using stdclock = std::chrono::high_resolution_clock; public: - enum TimeDisplayUnit { - eTimeAutoUnit = 0, - eTimeHourUnit, - eTimeMinuteUnit, - eTimeSecondUnit, - eTimeMilliSecUnit, - eTimeMicroSecUnit, - }; - TimeRecorder(const std::string &header, - TimeRecorder::TimeDisplayUnit unit = TimeRecorder::eTimeAutoUnit, - int64_t log_level = 1); //trace = 0, debug = 1, info = 2, warn = 3, error = 4, critical = 5 + int64_t log_level = 1); - void Record(const std::string &msg); + ~TimeRecorder();//trace = 0, debug = 1, info = 2, warn = 3, error = 4, critical = 5 - void Elapse(const std::string &msg); + double RecordSection(const std::string &msg); - double Span(); + double ElapseFromBegin(const std::string &msg); + + static std::string GetTimeSpanStr(double span); private: - std::string GetTimeSpanStr(TimeRecorder::TimeDisplayUnit &unit, double span) const; - void PrintTimeRecord(const std::string &msg, double span); private: std::string header_; - TimeRecorder::TimeDisplayUnit time_unit_; stdclock::time_point start_; stdclock::time_point last_; - double span_; int64_t log_level_; }; diff --git a/cpp/src/utils/ValidationUtil.cpp b/cpp/src/utils/ValidationUtil.cpp index 53f00a4fc7..89b5573999 100644 --- a/cpp/src/utils/ValidationUtil.cpp +++ b/cpp/src/utils/ValidationUtil.cpp @@ -61,7 +61,6 @@ ValidateTableIndexType(int32_t index_type) { return SERVER_INVALID_INDEX_TYPE; } - SERVER_LOG_DEBUG << "Index type: " << index_type; return SERVER_SUCCESS; } diff --git a/cpp/thirdparty/versions.txt b/cpp/thirdparty/versions.txt index 311760948d..02e20c32cb 100644 --- a/cpp/thirdparty/versions.txt +++ b/cpp/thirdparty/versions.txt @@ -2,7 +2,8 @@ ARROW_VERSION=zilliz BOOST_VERSION=1.70.0 BZIP2_VERSION=1.0.6 EASYLOGGINGPP_VERSION=v9.96.7 -FAISS_VERSION=7b07685 +FAISS_VERSION=v1.5.3 +MKL_VERSION=2019.4.243 GTEST_VERSION=1.8.1 JSONCONS_VERSION=0.126.0 LAPACK_VERSION=v3.8.0 @@ -19,5 +20,7 @@ YAMLCPP_VERSION=0.6.2 ZLIB_VERSION=v1.2.11 ZSTD_VERSION=v1.4.0 AWS_VERSION=1.7.125 +LIBUNWIND_VERSION=1.3.1 +GPERFTOOLS_VERSION=2.7 # vim: set filetype=sh: diff --git a/cpp/unittest/CMakeLists.txt b/cpp/unittest/CMakeLists.txt index 62e32f6d1d..6a6f94a632 100644 --- a/cpp/unittest/CMakeLists.txt +++ b/cpp/unittest/CMakeLists.txt @@ -28,7 +28,6 @@ set(unittest_libs easyloggingpp pthread metrics - openblas gfortran prometheus-cpp-pull prometheus-cpp-push diff --git a/cpp/unittest/db/CMakeLists.txt b/cpp/unittest/db/CMakeLists.txt index c0f3052848..0e3531cd33 100644 --- a/cpp/unittest/db/CMakeLists.txt +++ b/cpp/unittest/db/CMakeLists.txt @@ -23,10 +23,7 @@ link_directories("/usr/local/cuda/lib64") include_directories(/usr/include/mysql) -#add_definitions(-DBOOST_ERROR_CODE_HEADER_ONLY) - set(db_test_src - #${unittest_srcs} ${config_files} ${cache_srcs} ${db_srcs} @@ -38,7 +35,6 @@ set(db_test_src cuda_add_executable(db_test ${db_test_src}) set(db_libs - libgpufaiss.a faiss cudart cublas @@ -49,6 +45,11 @@ set(db_libs mysqlpp ) +if(${BUILD_FAISS_WITH_MKL} STREQUAL "ON") + set(db_libs ${db_libs} ${MKL_LIBS} ${MKL_LIBS}) +endif() + target_link_libraries(db_test ${db_libs} ${unittest_libs}) install(TARGETS db_test DESTINATION bin) + diff --git a/cpp/unittest/db/mem_test.cpp b/cpp/unittest/db/mem_test.cpp index 2f6952c79d..17d87030cd 100644 --- a/cpp/unittest/db/mem_test.cpp +++ b/cpp/unittest/db/mem_test.cpp @@ -273,7 +273,7 @@ TEST_F(NewMemManagerTest, INSERT_TEST) { int insert_loop = 20; for (int i = 0; i < insert_loop; ++i) { - int64_t nb = 409600; + int64_t nb = 40960; std::vector xb; BuildVectors(nb, xb); engine::IDNumbers vector_ids; @@ -308,7 +308,7 @@ TEST_F(NewMemManagerTest, CONCURRENT_INSERT_SEARCH_TEST) { engine::IDNumbers vector_ids; engine::IDNumbers target_ids; - int64_t nb = 409600; + int64_t nb = 40960; std::vector xb; BuildVectors(nb, xb); diff --git a/cpp/unittest/faiss_wrapper/CMakeLists.txt b/cpp/unittest/faiss_wrapper/CMakeLists.txt index ff2535e3fd..f7976fb604 100644 --- a/cpp/unittest/faiss_wrapper/CMakeLists.txt +++ b/cpp/unittest/faiss_wrapper/CMakeLists.txt @@ -24,7 +24,6 @@ set(wrapper_libs stdc++ boost_system_static boost_filesystem_static - libgpufaiss.a faiss cudart cublas @@ -35,6 +34,10 @@ set(wrapper_libs zstd lz4 ) +if(${BUILD_FAISS_WITH_MKL} STREQUAL "ON") + set(wrapper_libs ${wrapper_libs} ${MKL_LIBS} ${MKL_LIBS}) +endif() + target_link_libraries(wrapper_test ${wrapper_libs} ${unittest_libs}) set(topk_test_src diff --git a/cpp/unittest/metrics/CMakeLists.txt b/cpp/unittest/metrics/CMakeLists.txt index 513b37b8c2..d4ae934d6f 100644 --- a/cpp/unittest/metrics/CMakeLists.txt +++ b/cpp/unittest/metrics/CMakeLists.txt @@ -10,8 +10,6 @@ include_directories(../../src) - - aux_source_directory(../../src/db db_srcs) aux_source_directory(../../src/config config_files) aux_source_directory(../../src/cache cache_srcs) @@ -33,21 +31,10 @@ include_directories(../../third_party/build/include) link_directories(../../third_party/build/lib) include_directories(/usr/local/cuda/include) link_directories("/usr/local/cuda/lib64") -#include_directories(../db/utils.h) include_directories(../../src/metrics) include_directories(/usr/include/mysql) -#set(metrics_src_files -# ../../src/metrics/Metrics.cpp -# ../../src/metrics/Metrics.h -# ../../src/metrics/PrometheusMetrics.cpp -# ../../src/metrics/MetricBase.h -# ../../src/server/ServerConfig.cpp -# ../../src/utils/CommonUtil.cpp -# ../../src/utils/TimeRecorder.cpp -# ) - set(count_test_src ${config_files} ${cache_srcs} @@ -62,7 +49,6 @@ set(count_test_src add_executable(metrics_test ${count_test_src} ${require_files} ) target_link_libraries(metrics_test - libgpufaiss.a faiss cudart cublas @@ -77,5 +63,8 @@ target_link_libraries(metrics_test mysqlpp ${unittest_libs} ) +if(${BUILD_FAISS_WITH_MKL} STREQUAL "ON") + target_link_libraries(metrics_test ${MKL_LIBS} ${MKL_LIBS}) +endif() install(TARGETS metrics_test DESTINATION bin) \ No newline at end of file diff --git a/cpp/unittest/server/CMakeLists.txt b/cpp/unittest/server/CMakeLists.txt index e49aed500c..7a20e6287b 100644 --- a/cpp/unittest/server/CMakeLists.txt +++ b/cpp/unittest/server/CMakeLists.txt @@ -33,7 +33,6 @@ cuda_add_executable(server_test set(require_libs stdc++ - libgpufaiss.a faiss cudart cublas @@ -48,6 +47,10 @@ set(require_libs pthread ) +if(${BUILD_FAISS_WITH_MKL} STREQUAL "ON") + set(require_libs ${require_libs} ${MKL_LIBS} ${MKL_LIBS}) +endif() + target_link_libraries(server_test ${require_libs} ${cuda_library} diff --git a/cpp/unittest/server/util_test.cpp b/cpp/unittest/server/util_test.cpp index 200bdc9776..e4948fc5e8 100644 --- a/cpp/unittest/server/util_test.cpp +++ b/cpp/unittest/server/util_test.cpp @@ -19,15 +19,6 @@ namespace { static const std::string LOG_FILE_PATH = "./milvus/conf/log_config.conf"; -using TimeUnit = server::TimeRecorder::TimeDisplayUnit; -double TestTimeRecorder(TimeUnit unit, int64_t log_level, int64_t sleep_ms) { - server::TimeRecorder rc("test rc", unit, log_level); - rc.Record("begin"); - std::this_thread::sleep_for(std::chrono::milliseconds(sleep_ms)); - rc.Elapse("end"); - return rc.Span(); -} - } TEST(UtilTest, EXCEPTION_TEST) { @@ -124,23 +115,6 @@ TEST(UtilTest, STRINGFUNCTIONS_TEST) { } -TEST(UtilTest, TIMERECORDER_TEST) { - double span = TestTimeRecorder(TimeUnit::eTimeAutoUnit, 0, 1001); - ASSERT_GT(span, 0.0); - span = TestTimeRecorder(TimeUnit::eTimeAutoUnit, 0, 101); - ASSERT_GT(span, 0.0); - span = TestTimeRecorder(TimeUnit::eTimeHourUnit, 1, 10); - ASSERT_GT(span, 0.0); - span = TestTimeRecorder(TimeUnit::eTimeMinuteUnit, 2, 10); - ASSERT_GT(span, 0.0); - span = TestTimeRecorder(TimeUnit::eTimeSecondUnit, 3, 10); - ASSERT_GT(span, 0.0); - span = TestTimeRecorder(TimeUnit::eTimeMilliSecUnit, 4, 10); - ASSERT_GT(span, 0.0); - span = TestTimeRecorder(TimeUnit::eTimeMicroSecUnit, -1, 10); - ASSERT_GT(span, 0.0); -} - TEST(UtilTest, BLOCKINGQUEUE_TEST) { server::BlockingQueue bq;