admin/milvus
1
0
Fork 0
mirror of https://gitee.com/milvus-io/milvus.git synced 2026-01-05 02:12:48 +08:00
Code Issues Packages Projects Releases Wiki Activity

#127 Support new Index type IVFPQ

Browse Source
This commit is contained in:
zirui.chen 2019-11-14 16:40:36 +08:00
commit f4536ac888
170 changed files with 16033 additions and 5744 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.gitignore vendored
View File

@ -26,3 +26,6 @@ cmake_build
*.lo
*.tar.gz
*.log
.coverage
*.pyc
cov_html/

20
.travis.yml Normal file
View File

@ -0,0 +1,20 @@
language: cpp
sudo: required
dist: bionic
cache:
directories:
- $HOME/.ccache
addons:
apt:
update: true
before_install:
- source ci/travis/before-install.sh
install:
- source $TRAVIS_BUILD_DIR/ci/travis/install_dependency.sh
script:
- $TRAVIS_BUILD_DIR/ci/travis/travis_build.sh

1
CHANGELOG.md
View File

@ -8,7 +8,6 @@ Please mark all change in change log and use the ticket from JIRA.
## Feature
- \#12 - Pure CPU version for Milvus
- #127 - Support new Index type IVFPQ
## Improvement

5
ci/jenkins/step/build.groovy
View File

@ -1,8 +1,7 @@
timeout(time: 60, unit: 'MINUTES') {
dir ("ci/jenkins/scripts") {
sh "./build.sh -l"
dir ("ci/scripts") {
withCredentials([usernamePassword(credentialsId: "${params.JFROG_CREDENTIALS_ID}", usernameVariable: 'USERNAME', passwordVariable: 'PASSWORD')]) {
sh "export JFROG_ARTFACTORY_URL='${params.JFROG_ARTFACTORY_URL}' && export JFROG_USER_NAME='${USERNAME}' && export JFROG_PASSWORD='${PASSWORD}' && ./build.sh -t ${params.BUILD_TYPE} -o /opt/milvus -d /opt/milvus -j -u -c"
sh "export JFROG_ARTFACTORY_URL='${params.JFROG_ARTFACTORY_URL}' && export JFROG_USER_NAME='${USERNAME}' && export JFROG_PASSWORD='${PASSWORD}' && ./build.sh -t ${params.BUILD_TYPE} -o /opt/milvus -l -g -j -u -c"
}
}
}

2
ci/jenkins/step/coverage.groovy
View File

@ -1,5 +1,5 @@
timeout(time: 30, unit: 'MINUTES') {
dir ("ci/jenkins/scripts") {
dir ("ci/scripts") {
sh "./coverage.sh -o /opt/milvus -u root -p 123456 -t \$POD_IP"
// Set some env variables so codecov detection script works correctly
withCredentials([[$class: 'StringBinding', credentialsId: "${env.PIPELINE_NAME}-codecov-token", variable: 'CODECOV_TOKEN']]) {

80
ci/jenkins/scripts/build.sh → ci/scripts/build.sh
View File

@ -1,5 +1,7 @@
#!/bin/bash
set -e
SOURCE="${BASH_SOURCE[0]}"
while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink
DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
@ -8,38 +10,37 @@ while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symli
done
SCRIPTS_DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
CMAKE_BUILD_DIR="${SCRIPTS_DIR}/../../../core/cmake_build"
MILVUS_CORE_DIR="${SCRIPTS_DIR}/../../core"
CORE_BUILD_DIR="${MILVUS_CORE_DIR}/cmake_build"
BUILD_TYPE="Debug"
BUILD_UNITTEST="OFF"
INSTALL_PREFIX="/opt/milvus"
FAISS_ROOT=""
BUILD_COVERAGE="OFF"
DB_PATH="/opt/milvus"
PROFILING="OFF"
USE_JFROG_CACHE="OFF"
RUN_CPPLINT="OFF"
CUSTOMIZATION="OFF" # default use ori faiss
GPU_VERSION="OFF"
WITH_MKL="OFF"
CUDA_COMPILER=/usr/local/cuda/bin/nvcc
CUSTOMIZED_FAISS_URL="${FAISS_URL:-NONE}"
wget -q --method HEAD ${CUSTOMIZED_FAISS_URL}
if [ $? -eq 0 ]; then
CUSTOMIZATION="ON"
else
CUSTOMIZATION="OFF"
fi
while getopts "o:d:t:ulcgjhx" arg
while getopts "o:t:b:f:gulcjmh" arg
do
case $arg in
o)
INSTALL_PREFIX=$OPTARG
;;
d)
DB_PATH=$OPTARG
;;
t)
BUILD_TYPE=$OPTARG # BUILD_TYPE
;;
b)
CORE_BUILD_DIR=$OPTARG # CORE_BUILD_DIR
;;
f)
FAISS_ROOT=$OPTARG # FAISS ROOT PATH
;;
g)
GPU_VERSION="ON";
;;
u)
echo "Build and run unittest cases" ;
BUILD_UNITTEST="ON";
@ -50,31 +51,30 @@ do
c)
BUILD_COVERAGE="ON"
;;
g)
PROFILING="ON"
;;
j)
USE_JFROG_CACHE="ON"
;;
x)
CUSTOMIZATION="OFF" # force use ori faiss
m)
WITH_MKL="ON"
;;
h) # help
echo "
parameter:
-o: install prefix(default: /opt/milvus)
-d: db data path(default: /opt/milvus)
-t: build type(default: Debug)
-b: core code build directory
-f: faiss root path
-g: gpu version
-u: building unit test options(default: OFF)
-l: run cpplint, clang-format and clang-tidy(default: OFF)
-c: code coverage(default: OFF)
-g: profiling(default: OFF)
-j: use jfrog cache build directory(default: OFF)
-m: build with MKL(default: OFF)
-h: help
usage:
./build.sh -p \${INSTALL_PREFIX} -t \${BUILD_TYPE} [-u] [-l] [-r] [-c] [-g] [-j] [-h]
./build.sh -o \${INSTALL_PREFIX} -t \${BUILD_TYPE} -b \${CORE_BUILD_DIR} -f \${FAISS_ROOT} [-u] [-l] [-c] [-j] [-m] [-h]
"
exit 0
;;
@ -85,31 +85,29 @@ usage:
esac
done
if [[ ! -d ${CMAKE_BUILD_DIR} ]]; then
mkdir ${CMAKE_BUILD_DIR}
if [[ ! -d ${CORE_BUILD_DIR} ]]; then
mkdir ${CORE_BUILD_DIR}
fi
cd ${CMAKE_BUILD_DIR}
# remove make cache since build.sh -l use default variables
# force update the variables each time
make rebuild_cache
cd ${CORE_BUILD_DIR}
CMAKE_CMD="cmake \
-DBUILD_UNIT_TEST=${BUILD_UNITTEST} \
-DCMAKE_INSTALL_PREFIX=${INSTALL_PREFIX}
-DCMAKE_BUILD_TYPE=${BUILD_TYPE} \
-DCMAKE_CUDA_COMPILER=${CUDA_COMPILER} \
-DMILVUS_GPU_VERSION=${GPU_VERSION} \
-DBUILD_UNIT_TEST=${BUILD_UNITTEST} \
-DBUILD_COVERAGE=${BUILD_COVERAGE} \
-DMILVUS_DB_PATH=${DB_PATH} \
-DMILVUS_ENABLE_PROFILING=${PROFILING} \
-DUSE_JFROG_CACHE=${USE_JFROG_CACHE} \
-DCUSTOMIZATION=${CUSTOMIZATION} \
-DFAISS_URL=${CUSTOMIZED_FAISS_URL} \
.."
-DFAISS_ROOT=${FAISS_ROOT} \
-DFAISS_WITH_MKL=${WITH_MKL} \
-DArrow_SOURCE=AUTO \
-DFAISS_SOURCE=AUTO \
${MILVUS_CORE_DIR}"
echo ${CMAKE_CMD}
${CMAKE_CMD}
if [[ ${RUN_CPPLINT} == "ON" ]]; then
# cpplint check
make lint
@ -135,8 +133,8 @@ if [[ ${RUN_CPPLINT} == "ON" ]]; then
# exit 1
# fi
# echo "clang-tidy check passed!"
else
# compile and build
make -j8 || exit 1
make install || exit 1
fi
# compile and build
make -j8 || exit 1
make install || exit 1

22
ci/jenkins/scripts/coverage.sh → ci/scripts/coverage.sh
View File

@ -9,18 +9,22 @@ done
SCRIPTS_DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
INSTALL_PREFIX="/opt/milvus"
CMAKE_BUILD_DIR="${SCRIPTS_DIR}/../../../core/cmake_build"
MILVUS_CORE_DIR="${SCRIPTS_DIR}/../../core"
CORE_BUILD_DIR="${MILVUS_CORE_DIR}/cmake_build"
MYSQL_USER_NAME=root
MYSQL_PASSWORD=123456
MYSQL_HOST='127.0.0.1'
MYSQL_PORT='3306'
while getopts "o:u:p:t:h" arg
while getopts "o:b:u:p:t:h" arg
do
case $arg in
o)
INSTALL_PREFIX=$OPTARG
;;
b)
CORE_BUILD_DIR=$OPTARG # CORE_BUILD_DIR
;;
u)
MYSQL_USER_NAME=$OPTARG
;;
@ -35,13 +39,14 @@ do
parameter:
-o: milvus install prefix(default: /opt/milvus)
-b: core code build directory
-u: mysql account
-p: mysql password
-t: mysql host
-h: help
usage:
./coverage.sh -o \${INSTALL_PREFIX} -u \${MYSQL_USER} -p \${MYSQL_PASSWORD} -t \${MYSQL_HOST} [-h]
./coverage.sh -o \${INSTALL_PREFIX} -b \${CORE_BUILD_DIR} -u \${MYSQL_USER} -p \${MYSQL_PASSWORD} -t \${MYSQL_HOST} [-h]
"
exit 0
;;
@ -63,12 +68,14 @@ FILE_INFO_OUTPUT="output.info"
FILE_INFO_OUTPUT_NEW="output_new.info"
DIR_LCOV_OUTPUT="lcov_out"
DIR_GCNO="${CMAKE_BUILD_DIR}"
DIR_GCNO="${CORE_BUILD_DIR}"
DIR_UNITTEST="${INSTALL_PREFIX}/unittest"
cd ${SCRIPTS_DIR}
# delete old code coverage info files
rm -rf lcov_out
rm -f FILE_INFO_BASE FILE_INFO_MILVUS FILE_INFO_OUTPUT FILE_INFO_OUTPUT_NEW
rm -rf ${DIR_LCOV_OUTPUT}
rm -f ${FILE_INFO_BASE} ${FILE_INFO_MILVUS} ${FILE_INFO_OUTPUT} ${FILE_INFO_OUTPUT_NEW}
MYSQL_DB_NAME=milvus_`date +%s%N`
@ -132,8 +139,7 @@ ${LCOV_CMD} -r "${FILE_INFO_OUTPUT}" -o "${FILE_INFO_OUTPUT_NEW}" \
"*/src/server/Server.cpp" \
"*/src/server/DBWrapper.cpp" \
"*/src/server/grpc_impl/GrpcServer.cpp" \
"*/src/external/easyloggingpp/easylogging++.h" \
"*/src/external/easyloggingpp/easylogging++.cc"
"*/thirdparty/*"
if [ $? -ne 0 ]; then
echo "gen ${FILE_INFO_OUTPUT_NEW} failed"

13
ci/travis/before-install.sh Executable file
View File

@ -0,0 +1,13 @@
#!/bin/bash
set -ex
if [[ "${TRAVIS_OS_NAME}" == "linux" ]]; then
export CCACHE_COMPRESS=1
export CCACHE_COMPRESSLEVEL=5
export CCACHE_COMPILERCHECK=content
export PATH=/usr/lib/ccache/:$PATH
ccache --show-stats
fi
set +ex

44
ci/travis/install_dependency.sh Executable file
View File

@ -0,0 +1,44 @@
#!/usr/bin/env bash
set -e
wget -P /tmp https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2019.PUB
sudo apt-key add /tmp/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2019.PUB
echo "deb https://apt.repos.intel.com/mkl all main" | \
sudo tee /etc/apt/sources.list.d/intel-mkl.list
sudo wget -O /usr/share/keyrings/apache-arrow-keyring.gpg https://dl.bintray.com/apache/arrow/$(lsb_release --id --short | tr 'A-Z' 'a-z')/apache-arrow-keyring.gpg
sudo tee /etc/apt/sources.list.d/apache-arrow.list <<APT_LINE
deb [arch=amd64 signed-by=/usr/share/keyrings/apache-arrow-keyring.gpg] https://dl.bintray.com/apache/arrow/$(lsb_release --id --short | tr 'A-Z' 'a-z')/ $(lsb_release --codename --short) main
deb-src [signed-by=/usr/share/keyrings/apache-arrow-keyring.gpg] https://dl.bintray.com/apache/arrow/$(lsb_release --id --short | tr 'A-Z' 'a-z')/ $(lsb_release --codename --short) main
APT_LINE
sudo apt-get update -qq
sudo apt-get install -y -q --no-install-recommends \
gfortran \
lsb-core \
libtool \
automake \
ccache \
pkg-config \
libarrow-dev \
libjemalloc-dev \
libboost-serialization-dev \
libboost-filesystem-dev \
libboost-system-dev \
libboost-regex-dev \
intel-mkl-gnu-2019.5-281 \
intel-mkl-core-2019.5-281 \
libmysqlclient-dev \
clang-format-6.0 \
clang-tidy-6.0 \
lcov
sudo ln -s /usr/lib/x86_64-linux-gnu/libmysqlclient.so \
/usr/lib/x86_64-linux-gnu/libmysqlclient_r.so
export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/opt/intel/compilers_and_libraries_2019.5.281/linux/mkl/lib/intel64

24
ci/travis/travis_build.sh Executable file
View File

@ -0,0 +1,24 @@
#!/usr/bin/env bash
set -ex
source $TRAVIS_BUILD_DIR/ci/travis/travis_env_common.sh
only_library_mode=no
while true; do
case "$1" in
--only-library)
only_library_mode=yes
shift ;;
*) break ;;
esac
done
BUILD_COMMON_FLAGS="-t ${MILVUS_BUILD_TYPE} -o ${MILVUS_INSTALL_PREFIX} -b ${MILVUS_BUILD_DIR}"
if [ $only_library_mode == "yes" ]; then
${TRAVIS_BUILD_DIR}/ci/scripts/build.sh ${BUILD_COMMON_FLAGS} -m
else
${TRAVIS_BUILD_DIR}/ci/scripts/build.sh ${BUILD_COMMON_FLAGS} -m -u -c
fi

10
ci/travis/travis_env_common.sh Normal file
View File

@ -0,0 +1,10 @@
export MILVUS_CORE_DIR=${TRAVIS_BUILD_DIR}/core
export MILVUS_BUILD_DIR=${TRAVIS_BUILD_DIR}/core/cmake_build
export MILVUS_INSTALL_PREFIX=/opt/milvus
export MILVUS_TRAVIS_COVERAGE=${MILVUS_TRAVIS_COVERAGE:=0}
if [ "${MILVUS_TRAVIS_COVERAGE}" == "1" ]; then
export MILVUS_CPP_COVERAGE_FILE=${TRAVIS_BUILD_DIR}/output_new.info
fi
export MILVUS_BUILD_TYPE=${MILVUS_BUILD_TYPE:=Release}

51
core/CMakeLists.txt
View File

@ -18,7 +18,7 @@
#-------------------------------------------------------------------------------
cmake_minimum_required(VERSION 3.14)
cmake_minimum_required(VERSION 3.12)
message(STATUS "Building using CMake version: ${CMAKE_VERSION}")
set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_CURRENT_SOURCE_DIR}/cmake")
@ -31,16 +31,14 @@ GET_CURRENT_TIME(BUILD_TIME)
string(REGEX REPLACE "\n" "" BUILD_TIME ${BUILD_TIME})
message(STATUS "Build time = ${BUILD_TIME}")
MACRO (GET_GIT_BRANCH_NAME GIT_BRANCH_NAME)
execute_process(COMMAND "git" rev-parse --abbrev-ref HEAD OUTPUT_VARIABLE ${GIT_BRANCH_NAME})
if(GIT_BRANCH_NAME STREQUAL "")
execute_process(COMMAND "git" symbolic-ref --short -q HEAD OUTPUT_VARIABLE ${GIT_BRANCH_NAME})
endif()
ENDMACRO (GET_GIT_BRANCH_NAME)
MACRO(GET_GIT_BRANCH_NAME GIT_BRANCH_NAME)
execute_process(COMMAND sh "-c" "git log --decorate | head -n 1 | sed 's/.*(\\(.*\\))/\\1/' | sed 's/.* \\(.*\\),.*/\\1/' | sed 's=[a-zA-Z]*\/==g'"
OUTPUT_VARIABLE ${GIT_BRANCH_NAME})
ENDMACRO(GET_GIT_BRANCH_NAME)
GET_GIT_BRANCH_NAME(GIT_BRANCH_NAME)
message(STATUS "GIT_BRANCH_NAME = ${GIT_BRANCH_NAME}")
if(NOT GIT_BRANCH_NAME STREQUAL "")
if (NOT GIT_BRANCH_NAME STREQUAL "")
string(REGEX REPLACE "\n" "" GIT_BRANCH_NAME ${GIT_BRANCH_NAME})
endif ()
@ -69,7 +67,7 @@ if (MILVUS_VERSION_MAJOR STREQUAL ""
OR MILVUS_VERSION_PATCH STREQUAL "")
message(WARNING "Failed to determine Milvus version from git branch name")
set(MILVUS_VERSION "0.6.0")
endif()
endif ()
message(STATUS "Build version = ${MILVUS_VERSION}")
configure_file(${CMAKE_CURRENT_SOURCE_DIR}/src/config.h.in ${CMAKE_CURRENT_SOURCE_DIR}/src/config.h @ONLY)
@ -112,23 +110,36 @@ endif ()
set(MILVUS_SOURCE_DIR ${PROJECT_SOURCE_DIR})
set(MILVUS_BINARY_DIR ${PROJECT_BINARY_DIR})
set(MILVUS_ENGINE_SRC ${PROJECT_SOURCE_DIR}/src)
set(MILVUS_THIRDPARTY_SRC ${PROJECT_SOURCE_DIR}/thirdparty)
include(ExternalProject)
include(DefineOptions)
include(BuildUtils)
include(ThirdPartyPackages)
set(MILVUS_GPU_VERSION false)
if (MILVUS_CPU_VERSION)
message(STATUS "Building Milvus CPU version")
add_compile_definitions("MILVUS_CPU_VERSION")
else ()
if(MILVUS_USE_CCACHE)
find_program(CCACHE_FOUND ccache)
if(CCACHE_FOUND)
message(STATUS "Using ccache: ${CCACHE_FOUND}")
set_property(GLOBAL PROPERTY RULE_LAUNCH_COMPILE ${CCACHE_FOUND})
set_property(GLOBAL PROPERTY RULE_LAUNCH_LINK ${CCACHE_FOUND})
# let ccache preserve C++ comments, because some of them may be
# meaningful to the compiler
set(ENV{CCACHE_COMMENTS} "1")
endif(CCACHE_FOUND)
endif()
set(MILVUS_CPU_VERSION false)
if (MILVUS_GPU_VERSION)
message(STATUS "Building Milvus GPU version")
set(MILVUS_GPU_VERSION true)
add_compile_definitions("MILVUS_GPU_VERSION")
enable_language(CUDA)
find_package(CUDA 10 REQUIRED)
set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -Xcompiler -fPIC -std=c++11 -D_FORCE_INLINES --expt-extended-lambda")
else ()
message(STATUS "Building Milvus CPU version")
set(MILVUS_CPU_VERSION true)
add_compile_definitions("MILVUS_CPU_VERSION")
endif ()
if (CMAKE_BUILD_TYPE STREQUAL "Release")
@ -162,7 +173,13 @@ add_custom_target(Clean-All COMMAND ${CMAKE_BUILD_TOOL} clean)
if ("${MILVUS_DB_PATH}" STREQUAL "")
set(MILVUS_DB_PATH "/tmp/milvus")
endif ()
configure_file(${CMAKE_CURRENT_SOURCE_DIR}/conf/server_config.template ${CMAKE_CURRENT_SOURCE_DIR}/conf/server_config.yaml)
if (MILVUS_GPU_VERSION)
configure_file(${CMAKE_CURRENT_SOURCE_DIR}/conf/server_gpu_config.template ${CMAKE_CURRENT_SOURCE_DIR}/conf/server_config.yaml)
else()
configure_file(${CMAKE_CURRENT_SOURCE_DIR}/conf/server_cpu_config.template ${CMAKE_CURRENT_SOURCE_DIR}/conf/server_config.yaml)
endif()
configure_file(${CMAKE_CURRENT_SOURCE_DIR}/conf/log_config.template ${CMAKE_CURRENT_SOURCE_DIR}/conf/log_config.conf)
install(DIRECTORY scripts/
@ -273,4 +290,4 @@ if (${CLANG_TIDY_FOUND})
--source_dir
${CMAKE_CURRENT_SOURCE_DIR}/src
${MILVUS_LINT_QUIET})
endif ()
endif ()

2
core/build-support/lint_exclusions.txt
View File

@ -6,5 +6,5 @@
*easylogging++*
*SqliteMetaImpl.cpp
*src/grpc*
*src/external*
*thirdparty*
*milvus/include*

38
core/build.sh
View File

@ -12,18 +12,12 @@ USE_JFROG_CACHE="OFF"
RUN_CPPLINT="OFF"
CUSTOMIZATION="OFF" # default use ori faiss
CUDA_COMPILER=/usr/local/cuda/bin/nvcc
CPU_VERSION="OFF"
GPU_VERSION="OFF" #defaults to CPU version
WITH_MKL="OFF"
FAISS_ROOT=""
FAISS_SOURCE="BUNDLED"
CUSTOMIZED_FAISS_URL="${FAISS_URL:-NONE}"
wget -q --method HEAD ${CUSTOMIZED_FAISS_URL}
if [ $? -eq 0 ]; then
CUSTOMIZATION="ON"
else
CUSTOMIZATION="OFF"
fi
while getopts "p:d:t:ulrcgjhxzm" arg
while getopts "p:d:t:f:ulrcgjhxzm" arg
do
case $arg in
p)
@ -35,6 +29,10 @@ do
t)
BUILD_TYPE=$OPTARG # BUILD_TYPE
;;
f)
FAISS_ROOT=$OPTARG
FAISS_SOURCE="AUTO"
;;
u)
echo "Build and run unittest cases" ;
BUILD_UNITTEST="ON";
@ -51,7 +49,7 @@ do
c)
BUILD_COVERAGE="ON"
;;
g)
z)
PROFILING="ON"
;;
j)
@ -60,8 +58,8 @@ do
x)
CUSTOMIZATION="OFF" # force use ori faiss
;;
z)
CPU_VERSION="ON"
g)
GPU_VERSION="ON"
;;
m)
WITH_MKL="ON"
@ -73,18 +71,19 @@ parameter:
-p: install prefix(default: $(pwd)/milvus)
-d: db data path(default: /tmp/milvus)
-t: build type(default: Debug)
-f: faiss root path(default: empty)
-u: building unit test options(default: OFF)
-l: run cpplint, clang-format and clang-tidy(default: OFF)
-r: remove previous build directory(default: OFF)
-c: code coverage(default: OFF)
-g: profiling(default: OFF)
-z: profiling(default: OFF)
-j: use jfrog cache build directory(default: OFF)
-z: build pure CPU version(default: OFF)
-g: build GPU version(default: OFF)
-m: build with MKL(default: OFF)
-h: help
usage:
./build.sh -p \${INSTALL_PREFIX} -t \${BUILD_TYPE} [-u] [-l] [-r] [-c] [-g] [-j] [-z] [-m] [-h]
./build.sh -p \${INSTALL_PREFIX} -t \${BUILD_TYPE} -f \${FAISS_ROOT} [-u] [-l] [-r] [-c] [-z] [-j] [-g] [-m] [-h]
"
exit 0
;;
@ -109,15 +108,16 @@ CMAKE_CMD="cmake \
-DBUILD_UNIT_TEST=${BUILD_UNITTEST} \
-DCMAKE_INSTALL_PREFIX=${INSTALL_PREFIX}
-DCMAKE_BUILD_TYPE=${BUILD_TYPE} \
-DFAISS_ROOT=${FAISS_ROOT} \
-DFAISS_SOURCE=${FAISS_SOURCE} \
-DCMAKE_CUDA_COMPILER=${CUDA_COMPILER} \
-DBUILD_COVERAGE=${BUILD_COVERAGE} \
-DMILVUS_DB_PATH=${DB_PATH} \
-DMILVUS_ENABLE_PROFILING=${PROFILING} \
-DUSE_JFROG_CACHE=${USE_JFROG_CACHE} \
-DCUSTOMIZATION=${CUSTOMIZATION} \
-DFAISS_URL=${CUSTOMIZED_FAISS_URL} \
-DMILVUS_CPU_VERSION=${CPU_VERSION} \
-DBUILD_FAISS_WITH_MKL=${WITH_MKL} \
-DMILVUS_GPU_VERSION=${GPU_VERSION} \
-DFAISS_WITH_MKL=${WITH_MKL} \
../"
echo ${CMAKE_CMD}
${CMAKE_CMD}

88
core/cmake/DefineOptions.cmake
View File

@ -13,16 +13,16 @@ macro(define_option name description default)
endmacro()
function(list_join lst glue out)
if("${${lst}}" STREQUAL "")
if ("${${lst}}" STREQUAL "")
set(${out} "" PARENT_SCOPE)
return()
endif()
endif ()
list(GET ${lst} 0 joined)
list(REMOVE_AT ${lst} 0)
foreach(item ${${lst}})
foreach (item ${${lst}})
set(joined "${joined}${glue}${item}")
endforeach()
endforeach ()
set(${out} ${joined} PARENT_SCOPE)
endfunction()
@ -35,27 +35,29 @@ macro(define_option_string name description default)
set("${name}_OPTION_ENUM" ${ARGN})
list_join("${name}_OPTION_ENUM" "|" "${name}_OPTION_ENUM")
if(NOT ("${${name}_OPTION_ENUM}" STREQUAL ""))
if (NOT ("${${name}_OPTION_ENUM}" STREQUAL ""))
set_property(CACHE ${name} PROPERTY STRINGS ${ARGN})
endif()
endif ()
endmacro()
#----------------------------------------------------------------------
set_option_category("CPU version")
set_option_category("GPU version")
define_option(MILVUS_CPU_VERSION "Build CPU version only" OFF)
define_option(MILVUS_GPU_VERSION "Build GPU version" OFF)
#----------------------------------------------------------------------
set_option_category("Thirdparty")
set(MILVUS_DEPENDENCY_SOURCE_DEFAULT "AUTO")
set(MILVUS_DEPENDENCY_SOURCE_DEFAULT "BUNDLED")
define_option_string(MILVUS_DEPENDENCY_SOURCE
"Method to use for acquiring MILVUS's build dependencies"
"${MILVUS_DEPENDENCY_SOURCE_DEFAULT}"
"AUTO"
"BUNDLED"
"SYSTEM")
"Method to use for acquiring MILVUS's build dependencies"
"${MILVUS_DEPENDENCY_SOURCE_DEFAULT}"
"AUTO"
"BUNDLED"
"SYSTEM")
define_option(MILVUS_USE_CCACHE "Use ccache when compiling (if available)" ON)
define_option(MILVUS_VERBOSE_THIRDPARTY_BUILD
"Show output from ExternalProjects rather than just logging to files" ON)
@ -75,33 +77,21 @@ define_option(MILVUS_WITH_YAMLCPP "Build with yaml-cpp library" ON)
if (MILVUS_ENABLE_PROFILING STREQUAL "ON")
define_option(MILVUS_WITH_LIBUNWIND "Build with libunwind" ON)
define_option(MILVUS_WITH_GPERFTOOLS "Build with gperftools" ON)
endif()
endif ()
define_option(MILVUS_WITH_GRPC "Build with GRPC" ON)
define_option(MILVUS_WITH_ZLIB "Build with zlib compression" ON)
#----------------------------------------------------------------------
if(MSVC)
set_option_category("MSVC")
define_option(MSVC_LINK_VERBOSE
"Pass verbose linking options when linking libraries and executables"
OFF)
define_option(MILVUS_USE_STATIC_CRT "Build MILVUS with statically linked CRT" OFF)
endif()
#----------------------------------------------------------------------
set_option_category("Test and benchmark")
unset(MILVUS_BUILD_TESTS CACHE)
if (BUILD_UNIT_TEST)
define_option(MILVUS_BUILD_TESTS "Build the MILVUS googletest unit tests" ON)
else()
else ()
define_option(MILVUS_BUILD_TESTS "Build the MILVUS googletest unit tests" OFF)
endif(BUILD_UNIT_TEST)
endif (BUILD_UNIT_TEST)
#----------------------------------------------------------------------
macro(config_summary)
@ -113,12 +103,12 @@ macro(config_summary)
message(STATUS " Generator: ${CMAKE_GENERATOR}")
message(STATUS " Build type: ${CMAKE_BUILD_TYPE}")
message(STATUS " Source directory: ${CMAKE_CURRENT_SOURCE_DIR}")
if(${CMAKE_EXPORT_COMPILE_COMMANDS})
if (${CMAKE_EXPORT_COMPILE_COMMANDS})
message(
STATUS " Compile commands: ${CMAKE_CURRENT_BINARY_DIR}/compile_commands.json")
endif()
endif ()
foreach(category ${MILVUS_OPTION_CATEGORIES})
foreach (category ${MILVUS_OPTION_CATEGORIES})
message(STATUS)
message(STATUS "${category} options:")
@ -126,50 +116,50 @@ macro(config_summary)
set(option_names ${MILVUS_${category}_OPTION_NAMES})
set(max_value_length 0)
foreach(name ${option_names})
foreach (name ${option_names})
string(LENGTH "\"${${name}}\"" value_length)
if(${max_value_length} LESS ${value_length})
if (${max_value_length} LESS ${value_length})
set(max_value_length ${value_length})
endif()
endforeach()
endif ()
endforeach ()
foreach(name ${option_names})
if("${${name}_OPTION_TYPE}" STREQUAL "string")
foreach (name ${option_names})
if ("${${name}_OPTION_TYPE}" STREQUAL "string")
set(value "\"${${name}}\"")
else()
else ()
set(value "${${name}}")
endif()
endif ()
set(default ${${name}_OPTION_DEFAULT})
set(description ${${name}_OPTION_DESCRIPTION})
string(LENGTH ${description} description_length)
if(${description_length} LESS 70)
if (${description_length} LESS 70)
string(
SUBSTRING
" "
${description_length} -1 description_padding)
else()
else ()
set(description_padding "
")
endif()
endif ()
set(comment "[${name}]")
if("${value}" STREQUAL "${default}")
if ("${value}" STREQUAL "${default}")
set(comment "[default] ${comment}")
endif()
endif ()
if(NOT ("${${name}_OPTION_ENUM}" STREQUAL ""))
if (NOT ("${${name}_OPTION_ENUM}" STREQUAL ""))
set(comment "${comment} [${${name}_OPTION_ENUM}]")
endif()
endif ()
string(
SUBSTRING "${value} "
0 ${max_value_length} value)
message(STATUS " ${description} ${description_padding} ${value} ${comment}")
endforeach()
endforeach ()
endforeach()
endforeach ()
endmacro()

6
core/cmake/ThirdPartyPackages.cmake
View File

@ -164,8 +164,10 @@ endif ()
macro(resolve_dependency DEPENDENCY_NAME)
if (${DEPENDENCY_NAME}_SOURCE STREQUAL "AUTO")
#disable find_package for now
build_dependency(${DEPENDENCY_NAME})
find_package(${DEPENDENCY_NAME} MODULE)
if(NOT ${${DEPENDENCY_NAME}_FOUND})
build_dependency(${DEPENDENCY_NAME})
endif()
elseif (${DEPENDENCY_NAME}_SOURCE STREQUAL "BUNDLED")
build_dependency(${DEPENDENCY_NAME})
elseif (${DEPENDENCY_NAME}_SOURCE STREQUAL "SYSTEM")

43
core/conf/server_cpu_config.template Normal file
View File

@ -0,0 +1,43 @@
# Default values are used when you make no changes to the following parameters.
server_config:
address: 0.0.0.0 # milvus server ip address (IPv4)
port: 19530 # milvus server port, must in range [1025, 65534]
deploy_mode: single # deployment type: single, cluster_readonly, cluster_writable
time_zone: UTC+8 # time zone, must be in format: UTC+X
db_config:
primary_path: @MILVUS_DB_PATH@ # path used to store data and meta
secondary_path: # path used to store data only, split by semicolon
backend_url: sqlite://:@:/ # URI format: dialect://username:password@host:port/database
# Keep 'dialect://:@:/', and replace other texts with real values
# Replace 'dialect' with 'mysql' or 'sqlite'
insert_buffer_size: 4 # GB, maximum insert buffer size allowed, must be a positive integer
# sum of insert_buffer_size and cpu_cache_capacity cannot exceed total memory
preload_table: # preload data at startup, '*' means load all tables, empty value means no preload
# you can specify preload tables like this: table1,table2,table3
metric_config:
enable_monitor: false # enable monitoring or not, must be a boolean
collector: prometheus # prometheus
prometheus_config:
port: 8080 # port prometheus uses to fetch metrics, must in range [1025, 65534]
cache_config:
cpu_cache_capacity: 16 # GB, CPU memory used for cache, must be a positive integer
cpu_cache_threshold: 0.85 # percentage of data that will be kept when cache cleanup is triggered, must be in range (0.0, 1.0]
cache_insert_data: false # whether to load inserted data into cache, must be a boolean
engine_config:
use_blas_threshold: 1100 # if nq < use_blas_threshold, use SSE, faster with fluctuated response times
# if nq >= use_blas_threshold, use OpenBlas, slower with stable response times
gpu_search_threshold: 1000 # threshold beyond which the search computation is executed on GPUs only
resource_config:
search_resources: # define the device used for search computation
- cpu
index_build_device: cpu # CPU used for building index

9
core/conf/server_config.template → core/conf/server_gpu_config.template
View File

@ -27,14 +27,11 @@ metric_config:
port: 8080 # port prometheus uses to fetch metrics, must in range [1025, 65534]
cache_config:
cpu_cache_capacity: 16 # GB, CPU memory used for cache, must be a positive integer
cpu_cache_threshold: 0.85 # percentage of data that will be kept when cache cleanup is triggered, must be in range (0.0, 1.0]
cache_insert_data: false # whether to load inserted data into cache, must be a boolean
# Skip the following config if you are using GPU version
gpu_cache_capacity: 4 # GB, GPU memory used for cache, must be a positive integer
gpu_cache_capacity: 3 # GB, GPU memory used for cache, must be a positive integer
gpu_cache_threshold: 0.85 # percentage of data that will be kept when cache cleanup is triggered, must be in range (0.0, 1.0]
cache_insert_data: false # whether to load inserted data into cache, must be a boolean
engine_config:
use_blas_threshold: 1100 # if nq < use_blas_threshold, use SSE, faster with fluctuated response times
@ -45,4 +42,4 @@ resource_config:
search_resources: # define the devices used for search computation, must be in format: cpu or gpux
- cpu
- gpu0
index_build_device: gpu0 # CPU / GPU used for building index, must be in format: cpu / gpux
index_build_device: gpu0 # CPU / GPU used for building index, must be in format: cpu or gpux

4
core/coverage.sh
View File

@ -122,9 +122,7 @@ ${LCOV_CMD} -r "${FILE_INFO_OUTPUT}" -o "${FILE_INFO_OUTPUT_NEW}" \
"*/src/server/Server.cpp" \
"*/src/server/DBWrapper.cpp" \
"*/src/server/grpc_impl/GrpcServer.cpp" \
"*/easylogging++.h" \
"*/easylogging++.cc" \
"*/src/external/*"
"*/thirdparty/*"
if [ $? -ne 0 ]; then
echo "generate ${FILE_INFO_OUTPUT_NEW} failed"

28
core/migration/README.md Normal file
View File

@ -0,0 +1,28 @@
## Data Migration
####0.3.x
legacy data is not migrate-able for later versions
####0.4.x
legacy data can be reused directly by 0.5.x
legacy data can be migrated to 0.6.x
####0.5.x
legacy data can be migrated to 0.6.x
####0.6.x
how to migrate legacy 0.4.x/0.5.x data
for sqlite meta:
```shell
$ sqlite3 [parth_to]/meta.sqlite < sqlite_4_to_6.sql
```
for mysql meta:
```shell
$ mysql -h127.0.0.1 -uroot -p123456 -Dmilvus < mysql_4_to_6.sql
```

4
core/migration/mysql_4_to_6.sql Normal file
View File

@ -0,0 +1,4 @@
alter table Tables add column owner_table VARCHAR(255) DEFAULT '' NOT NULL;
alter table Tables add column partition_tag VARCHAR(255) DEFAULT '' NOT NULL;
alter table Tables add column version VARCHAR(64) DEFAULT '0.6.0' NOT NULL;
update Tables set version='0.6.0';

4
core/migration/sqlite_4_to_6.sql Normal file
View File

@ -0,0 +1,4 @@
alter table Tables add column 'owner_table' TEXT DEFAULT '' NOT NULL;
alter table Tables add column 'partition_tag' TEXT DEFAULT '' NOT NULL;
alter table Tables add column 'version' TEXT DEFAULT '0.6.0' NOT NULL;
update Tables set version='0.6.0';

16
core/src/CMakeLists.txt
View File

@ -19,11 +19,15 @@
include_directories(${MILVUS_SOURCE_DIR})
include_directories(${MILVUS_ENGINE_SRC})
include_directories(${MILVUS_THIRDPARTY_SRC})
include_directories(${MILVUS_ENGINE_SRC}/grpc/gen-status)
include_directories(${MILVUS_ENGINE_SRC}/grpc/gen-milvus)
add_subdirectory(index)
if (FAISS_WITH_MKL)
add_compile_definitions("WITH_MKL")
endif ()
set(INDEX_INCLUDE_DIRS ${INDEX_INCLUDE_DIRS} PARENT_SCOPE)
foreach (dir ${INDEX_INCLUDE_DIRS})
@ -62,11 +66,11 @@ set(scheduler_files
${scheduler_task_files}
)
aux_source_directory(${MILVUS_ENGINE_SRC}/external/easyloggingpp external_easyloggingpp_files)
aux_source_directory(${MILVUS_ENGINE_SRC}/external/nlohmann external_nlohmann_files)
set(external_files
${external_easyloggingpp_files}
${external_nlohmann_files}
aux_source_directory(${MILVUS_THIRDPARTY_SRC}/easyloggingpp thirdparty_easyloggingpp_files)
aux_source_directory(${MILVUS_THIRDPARTY_SRC}/nlohmann thirdparty_nlohmann_files)
set(thirdparty_files
${thirdparty_easyloggingpp_files}
${thirdparty_nlohmann_files}
)
aux_source_directory(${MILVUS_ENGINE_SRC}/server server_files)
@ -82,7 +86,7 @@ set(engine_files
${db_insert_files}
${db_meta_files}
${metrics_files}
${external_files}
${thirdparty_files}
${utils_files}
${wrapper_files}
)

41
core/src/db/DB.h
View File

@ -47,43 +47,68 @@ class DB {
virtual Status
CreateTable(meta::TableSchema& table_schema_) = 0;
virtual Status
DeleteTable(const std::string& table_id, const meta::DatesT& dates) = 0;
DropTable(const std::string& table_id, const meta::DatesT& dates) = 0;
virtual Status
DescribeTable(meta::TableSchema& table_schema_) = 0;
virtual Status
HasTable(const std::string& table_id, bool& has_or_not_) = 0;
virtual Status
AllTables(std::vector<meta::TableSchema>& table_schema_array) = 0;
virtual Status
GetTableRowCount(const std::string& table_id, uint64_t& row_count) = 0;
virtual Status
PreloadTable(const std::string& table_id) = 0;
virtual Status
UpdateTableFlag(const std::string& table_id, int64_t flag) = 0;
virtual Status
InsertVectors(const std::string& table_id_, uint64_t n, const float* vectors, IDNumbers& vector_ids_) = 0;
CreatePartition(const std::string& table_id, const std::string& partition_name,
const std::string& partition_tag) = 0;
virtual Status
Query(const std::string& table_id, uint64_t k, uint64_t nq, uint64_t nprobe, const float* vectors,
QueryResults& results) = 0;
DropPartition(const std::string& partition_name) = 0;
virtual Status
Query(const std::string& table_id, uint64_t k, uint64_t nq, uint64_t nprobe, const float* vectors,
const meta::DatesT& dates, QueryResults& results) = 0;
DropPartitionByTag(const std::string& table_id, const std::string& partition_tag) = 0;
virtual Status
Query(const std::string& table_id, const std::vector<std::string>& file_ids, uint64_t k, uint64_t nq,
uint64_t nprobe, const float* vectors, const meta::DatesT& dates, QueryResults& results) = 0;
ShowPartitions(const std::string& table_id, std::vector<meta::TableSchema>& partiton_schema_array) = 0;
virtual Status
InsertVectors(const std::string& table_id, const std::string& partition_tag, uint64_t n, const float* vectors,
IDNumbers& vector_ids_) = 0;
virtual Status
Query(const std::string& table_id, const std::vector<std::string>& partition_tags, uint64_t k, uint64_t nq,
uint64_t nprobe, const float* vectors, ResultIds& result_ids, ResultDistances& result_distances) = 0;
virtual Status
Query(const std::string& table_id, const std::vector<std::string>& partition_tags, uint64_t k, uint64_t nq,
uint64_t nprobe, const float* vectors, const meta::DatesT& dates, ResultIds& result_ids,
ResultDistances& result_distances) = 0;
virtual Status
QueryByFileID(const std::string& table_id, const std::vector<std::string>& file_ids, uint64_t k, uint64_t nq,
uint64_t nprobe, const float* vectors, const meta::DatesT& dates, ResultIds& result_ids,
ResultDistances& result_distances) = 0;
virtual Status
Size(uint64_t& result) = 0;
virtual Status
CreateIndex(const std::string& table_id, const TableIndex& index) = 0;
virtual Status
DescribeIndex(const std::string& table_id, TableIndex& index) = 0;
virtual Status
DropIndex(const std::string& table_id) = 0;

484
core/src/db/DBImpl.cpp
View File

@ -30,6 +30,7 @@
#include "scheduler/job/DeleteJob.h"
#include "scheduler/job/SearchJob.h"
#include "utils/Log.h"
#include "utils/StringHelpFunctions.h"
#include "utils/TimeRecorder.h"
#include <assert.h>
@ -38,6 +39,7 @@
#include <chrono>
#include <cstring>
#include <iostream>
#include <set>
#include <thread>
namespace milvus {
@ -49,6 +51,17 @@ constexpr uint64_t METRIC_ACTION_INTERVAL = 1;
constexpr uint64_t COMPACT_ACTION_INTERVAL = 1;
constexpr uint64_t INDEX_ACTION_INTERVAL = 1;
static const Status SHUTDOWN_ERROR = Status(DB_ERROR, "Milsvus server is shutdown!");
void
TraverseFiles(const meta::DatePartionedTableFilesSchema& date_files, meta::TableFilesSchema& files_array) {
for (auto& day_files : date_files) {
for (auto& file : day_files.second) {
files_array.push_back(file);
}
}
}
} // namespace
DBImpl::DBImpl(const DBOptions& options)
@ -113,7 +126,7 @@ DBImpl::DropAll() {
Status
DBImpl::CreateTable(meta::TableSchema& table_schema) {
if (shutting_down_.load(std::memory_order_acquire)) {
return Status(DB_ERROR, "Milsvus server is shutdown!");
return SHUTDOWN_ERROR;
}
meta::TableSchema temp_schema = table_schema;
@ -122,34 +135,18 @@ DBImpl::CreateTable(meta::TableSchema& table_schema) {
}
Status
DBImpl::DeleteTable(const std::string& table_id, const meta::DatesT& dates) {
DBImpl::DropTable(const std::string& table_id, const meta::DatesT& dates) {
if (shutting_down_.load(std::memory_order_acquire)) {
return Status(DB_ERROR, "Milsvus server is shutdown!");
return SHUTDOWN_ERROR;
}
// dates partly delete files of the table but currently we don't support
ENGINE_LOG_DEBUG << "Prepare to delete table " << table_id;
if (dates.empty()) {
mem_mgr_->EraseMemVector(table_id); // not allow insert
meta_ptr_->DeleteTable(table_id); // soft delete table
// scheduler will determine when to delete table files
auto nres = scheduler::ResMgrInst::GetInstance()->GetNumOfComputeResource();
scheduler::DeleteJobPtr job = std::make_shared<scheduler::DeleteJob>(table_id, meta_ptr_, nres);
scheduler::JobMgrInst::GetInstance()->Put(job);
job->WaitAndDelete();
} else {
meta_ptr_->DropPartitionsByDates(table_id, dates);
}
return Status::OK();
return DropTableRecursively(table_id, dates);
}
Status
DBImpl::DescribeTable(meta::TableSchema& table_schema) {
if (shutting_down_.load(std::memory_order_acquire)) {
return Status(DB_ERROR, "Milsvus server is shutdown!");
return SHUTDOWN_ERROR;
}
auto stat = meta_ptr_->DescribeTable(table_schema);
@ -160,7 +157,7 @@ DBImpl::DescribeTable(meta::TableSchema& table_schema) {
Status
DBImpl::HasTable(const std::string& table_id, bool& has_or_not) {
if (shutting_down_.load(std::memory_order_acquire)) {
return Status(DB_ERROR, "Milsvus server is shutdown!");
return SHUTDOWN_ERROR;
}
return meta_ptr_->HasTable(table_id, has_or_not);
@ -169,7 +166,7 @@ DBImpl::HasTable(const std::string& table_id, bool& has_or_not) {
Status
DBImpl::AllTables(std::vector<meta::TableSchema>& table_schema_array) {
if (shutting_down_.load(std::memory_order_acquire)) {
return Status(DB_ERROR, "Milsvus server is shutdown!");
return SHUTDOWN_ERROR;
}
return meta_ptr_->AllTables(table_schema_array);
@ -178,55 +175,59 @@ DBImpl::AllTables(std::vector<meta::TableSchema>& table_schema_array) {
Status
DBImpl::PreloadTable(const std::string& table_id) {
if (shutting_down_.load(std::memory_order_acquire)) {
return Status(DB_ERROR, "Milsvus server is shutdown!");
return SHUTDOWN_ERROR;
}
meta::DatePartionedTableFilesSchema files;
meta::DatesT dates;
// get all table files from parent table
std::vector<size_t> ids;
auto status = meta_ptr_->FilesToSearch(table_id, ids, dates, files);
meta::TableFilesSchema files_array;
auto status = GetFilesToSearch(table_id, ids, files_array);
if (!status.ok()) {
return status;
}
// get files from partition tables
std::vector<meta::TableSchema> partiton_array;
status = meta_ptr_->ShowPartitions(table_id, partiton_array);
for (auto& schema : partiton_array) {
status = GetFilesToSearch(schema.table_id_, ids, files_array);
}
int64_t size = 0;
int64_t cache_total = cache::CpuCacheMgr::GetInstance()->CacheCapacity();
int64_t cache_usage = cache::CpuCacheMgr::GetInstance()->CacheUsage();
int64_t available_size = cache_total - cache_usage;
for (auto& day_files : files) {
for (auto& file : day_files.second) {
ExecutionEnginePtr engine =
EngineFactory::Build(file.dimension_, file.location_, (EngineType)file.engine_type_,
(MetricType)file.metric_type_, file.nlist_);
if (engine == nullptr) {
ENGINE_LOG_ERROR << "Invalid engine type";
return Status(DB_ERROR, "Invalid engine type");
}
for (auto& file : files_array) {
ExecutionEnginePtr engine = EngineFactory::Build(file.dimension_, file.location_, (EngineType)file.engine_type_,
(MetricType)file.metric_type_, file.nlist_);
if (engine == nullptr) {
ENGINE_LOG_ERROR << "Invalid engine type";
return Status(DB_ERROR, "Invalid engine type");
}
size += engine->PhysicalSize();
if (size > available_size) {
return Status(SERVER_CACHE_FULL, "Cache is full");
} else {
try {
// step 1: load index
engine->Load(true);
} catch (std::exception& ex) {
std::string msg = "Pre-load table encounter exception: " + std::string(ex.what());
ENGINE_LOG_ERROR << msg;
return Status(DB_ERROR, msg);
}
size += engine->PhysicalSize();
if (size > available_size) {
return Status(SERVER_CACHE_FULL, "Cache is full");
} else {
try {
// step 1: load index
engine->Load(true);
} catch (std::exception& ex) {
std::string msg = "Pre-load table encounter exception: " + std::string(ex.what());
ENGINE_LOG_ERROR << msg;
return Status(DB_ERROR, msg);
}
}
}
return Status::OK();
}
Status
DBImpl::UpdateTableFlag(const std::string& table_id, int64_t flag) {
if (shutting_down_.load(std::memory_order_acquire)) {
return Status(DB_ERROR, "Milsvus server is shutdown!");
return SHUTDOWN_ERROR;
}
return meta_ptr_->UpdateTableFlag(table_id, flag);
@ -235,34 +236,96 @@ DBImpl::UpdateTableFlag(const std::string& table_id, int64_t flag) {
Status
DBImpl::GetTableRowCount(const std::string& table_id, uint64_t& row_count) {
if (shutting_down_.load(std::memory_order_acquire)) {
return Status(DB_ERROR, "Milsvus server is shutdown!");
return SHUTDOWN_ERROR;
}
return meta_ptr_->Count(table_id, row_count);
return GetTableRowCountRecursively(table_id, row_count);
}
Status
DBImpl::InsertVectors(const std::string& table_id, uint64_t n, const float* vectors, IDNumbers& vector_ids) {
// ENGINE_LOG_DEBUG << "Insert " << n << " vectors to cache";
DBImpl::CreatePartition(const std::string& table_id, const std::string& partition_name,
const std::string& partition_tag) {
if (shutting_down_.load(std::memory_order_acquire)) {
return Status(DB_ERROR, "Milsvus server is shutdown!");
return SHUTDOWN_ERROR;
}
return meta_ptr_->CreatePartition(table_id, partition_name, partition_tag);
}
Status
DBImpl::DropPartition(const std::string& partition_name) {
if (shutting_down_.load(std::memory_order_acquire)) {
return SHUTDOWN_ERROR;
}
auto status = mem_mgr_->EraseMemVector(partition_name); // not allow insert
status = meta_ptr_->DropPartition(partition_name); // soft delete table
// scheduler will determine when to delete table files
auto nres = scheduler::ResMgrInst::GetInstance()->GetNumOfComputeResource();
scheduler::DeleteJobPtr job = std::make_shared<scheduler::DeleteJob>(partition_name, meta_ptr_, nres);
scheduler::JobMgrInst::GetInstance()->Put(job);
job->WaitAndDelete();
return Status::OK();
}
Status
DBImpl::DropPartitionByTag(const std::string& table_id, const std::string& partition_tag) {
if (shutting_down_.load(std::memory_order_acquire)) {
return SHUTDOWN_ERROR;
}
std::string partition_name;
auto status = meta_ptr_->GetPartitionName(table_id, partition_tag, partition_name);
return DropPartition(partition_name);
}
Status
DBImpl::ShowPartitions(const std::string& table_id, std::vector<meta::TableSchema>& partiton_schema_array) {
if (shutting_down_.load(std::memory_order_acquire)) {
return SHUTDOWN_ERROR;
}
return meta_ptr_->ShowPartitions(table_id, partiton_schema_array);
}
Status
DBImpl::InsertVectors(const std::string& table_id, const std::string& partition_tag, uint64_t n, const float* vectors,
IDNumbers& vector_ids) {
// ENGINE_LOG_DEBUG << "Insert " << n << " vectors to cache";
if (shutting_down_.load(std::memory_order_acquire)) {
return SHUTDOWN_ERROR;
}
// if partition is specified, use partition as target table
Status status;
std::string target_table_name = table_id;
if (!partition_tag.empty()) {
std::string partition_name;
status = meta_ptr_->GetPartitionName(table_id, partition_tag, target_table_name);
}
// insert vectors into target table
milvus::server::CollectInsertMetrics metrics(n, status);
status = mem_mgr_->InsertVectors(table_id, n, vectors, vector_ids);
status = mem_mgr_->InsertVectors(target_table_name, n, vectors, vector_ids);
return status;
}
Status
DBImpl::CreateIndex(const std::string& table_id, const TableIndex& index) {
if (shutting_down_.load(std::memory_order_acquire)) {
return SHUTDOWN_ERROR;
}
Status status;
{
std::unique_lock<std::mutex> lock(build_index_mutex_);
// step 1: check index difference
TableIndex old_index;
auto status = DescribeIndex(table_id, old_index);
status = DescribeIndex(table_id, old_index);
if (!status.ok()) {
ENGINE_LOG_ERROR << "Failed to get table index info for table: " << table_id;
return status;
@ -272,11 +335,8 @@ DBImpl::CreateIndex(const std::string& table_id, const TableIndex& index) {
TableIndex new_index = index;
new_index.metric_type_ = old_index.metric_type_; // dont change metric type, it was defined by CreateTable
if (!utils::IsSameIndex(old_index, new_index)) {
DropIndex(table_id);
status = meta_ptr_->UpdateTableIndex(table_id, new_index);
status = UpdateTableIndexRecursively(table_id, new_index);
if (!status.ok()) {
ENGINE_LOG_ERROR << "Failed to update table index info for table: " << table_id;
return status;
}
}
@ -287,101 +347,91 @@ DBImpl::CreateIndex(const std::string& table_id, const TableIndex& index) {
WaitMergeFileFinish();
// step 4: wait and build index
// for IDMAP type, only wait all NEW file converted to RAW file
// for other type, wait NEW/RAW/NEW_MERGE/NEW_INDEX/TO_INDEX files converted to INDEX files
std::vector<int> file_types;
if (index.engine_type_ == static_cast<int32_t>(EngineType::FAISS_IDMAP)) {
file_types = {
static_cast<int32_t>(meta::TableFileSchema::NEW),
static_cast<int32_t>(meta::TableFileSchema::NEW_MERGE),
};
} else {
file_types = {
static_cast<int32_t>(meta::TableFileSchema::RAW),
static_cast<int32_t>(meta::TableFileSchema::NEW),
static_cast<int32_t>(meta::TableFileSchema::NEW_MERGE),
static_cast<int32_t>(meta::TableFileSchema::NEW_INDEX),
static_cast<int32_t>(meta::TableFileSchema::TO_INDEX),
};
}
status = BuildTableIndexRecursively(table_id, index);
std::vector<std::string> file_ids;
auto status = meta_ptr_->FilesByType(table_id, file_types, file_ids);
int times = 1;
while (!file_ids.empty()) {
ENGINE_LOG_DEBUG << "Non index files detected! Will build index " << times;
if (index.engine_type_ != (int)EngineType::FAISS_IDMAP) {
status = meta_ptr_->UpdateTableFilesToIndex(table_id);
}
std::this_thread::sleep_for(std::chrono::milliseconds(std::min(10 * 1000, times * 100)));
status = meta_ptr_->FilesByType(table_id, file_types, file_ids);
times++;
}
return Status::OK();
return status;
}
Status
DBImpl::DescribeIndex(const std::string& table_id, TableIndex& index) {
if (shutting_down_.load(std::memory_order_acquire)) {
return SHUTDOWN_ERROR;
}
return meta_ptr_->DescribeTableIndex(table_id, index);
}
Status
DBImpl::DropIndex(const std::string& table_id) {
if (shutting_down_.load(std::memory_order_acquire)) {
return SHUTDOWN_ERROR;
}
ENGINE_LOG_DEBUG << "Drop index for table: " << table_id;
return meta_ptr_->DropTableIndex(table_id);
return DropTableIndexRecursively(table_id);
}
Status
DBImpl::Query(const std::string& table_id, uint64_t k, uint64_t nq, uint64_t nprobe, const float* vectors,
QueryResults& results) {
DBImpl::Query(const std::string& table_id, const std::vector<std::string>& partition_tags, uint64_t k, uint64_t nq,
uint64_t nprobe, const float* vectors, ResultIds& result_ids, ResultDistances& result_distances) {
if (shutting_down_.load(std::memory_order_acquire)) {
return Status(DB_ERROR, "Milsvus server is shutdown!");
return SHUTDOWN_ERROR;
}
meta::DatesT dates = {utils::GetDate()};
Status result = Query(table_id, k, nq, nprobe, vectors, dates, results);
Status result = Query(table_id, partition_tags, k, nq, nprobe, vectors, dates, result_ids, result_distances);
return result;
}
Status
DBImpl::Query(const std::string& table_id, uint64_t k, uint64_t nq, uint64_t nprobe, const float* vectors,
const meta::DatesT& dates, QueryResults& results) {
DBImpl::Query(const std::string& table_id, const std::vector<std::string>& partition_tags, uint64_t k, uint64_t nq,
uint64_t nprobe, const float* vectors, const meta::DatesT& dates, ResultIds& result_ids,
ResultDistances& result_distances) {
if (shutting_down_.load(std::memory_order_acquire)) {
return Status(DB_ERROR, "Milsvus server is shutdown!");
return SHUTDOWN_ERROR;
}
ENGINE_LOG_DEBUG << "Query by dates for table: " << table_id << " date range count: " << dates.size();
// get all table files from table
meta::DatePartionedTableFilesSchema files;
Status status;
std::vector<size_t> ids;
auto status = meta_ptr_->FilesToSearch(table_id, ids, dates, files);
if (!status.ok()) {
return status;
}
meta::TableFilesSchema files_array;
meta::TableFilesSchema file_id_array;
for (auto& day_files : files) {
for (auto& file : day_files.second) {
file_id_array.push_back(file);
if (partition_tags.empty()) {
// no partition tag specified, means search in whole table
// get all table files from parent table
status = GetFilesToSearch(table_id, ids, files_array);
if (!status.ok()) {
return status;
}
std::vector<meta::TableSchema> partiton_array;
status = meta_ptr_->ShowPartitions(table_id, partiton_array);
for (auto& schema : partiton_array) {
status = GetFilesToSearch(schema.table_id_, ids, files_array);
}
} else {
// get files from specified partitions
std::set<std::string> partition_name_array;
GetPartitionsByTags(table_id, partition_tags, partition_name_array);
for (auto& partition_name : partition_name_array) {
status = GetFilesToSearch(partition_name, ids, files_array);
}
}
cache::CpuCacheMgr::GetInstance()->PrintInfo(); // print cache info before query
status = QueryAsync(table_id, file_id_array, k, nq, nprobe, vectors, results);
status = QueryAsync(table_id, files_array, k, nq, nprobe, vectors, result_ids, result_distances);
cache::CpuCacheMgr::GetInstance()->PrintInfo(); // print cache info after query
return status;
}
Status
DBImpl::Query(const std::string& table_id, const std::vector<std::string>& file_ids, uint64_t k, uint64_t nq,
uint64_t nprobe, const float* vectors, const meta::DatesT& dates, QueryResults& results) {
DBImpl::QueryByFileID(const std::string& table_id, const std::vector<std::string>& file_ids, uint64_t k, uint64_t nq,
uint64_t nprobe, const float* vectors, const meta::DatesT& dates, ResultIds& result_ids,
ResultDistances& result_distances) {
if (shutting_down_.load(std::memory_order_acquire)) {
return Status(DB_ERROR, "Milsvus server is shutdown!");
return SHUTDOWN_ERROR;
}
ENGINE_LOG_DEBUG << "Query by file ids for table: " << table_id << " date range count: " << dates.size();
@ -395,25 +445,18 @@ DBImpl::Query(const std::string& table_id, const std::vector<std::string>& file_
ids.push_back(std::stoul(id, &sz));
}
meta::DatePartionedTableFilesSchema files_array;
auto status = meta_ptr_->FilesToSearch(table_id, ids, dates, files_array);
meta::TableFilesSchema files_array;
auto status = GetFilesToSearch(table_id, ids, files_array);
if (!status.ok()) {
return status;
}
meta::TableFilesSchema file_id_array;
for (auto& day_files : files_array) {
for (auto& file : day_files.second) {
file_id_array.push_back(file);
}
}
if (file_id_array.empty()) {
if (files_array.empty()) {
return Status(DB_ERROR, "Invalid file id");
}
cache::CpuCacheMgr::GetInstance()->PrintInfo(); // print cache info before query
status = QueryAsync(table_id, file_id_array, k, nq, nprobe, vectors, results);
status = QueryAsync(table_id, files_array, k, nq, nprobe, vectors, result_ids, result_distances);
cache::CpuCacheMgr::GetInstance()->PrintInfo(); // print cache info after query
return status;
}
@ -421,7 +464,7 @@ DBImpl::Query(const std::string& table_id, const std::vector<std::string>& file_
Status
DBImpl::Size(uint64_t& result) {
if (shutting_down_.load(std::memory_order_acquire)) {
return Status(DB_ERROR, "Milsvus server is shutdown!");
return SHUTDOWN_ERROR;
}
return meta_ptr_->Size(result);
@ -432,7 +475,7 @@ DBImpl::Size(uint64_t& result) {
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
Status
DBImpl::QueryAsync(const std::string& table_id, const meta::TableFilesSchema& files, uint64_t k, uint64_t nq,
uint64_t nprobe, const float* vectors, QueryResults& results) {
uint64_t nprobe, const float* vectors, ResultIds& result_ids, ResultDistances& result_distances) {
server::CollectQueryMetrics metrics(nq);
TimeRecorder rc("");
@ -453,7 +496,8 @@ DBImpl::QueryAsync(const std::string& table_id, const meta::TableFilesSchema& fi
}
// step 3: construct results
results = job->GetResult();
result_ids = job->GetResultIds();
result_distances = job->GetResultDistances();
rc.ElapseFromBegin("Engine query totally cost");
return Status::OK();
@ -772,5 +816,183 @@ DBImpl::BackgroundBuildIndex() {
ENGINE_LOG_TRACE << "Background build index thread exit";
}
Status
DBImpl::GetFilesToSearch(const std::string& table_id, const std::vector<size_t>& file_ids,
meta::TableFilesSchema& files) {
meta::DatesT dates;
meta::DatePartionedTableFilesSchema date_files;
auto status = meta_ptr_->FilesToSearch(table_id, file_ids, dates, date_files);
if (!status.ok()) {
return status;
}
TraverseFiles(date_files, files);
return Status::OK();
}
Status
DBImpl::GetPartitionsByTags(const std::string& table_id, const std::vector<std::string>& partition_tags,
std::set<std::string>& partition_name_array) {
std::vector<meta::TableSchema> partiton_array;
auto status = meta_ptr_->ShowPartitions(table_id, partiton_array);
for (auto& tag : partition_tags) {
for (auto& schema : partiton_array) {
if (server::StringHelpFunctions::IsRegexMatch(schema.partition_tag_, tag)) {
partition_name_array.insert(schema.table_id_);
}
}
}
return Status::OK();
}
Status
DBImpl::DropTableRecursively(const std::string& table_id, const meta::DatesT& dates) {
// dates partly delete files of the table but currently we don't support
ENGINE_LOG_DEBUG << "Prepare to delete table " << table_id;
Status status;
if (dates.empty()) {
status = mem_mgr_->EraseMemVector(table_id); // not allow insert
status = meta_ptr_->DropTable(table_id); // soft delete table
// scheduler will determine when to delete table files
auto nres = scheduler::ResMgrInst::GetInstance()->GetNumOfComputeResource();
scheduler::DeleteJobPtr job = std::make_shared<scheduler::DeleteJob>(table_id, meta_ptr_, nres);
scheduler::JobMgrInst::GetInstance()->Put(job);
job->WaitAndDelete();
} else {
status = meta_ptr_->DropDataByDate(table_id, dates);
}
std::vector<meta::TableSchema> partiton_array;
status = meta_ptr_->ShowPartitions(table_id, partiton_array);
for (auto& schema : partiton_array) {
status = DropTableRecursively(schema.table_id_, dates);
if (!status.ok()) {
return status;
}
}
return Status::OK();
}
Status
DBImpl::UpdateTableIndexRecursively(const std::string& table_id, const TableIndex& index) {
DropIndex(table_id);
auto status = meta_ptr_->UpdateTableIndex(table_id, index);
if (!status.ok()) {
ENGINE_LOG_ERROR << "Failed to update table index info for table: " << table_id;
return status;
}
std::vector<meta::TableSchema> partiton_array;
status = meta_ptr_->ShowPartitions(table_id, partiton_array);
for (auto& schema : partiton_array) {
status = UpdateTableIndexRecursively(schema.table_id_, index);
if (!status.ok()) {
return status;
}
}
return Status::OK();
}
Status
DBImpl::BuildTableIndexRecursively(const std::string& table_id, const TableIndex& index) {
// for IDMAP type, only wait all NEW file converted to RAW file
// for other type, wait NEW/RAW/NEW_MERGE/NEW_INDEX/TO_INDEX files converted to INDEX files
std::vector<int> file_types;
if (index.engine_type_ == static_cast<int32_t>(EngineType::FAISS_IDMAP)) {
file_types = {
static_cast<int32_t>(meta::TableFileSchema::NEW),
static_cast<int32_t>(meta::TableFileSchema::NEW_MERGE),
};
} else {
file_types = {
static_cast<int32_t>(meta::TableFileSchema::RAW),
static_cast<int32_t>(meta::TableFileSchema::NEW),
static_cast<int32_t>(meta::TableFileSchema::NEW_MERGE),
static_cast<int32_t>(meta::TableFileSchema::NEW_INDEX),
static_cast<int32_t>(meta::TableFileSchema::TO_INDEX),
};
}
// get files to build index
std::vector<std::string> file_ids;
auto status = meta_ptr_->FilesByType(table_id, file_types, file_ids);
int times = 1;
while (!file_ids.empty()) {
ENGINE_LOG_DEBUG << "Non index files detected! Will build index " << times;
if (index.engine_type_ != (int)EngineType::FAISS_IDMAP) {
status = meta_ptr_->UpdateTableFilesToIndex(table_id);
}
std::this_thread::sleep_for(std::chrono::milliseconds(std::min(10 * 1000, times * 100)));
status = meta_ptr_->FilesByType(table_id, file_types, file_ids);
times++;
}
// build index for partition
std::vector<meta::TableSchema> partiton_array;
status = meta_ptr_->ShowPartitions(table_id, partiton_array);
for (auto& schema : partiton_array) {
status = BuildTableIndexRecursively(schema.table_id_, index);
if (!status.ok()) {
return status;
}
}
return Status::OK();
}
Status
DBImpl::DropTableIndexRecursively(const std::string& table_id) {
ENGINE_LOG_DEBUG << "Drop index for table: " << table_id;
auto status = meta_ptr_->DropTableIndex(table_id);
if (!status.ok()) {
return status;
}
// drop partition index
std::vector<meta::TableSchema> partiton_array;
status = meta_ptr_->ShowPartitions(table_id, partiton_array);
for (auto& schema : partiton_array) {
status = DropTableIndexRecursively(schema.table_id_);
if (!status.ok()) {
return status;
}
}
return Status::OK();
}
Status
DBImpl::GetTableRowCountRecursively(const std::string& table_id, uint64_t& row_count) {
row_count = 0;
auto status = meta_ptr_->Count(table_id, row_count);
if (!status.ok()) {
return status;
}
// get partition row count
std::vector<meta::TableSchema> partiton_array;
status = meta_ptr_->ShowPartitions(table_id, partiton_array);
for (auto& schema : partiton_array) {
uint64_t partition_row_count = 0;
status = GetTableRowCountRecursively(schema.table_id_, partition_row_count);
if (!status.ok()) {
return status;
}
row_count += partition_row_count;
}
return Status::OK();
}
} // namespace engine
} // namespace milvus

56
core/src/db/DBImpl.h
View File

@ -57,7 +57,7 @@ class DBImpl : public DB {
CreateTable(meta::TableSchema& table_schema) override;
Status
DeleteTable(const std::string& table_id, const meta::DatesT& dates) override;
DropTable(const std::string& table_id, const meta::DatesT& dates) override;
Status
DescribeTable(meta::TableSchema& table_schema) override;
@ -78,7 +78,21 @@ class DBImpl : public DB {
GetTableRowCount(const std::string& table_id, uint64_t& row_count) override;
Status
InsertVectors(const std::string& table_id, uint64_t n, const float* vectors, IDNumbers& vector_ids) override;
CreatePartition(const std::string& table_id, const std::string& partition_name,
const std::string& partition_tag) override;
Status
DropPartition(const std::string& partition_name) override;
Status
DropPartitionByTag(const std::string& table_id, const std::string& partition_tag) override;
Status
ShowPartitions(const std::string& table_id, std::vector<meta::TableSchema>& partiton_schema_array) override;
Status
InsertVectors(const std::string& table_id, const std::string& partition_tag, uint64_t n, const float* vectors,
IDNumbers& vector_ids) override;
Status
CreateIndex(const std::string& table_id, const TableIndex& index) override;
@ -90,16 +104,18 @@ class DBImpl : public DB {
DropIndex(const std::string& table_id) override;
Status
Query(const std::string& table_id, uint64_t k, uint64_t nq, uint64_t nprobe, const float* vectors,
QueryResults& results) override;
Query(const std::string& table_id, const std::vector<std::string>& partition_tags, uint64_t k, uint64_t nq,
uint64_t nprobe, const float* vectors, ResultIds& result_ids, ResultDistances& result_distances) override;
Status
Query(const std::string& table_id, uint64_t k, uint64_t nq, uint64_t nprobe, const float* vectors,
const meta::DatesT& dates, QueryResults& results) override;
Query(const std::string& table_id, const std::vector<std::string>& partition_tags, uint64_t k, uint64_t nq,
uint64_t nprobe, const float* vectors, const meta::DatesT& dates, ResultIds& result_ids,
ResultDistances& result_distances) override;
Status
Query(const std::string& table_id, const std::vector<std::string>& file_ids, uint64_t k, uint64_t nq,
uint64_t nprobe, const float* vectors, const meta::DatesT& dates, QueryResults& results) override;
QueryByFileID(const std::string& table_id, const std::vector<std::string>& file_ids, uint64_t k, uint64_t nq,
uint64_t nprobe, const float* vectors, const meta::DatesT& dates, ResultIds& result_ids,
ResultDistances& result_distances) override;
Status
Size(uint64_t& result) override;
@ -107,7 +123,7 @@ class DBImpl : public DB {
private:
Status
QueryAsync(const std::string& table_id, const meta::TableFilesSchema& files, uint64_t k, uint64_t nq,
uint64_t nprobe, const float* vectors, QueryResults& results);
uint64_t nprobe, const float* vectors, ResultIds& result_ids, ResultDistances& result_distances);
void
BackgroundTimerTask();
@ -136,6 +152,28 @@ class DBImpl : public DB {
Status
MemSerialize();
Status
GetFilesToSearch(const std::string& table_id, const std::vector<size_t>& file_ids, meta::TableFilesSchema& files);
Status
GetPartitionsByTags(const std::string& table_id, const std::vector<std::string>& partition_tags,
std::set<std::string>& partition_name_array);
Status
DropTableRecursively(const std::string& table_id, const meta::DatesT& dates);
Status
UpdateTableIndexRecursively(const std::string& table_id, const TableIndex& index);
Status
BuildTableIndexRecursively(const std::string& table_id, const TableIndex& index);
Status
DropTableIndexRecursively(const std::string& table_id);
Status
GetTableRowCountRecursively(const std::string& table_id, uint64_t& row_count);
private:
const DBOptions options_;

5
core/src/db/Types.h
View File

@ -19,6 +19,7 @@
#include "db/engine/ExecutionEngine.h"
#include <faiss/Index.h>
#include <stdint.h>
#include <utility>
#include <vector>
@ -30,8 +31,8 @@ typedef int64_t IDNumber;
typedef IDNumber* IDNumberPtr;
typedef std::vector<IDNumber> IDNumbers;
typedef std::vector<std::pair<IDNumber, double>> QueryResult;
typedef std::vector<QueryResult> QueryResults;
typedef std::vector<faiss::Index::idx_t> ResultIds;
typedef std::vector<faiss::Index::distance_t> ResultDistances;
struct TableIndex {
int32_t engine_type_ = (int)EngineType::FAISS_IDMAP;

4
core/src/db/engine/ExecutionEngineImpl.cpp
View File

@ -117,7 +117,11 @@ ExecutionEngineImpl::CreatetVecIndex(EngineType type) {
break;
}
case EngineType::FAISS_PQ: {
#ifdef MILVUS_CPU_VERSION
index = GetVecIndexFactory(IndexType::FAISS_IVFPQ_CPU);
#else
index = GetVecIndexFactory(IndexType::FAISS_IVFPQ_MIX);
#endif
break;
}
default: {

44
core/src/db/meta/Meta.h
View File

@ -50,14 +50,11 @@ class Meta {
virtual Status
AllTables(std::vector<TableSchema>& table_schema_array) = 0;
virtual Status
UpdateTableIndex(const std::string& table_id, const TableIndex& index) = 0;
virtual Status
UpdateTableFlag(const std::string& table_id, int64_t flag) = 0;
virtual Status
DeleteTable(const std::string& table_id) = 0;
DropTable(const std::string& table_id) = 0;
virtual Status
DeleteTableFiles(const std::string& table_id) = 0;
@ -66,20 +63,41 @@ class Meta {
CreateTableFile(TableFileSchema& file_schema) = 0;
virtual Status
DropPartitionsByDates(const std::string& table_id, const DatesT& dates) = 0;
DropDataByDate(const std::string& table_id, const DatesT& dates) = 0;
virtual Status
GetTableFiles(const std::string& table_id, const std::vector<size_t>& ids, TableFilesSchema& table_files) = 0;
virtual Status
UpdateTableFilesToIndex(const std::string& table_id) = 0;
virtual Status
UpdateTableFile(TableFileSchema& file_schema) = 0;
virtual Status
UpdateTableFiles(TableFilesSchema& files) = 0;
virtual Status
UpdateTableIndex(const std::string& table_id, const TableIndex& index) = 0;
virtual Status
UpdateTableFilesToIndex(const std::string& table_id) = 0;
virtual Status
DescribeTableIndex(const std::string& table_id, TableIndex& index) = 0;
virtual Status
DropTableIndex(const std::string& table_id) = 0;
virtual Status
CreatePartition(const std::string& table_name, const std::string& partition_name, const std::string& tag) = 0;
virtual Status
DropPartition(const std::string& partition_name) = 0;
virtual Status
ShowPartitions(const std::string& table_name, std::vector<meta::TableSchema>& partiton_schema_array) = 0;
virtual Status
GetPartitionName(const std::string& table_name, const std::string& tag, std::string& partition_name) = 0;
virtual Status
FilesToSearch(const std::string& table_id, const std::vector<size_t>& ids, const DatesT& dates,
DatePartionedTableFilesSchema& files) = 0;
@ -87,12 +105,6 @@ class Meta {
virtual Status
FilesToMerge(const std::string& table_id, DatePartionedTableFilesSchema& files) = 0;
virtual Status
Size(uint64_t& result) = 0;
virtual Status
Archive() = 0;
virtual Status
FilesToIndex(TableFilesSchema&) = 0;
@ -101,10 +113,10 @@ class Meta {
std::vector<std::string>& file_ids) = 0;
virtual Status
DescribeTableIndex(const std::string& table_id, TableIndex& index) = 0;
Size(uint64_t& result) = 0;
virtual Status
DropTableIndex(const std::string& table_id) = 0;
Archive() = 0;
virtual Status
CleanUp() = 0;

5
core/src/db/meta/MetaTypes.h
View File

@ -19,6 +19,7 @@
#include "db/Constants.h"
#include "db/engine/ExecutionEngine.h"
#include "src/config.h"
#include <map>
#include <memory>
@ -33,6 +34,7 @@ constexpr int32_t DEFAULT_ENGINE_TYPE = (int)EngineType::FAISS_IDMAP;
constexpr int32_t DEFAULT_NLIST = 16384;
constexpr int32_t DEFAULT_METRIC_TYPE = (int)MetricType::L2;
constexpr int32_t DEFAULT_INDEX_FILE_SIZE = ONE_GB;
constexpr char CURRENT_VERSION[] = MILVUS_VERSION;
constexpr int64_t FLAG_MASK_NO_USERID = 0x1;
constexpr int64_t FLAG_MASK_HAS_USERID = 0x1 << 1;
@ -57,6 +59,9 @@ struct TableSchema {
int32_t engine_type_ = DEFAULT_ENGINE_TYPE;
int32_t nlist_ = DEFAULT_NLIST;
int32_t metric_type_ = DEFAULT_METRIC_TYPE;
std::string owner_table_;
std::string partition_tag_;
std::string version_ = CURRENT_VERSION;
}; // TableSchema
struct TableFileSchema {

2698
core/src/db/meta/MySQLMetaImpl.cpp
View File

File diff suppressed because it is too large Load Diff

36
core/src/db/meta/MySQLMetaImpl.h
View File

@ -49,7 +49,7 @@ class MySQLMetaImpl : public Meta {
AllTables(std::vector<TableSchema>& table_schema_array) override;
Status
DeleteTable(const std::string& table_id) override;
DropTable(const std::string& table_id) override;
Status
DeleteTableFiles(const std::string& table_id) override;
@ -58,27 +58,17 @@ class MySQLMetaImpl : public Meta {
CreateTableFile(TableFileSchema& file_schema) override;
Status
DropPartitionsByDates(const std::string& table_id, const DatesT& dates) override;
DropDataByDate(const std::string& table_id, const DatesT& dates) override;
Status
GetTableFiles(const std::string& table_id, const std::vector<size_t>& ids, TableFilesSchema& table_files) override;
Status
FilesByType(const std::string& table_id, const std::vector<int>& file_types,
std::vector<std::string>& file_ids) override;
Status
UpdateTableIndex(const std::string& table_id, const TableIndex& index) override;
Status
UpdateTableFlag(const std::string& table_id, int64_t flag) override;
Status
DescribeTableIndex(const std::string& table_id, TableIndex& index) override;
Status
DropTableIndex(const std::string& table_id) override;
Status
UpdateTableFile(TableFileSchema& file_schema) override;
@ -88,6 +78,24 @@ class MySQLMetaImpl : public Meta {
Status
UpdateTableFiles(TableFilesSchema& files) override;
Status
DescribeTableIndex(const std::string& table_id, TableIndex& index) override;
Status
DropTableIndex(const std::string& table_id) override;
Status
CreatePartition(const std::string& table_id, const std::string& partition_name, const std::string& tag) override;
Status
DropPartition(const std::string& partition_name) override;
Status
ShowPartitions(const std::string& table_id, std::vector<meta::TableSchema>& partiton_schema_array) override;
Status
GetPartitionName(const std::string& table_id, const std::string& tag, std::string& partition_name) override;
Status
FilesToSearch(const std::string& table_id, const std::vector<size_t>& ids, const DatesT& dates,
DatePartionedTableFilesSchema& files) override;
@ -98,6 +106,10 @@ class MySQLMetaImpl : public Meta {
Status
FilesToIndex(TableFilesSchema&) override;
Status
FilesByType(const std::string& table_id, const std::vector<int>& file_types,
std::vector<std::string>& file_ids) override;
Status
Archive() override;

1241
core/src/db/meta/SqliteMetaImpl.cpp
View File

File diff suppressed because it is too large Load Diff

32
core/src/db/meta/SqliteMetaImpl.h
View File

@ -49,7 +49,7 @@ class SqliteMetaImpl : public Meta {
AllTables(std::vector<TableSchema>& table_schema_array) override;
Status
DeleteTable(const std::string& table_id) override;
DropTable(const std::string& table_id) override;
Status
DeleteTableFiles(const std::string& table_id) override;
@ -58,21 +58,26 @@ class SqliteMetaImpl : public Meta {
CreateTableFile(TableFileSchema& file_schema) override;
Status
DropPartitionsByDates(const std::string& table_id, const DatesT& dates) override;
DropDataByDate(const std::string& table_id, const DatesT& dates) override;
Status
GetTableFiles(const std::string& table_id, const std::vector<size_t>& ids, TableFilesSchema& table_files) override;
Status
FilesByType(const std::string& table_id, const std::vector<int>& file_types,
std::vector<std::string>& file_ids) override;
Status
UpdateTableIndex(const std::string& table_id, const TableIndex& index) override;
Status
UpdateTableFlag(const std::string& table_id, int64_t flag) override;
Status
UpdateTableFile(TableFileSchema& file_schema) override;
Status
UpdateTableFilesToIndex(const std::string& table_id) override;
Status
UpdateTableFiles(TableFilesSchema& files) override;
Status
DescribeTableIndex(const std::string& table_id, TableIndex& index) override;
@ -80,13 +85,16 @@ class SqliteMetaImpl : public Meta {
DropTableIndex(const std::string& table_id) override;
Status
UpdateTableFilesToIndex(const std::string& table_id) override;
CreatePartition(const std::string& table_id, const std::string& partition_name, const std::string& tag) override;
Status
UpdateTableFile(TableFileSchema& file_schema) override;
DropPartition(const std::string& partition_name) override;
Status
UpdateTableFiles(TableFilesSchema& files) override;
ShowPartitions(const std::string& table_id, std::vector<meta::TableSchema>& partiton_schema_array) override;
Status
GetPartitionName(const std::string& table_id, const std::string& tag, std::string& partition_name) override;
Status
FilesToSearch(const std::string& table_id, const std::vector<size_t>& ids, const DatesT& dates,
@ -99,11 +107,15 @@ class SqliteMetaImpl : public Meta {
FilesToIndex(TableFilesSchema&) override;
Status
Archive() override;
FilesByType(const std::string& table_id, const std::vector<int>& file_types,
std::vector<std::string>& file_ids) override;
Status
Size(uint64_t& result) override;
Status
Archive() override;
Status
CleanUp() override;

730
core/src/grpc/gen-milvus/milvus.grpc.pb.cc
View File

@ -22,19 +22,22 @@ namespace grpc {
static const char* MilvusService_method_names[] = {
"/milvus.grpc.MilvusService/CreateTable",
"/milvus.grpc.MilvusService/HasTable",
"/milvus.grpc.MilvusService/DropTable",
"/milvus.grpc.MilvusService/CreateIndex",
"/milvus.grpc.MilvusService/Insert",
"/milvus.grpc.MilvusService/Search",
"/milvus.grpc.MilvusService/SearchInFiles",
"/milvus.grpc.MilvusService/DescribeTable",
"/milvus.grpc.MilvusService/CountTable",
"/milvus.grpc.MilvusService/ShowTables",
"/milvus.grpc.MilvusService/Cmd",
"/milvus.grpc.MilvusService/DeleteByRange",
"/milvus.grpc.MilvusService/PreloadTable",
"/milvus.grpc.MilvusService/DropTable",
"/milvus.grpc.MilvusService/CreateIndex",
"/milvus.grpc.MilvusService/DescribeIndex",
"/milvus.grpc.MilvusService/DropIndex",
"/milvus.grpc.MilvusService/CreatePartition",
"/milvus.grpc.MilvusService/ShowPartitions",
"/milvus.grpc.MilvusService/DropPartition",
"/milvus.grpc.MilvusService/Insert",
"/milvus.grpc.MilvusService/Search",
"/milvus.grpc.MilvusService/SearchInFiles",
"/milvus.grpc.MilvusService/Cmd",
"/milvus.grpc.MilvusService/DeleteByDate",
"/milvus.grpc.MilvusService/PreloadTable",
};
std::unique_ptr< MilvusService::Stub> MilvusService::NewStub(const std::shared_ptr< ::grpc::ChannelInterface>& channel, const ::grpc::StubOptions& options) {
@ -46,19 +49,22 @@ std::unique_ptr< MilvusService::Stub> MilvusService::NewStub(const std::shared_p
MilvusService::Stub::Stub(const std::shared_ptr< ::grpc::ChannelInterface>& channel)
: channel_(channel), rpcmethod_CreateTable_(MilvusService_method_names[0], ::grpc::internal::RpcMethod::NORMAL_RPC, channel)
, rpcmethod_HasTable_(MilvusService_method_names[1], ::grpc::internal::RpcMethod::NORMAL_RPC, channel)
, rpcmethod_DropTable_(MilvusService_method_names[2], ::grpc::internal::RpcMethod::NORMAL_RPC, channel)
, rpcmethod_CreateIndex_(MilvusService_method_names[3], ::grpc::internal::RpcMethod::NORMAL_RPC, channel)
, rpcmethod_Insert_(MilvusService_method_names[4], ::grpc::internal::RpcMethod::NORMAL_RPC, channel)
, rpcmethod_Search_(MilvusService_method_names[5], ::grpc::internal::RpcMethod::NORMAL_RPC, channel)
, rpcmethod_SearchInFiles_(MilvusService_method_names[6], ::grpc::internal::RpcMethod::NORMAL_RPC, channel)
, rpcmethod_DescribeTable_(MilvusService_method_names[7], ::grpc::internal::RpcMethod::NORMAL_RPC, channel)
, rpcmethod_CountTable_(MilvusService_method_names[8], ::grpc::internal::RpcMethod::NORMAL_RPC, channel)
, rpcmethod_ShowTables_(MilvusService_method_names[9], ::grpc::internal::RpcMethod::NORMAL_RPC, channel)
, rpcmethod_Cmd_(MilvusService_method_names[10], ::grpc::internal::RpcMethod::NORMAL_RPC, channel)
, rpcmethod_DeleteByRange_(MilvusService_method_names[11], ::grpc::internal::RpcMethod::NORMAL_RPC, channel)
, rpcmethod_PreloadTable_(MilvusService_method_names[12], ::grpc::internal::RpcMethod::NORMAL_RPC, channel)
, rpcmethod_DescribeIndex_(MilvusService_method_names[13], ::grpc::internal::RpcMethod::NORMAL_RPC, channel)
, rpcmethod_DropIndex_(MilvusService_method_names[14], ::grpc::internal::RpcMethod::NORMAL_RPC, channel)
, rpcmethod_DescribeTable_(MilvusService_method_names[2], ::grpc::internal::RpcMethod::NORMAL_RPC, channel)
, rpcmethod_CountTable_(MilvusService_method_names[3], ::grpc::internal::RpcMethod::NORMAL_RPC, channel)
, rpcmethod_ShowTables_(MilvusService_method_names[4], ::grpc::internal::RpcMethod::NORMAL_RPC, channel)
, rpcmethod_DropTable_(MilvusService_method_names[5], ::grpc::internal::RpcMethod::NORMAL_RPC, channel)
, rpcmethod_CreateIndex_(MilvusService_method_names[6], ::grpc::internal::RpcMethod::NORMAL_RPC, channel)
, rpcmethod_DescribeIndex_(MilvusService_method_names[7], ::grpc::internal::RpcMethod::NORMAL_RPC, channel)
, rpcmethod_DropIndex_(MilvusService_method_names[8], ::grpc::internal::RpcMethod::NORMAL_RPC, channel)
, rpcmethod_CreatePartition_(MilvusService_method_names[9], ::grpc::internal::RpcMethod::NORMAL_RPC, channel)
, rpcmethod_ShowPartitions_(MilvusService_method_names[10], ::grpc::internal::RpcMethod::NORMAL_RPC, channel)
, rpcmethod_DropPartition_(MilvusService_method_names[11], ::grpc::internal::RpcMethod::NORMAL_RPC, channel)
, rpcmethod_Insert_(MilvusService_method_names[12], ::grpc::internal::RpcMethod::NORMAL_RPC, channel)
, rpcmethod_Search_(MilvusService_method_names[13], ::grpc::internal::RpcMethod::NORMAL_RPC, channel)
, rpcmethod_SearchInFiles_(MilvusService_method_names[14], ::grpc::internal::RpcMethod::NORMAL_RPC, channel)
, rpcmethod_Cmd_(MilvusService_method_names[15], ::grpc::internal::RpcMethod::NORMAL_RPC, channel)
, rpcmethod_DeleteByDate_(MilvusService_method_names[16], ::grpc::internal::RpcMethod::NORMAL_RPC, channel)
, rpcmethod_PreloadTable_(MilvusService_method_names[17], ::grpc::internal::RpcMethod::NORMAL_RPC, channel)
{}
::grpc::Status MilvusService::Stub::CreateTable(::grpc::ClientContext* context, const ::milvus::grpc::TableSchema& request, ::milvus::grpc::Status* response) {
@ -117,146 +123,6 @@ void MilvusService::Stub::experimental_async::HasTable(::grpc::ClientContext* co
return ::grpc_impl::internal::ClientAsyncResponseReaderFactory< ::milvus::grpc::BoolReply>::Create(channel_.get(), cq, rpcmethod_HasTable_, context, request, false);
}
::grpc::Status MilvusService::Stub::DropTable(::grpc::ClientContext* context, const ::milvus::grpc::TableName& request, ::milvus::grpc::Status* response) {
return ::grpc::internal::BlockingUnaryCall(channel_.get(), rpcmethod_DropTable_, context, request, response);
}
void MilvusService::Stub::experimental_async::DropTable(::grpc::ClientContext* context, const ::milvus::grpc::TableName* request, ::milvus::grpc::Status* response, std::function<void(::grpc::Status)> f) {
::grpc_impl::internal::CallbackUnaryCall(stub_->channel_.get(), stub_->rpcmethod_DropTable_, context, request, response, std::move(f));
}
void MilvusService::Stub::experimental_async::DropTable(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::Status* response, std::function<void(::grpc::Status)> f) {
::grpc_impl::internal::CallbackUnaryCall(stub_->channel_.get(), stub_->rpcmethod_DropTable_, context, request, response, std::move(f));
}
void MilvusService::Stub::experimental_async::DropTable(::grpc::ClientContext* context, const ::milvus::grpc::TableName* request, ::milvus::grpc::Status* response, ::grpc::experimental::ClientUnaryReactor* reactor) {
::grpc_impl::internal::ClientCallbackUnaryFactory::Create(stub_->channel_.get(), stub_->rpcmethod_DropTable_, context, request, response, reactor);
}
void MilvusService::Stub::experimental_async::DropTable(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::Status* response, ::grpc::experimental::ClientUnaryReactor* reactor) {
::grpc_impl::internal::ClientCallbackUnaryFactory::Create(stub_->channel_.get(), stub_->rpcmethod_DropTable_, context, request, response, reactor);
}
::grpc::ClientAsyncResponseReader< ::milvus::grpc::Status>* MilvusService::Stub::AsyncDropTableRaw(::grpc::ClientContext* context, const ::milvus::grpc::TableName& request, ::grpc::CompletionQueue* cq) {
return ::grpc_impl::internal::ClientAsyncResponseReaderFactory< ::milvus::grpc::Status>::Create(channel_.get(), cq, rpcmethod_DropTable_, context, request, true);
}
::grpc::ClientAsyncResponseReader< ::milvus::grpc::Status>* MilvusService::Stub::PrepareAsyncDropTableRaw(::grpc::ClientContext* context, const ::milvus::grpc::TableName& request, ::grpc::CompletionQueue* cq) {
return ::grpc_impl::internal::ClientAsyncResponseReaderFactory< ::milvus::grpc::Status>::Create(channel_.get(), cq, rpcmethod_DropTable_, context, request, false);
}
::grpc::Status MilvusService::Stub::CreateIndex(::grpc::ClientContext* context, const ::milvus::grpc::IndexParam& request, ::milvus::grpc::Status* response) {
return ::grpc::internal::BlockingUnaryCall(channel_.get(), rpcmethod_CreateIndex_, context, request, response);
}
void MilvusService::Stub::experimental_async::CreateIndex(::grpc::ClientContext* context, const ::milvus::grpc::IndexParam* request, ::milvus::grpc::Status* response, std::function<void(::grpc::Status)> f) {
::grpc_impl::internal::CallbackUnaryCall(stub_->channel_.get(), stub_->rpcmethod_CreateIndex_, context, request, response, std::move(f));
}
void MilvusService::Stub::experimental_async::CreateIndex(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::Status* response, std::function<void(::grpc::Status)> f) {
::grpc_impl::internal::CallbackUnaryCall(stub_->channel_.get(), stub_->rpcmethod_CreateIndex_, context, request, response, std::move(f));
}
void MilvusService::Stub::experimental_async::CreateIndex(::grpc::ClientContext* context, const ::milvus::grpc::IndexParam* request, ::milvus::grpc::Status* response, ::grpc::experimental::ClientUnaryReactor* reactor) {
::grpc_impl::internal::ClientCallbackUnaryFactory::Create(stub_->channel_.get(), stub_->rpcmethod_CreateIndex_, context, request, response, reactor);
}
void MilvusService::Stub::experimental_async::CreateIndex(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::Status* response, ::grpc::experimental::ClientUnaryReactor* reactor) {
::grpc_impl::internal::ClientCallbackUnaryFactory::Create(stub_->channel_.get(), stub_->rpcmethod_CreateIndex_, context, request, response, reactor);
}
::grpc::ClientAsyncResponseReader< ::milvus::grpc::Status>* MilvusService::Stub::AsyncCreateIndexRaw(::grpc::ClientContext* context, const ::milvus::grpc::IndexParam& request, ::grpc::CompletionQueue* cq) {
return ::grpc_impl::internal::ClientAsyncResponseReaderFactory< ::milvus::grpc::Status>::Create(channel_.get(), cq, rpcmethod_CreateIndex_, context, request, true);
}
::grpc::ClientAsyncResponseReader< ::milvus::grpc::Status>* MilvusService::Stub::PrepareAsyncCreateIndexRaw(::grpc::ClientContext* context, const ::milvus::grpc::IndexParam& request, ::grpc::CompletionQueue* cq) {
return ::grpc_impl::internal::ClientAsyncResponseReaderFactory< ::milvus::grpc::Status>::Create(channel_.get(), cq, rpcmethod_CreateIndex_, context, request, false);
}
::grpc::Status MilvusService::Stub::Insert(::grpc::ClientContext* context, const ::milvus::grpc::InsertParam& request, ::milvus::grpc::VectorIds* response) {
return ::grpc::internal::BlockingUnaryCall(channel_.get(), rpcmethod_Insert_, context, request, response);
}
void MilvusService::Stub::experimental_async::Insert(::grpc::ClientContext* context, const ::milvus::grpc::InsertParam* request, ::milvus::grpc::VectorIds* response, std::function<void(::grpc::Status)> f) {
::grpc_impl::internal::CallbackUnaryCall(stub_->channel_.get(), stub_->rpcmethod_Insert_, context, request, response, std::move(f));
}
void MilvusService::Stub::experimental_async::Insert(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::VectorIds* response, std::function<void(::grpc::Status)> f) {
::grpc_impl::internal::CallbackUnaryCall(stub_->channel_.get(), stub_->rpcmethod_Insert_, context, request, response, std::move(f));
}
void MilvusService::Stub::experimental_async::Insert(::grpc::ClientContext* context, const ::milvus::grpc::InsertParam* request, ::milvus::grpc::VectorIds* response, ::grpc::experimental::ClientUnaryReactor* reactor) {
::grpc_impl::internal::ClientCallbackUnaryFactory::Create(stub_->channel_.get(), stub_->rpcmethod_Insert_, context, request, response, reactor);
}
void MilvusService::Stub::experimental_async::Insert(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::VectorIds* response, ::grpc::experimental::ClientUnaryReactor* reactor) {
::grpc_impl::internal::ClientCallbackUnaryFactory::Create(stub_->channel_.get(), stub_->rpcmethod_Insert_, context, request, response, reactor);
}
::grpc::ClientAsyncResponseReader< ::milvus::grpc::VectorIds>* MilvusService::Stub::AsyncInsertRaw(::grpc::ClientContext* context, const ::milvus::grpc::InsertParam& request, ::grpc::CompletionQueue* cq) {
return ::grpc_impl::internal::ClientAsyncResponseReaderFactory< ::milvus::grpc::VectorIds>::Create(channel_.get(), cq, rpcmethod_Insert_, context, request, true);
}
::grpc::ClientAsyncResponseReader< ::milvus::grpc::VectorIds>* MilvusService::Stub::PrepareAsyncInsertRaw(::grpc::ClientContext* context, const ::milvus::grpc::InsertParam& request, ::grpc::CompletionQueue* cq) {
return ::grpc_impl::internal::ClientAsyncResponseReaderFactory< ::milvus::grpc::VectorIds>::Create(channel_.get(), cq, rpcmethod_Insert_, context, request, false);
}
::grpc::Status MilvusService::Stub::Search(::grpc::ClientContext* context, const ::milvus::grpc::SearchParam& request, ::milvus::grpc::TopKQueryResultList* response) {
return ::grpc::internal::BlockingUnaryCall(channel_.get(), rpcmethod_Search_, context, request, response);
}
void MilvusService::Stub::experimental_async::Search(::grpc::ClientContext* context, const ::milvus::grpc::SearchParam* request, ::milvus::grpc::TopKQueryResultList* response, std::function<void(::grpc::Status)> f) {
::grpc_impl::internal::CallbackUnaryCall(stub_->channel_.get(), stub_->rpcmethod_Search_, context, request, response, std::move(f));
}
void MilvusService::Stub::experimental_async::Search(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::TopKQueryResultList* response, std::function<void(::grpc::Status)> f) {
::grpc_impl::internal::CallbackUnaryCall(stub_->channel_.get(), stub_->rpcmethod_Search_, context, request, response, std::move(f));
}
void MilvusService::Stub::experimental_async::Search(::grpc::ClientContext* context, const ::milvus::grpc::SearchParam* request, ::milvus::grpc::TopKQueryResultList* response, ::grpc::experimental::ClientUnaryReactor* reactor) {
::grpc_impl::internal::ClientCallbackUnaryFactory::Create(stub_->channel_.get(), stub_->rpcmethod_Search_, context, request, response, reactor);
}
void MilvusService::Stub::experimental_async::Search(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::TopKQueryResultList* response, ::grpc::experimental::ClientUnaryReactor* reactor) {
::grpc_impl::internal::ClientCallbackUnaryFactory::Create(stub_->channel_.get(), stub_->rpcmethod_Search_, context, request, response, reactor);
}
::grpc::ClientAsyncResponseReader< ::milvus::grpc::TopKQueryResultList>* MilvusService::Stub::AsyncSearchRaw(::grpc::ClientContext* context, const ::milvus::grpc::SearchParam& request, ::grpc::CompletionQueue* cq) {
return ::grpc_impl::internal::ClientAsyncResponseReaderFactory< ::milvus::grpc::TopKQueryResultList>::Create(channel_.get(), cq, rpcmethod_Search_, context, request, true);
}
::grpc::ClientAsyncResponseReader< ::milvus::grpc::TopKQueryResultList>* MilvusService::Stub::PrepareAsyncSearchRaw(::grpc::ClientContext* context, const ::milvus::grpc::SearchParam& request, ::grpc::CompletionQueue* cq) {
return ::grpc_impl::internal::ClientAsyncResponseReaderFactory< ::milvus::grpc::TopKQueryResultList>::Create(channel_.get(), cq, rpcmethod_Search_, context, request, false);
}
::grpc::Status MilvusService::Stub::SearchInFiles(::grpc::ClientContext* context, const ::milvus::grpc::SearchInFilesParam& request, ::milvus::grpc::TopKQueryResultList* response) {
return ::grpc::internal::BlockingUnaryCall(channel_.get(), rpcmethod_SearchInFiles_, context, request, response);
}
void MilvusService::Stub::experimental_async::SearchInFiles(::grpc::ClientContext* context, const ::milvus::grpc::SearchInFilesParam* request, ::milvus::grpc::TopKQueryResultList* response, std::function<void(::grpc::Status)> f) {
::grpc_impl::internal::CallbackUnaryCall(stub_->channel_.get(), stub_->rpcmethod_SearchInFiles_, context, request, response, std::move(f));
}
void MilvusService::Stub::experimental_async::SearchInFiles(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::TopKQueryResultList* response, std::function<void(::grpc::Status)> f) {
::grpc_impl::internal::CallbackUnaryCall(stub_->channel_.get(), stub_->rpcmethod_SearchInFiles_, context, request, response, std::move(f));
}
void MilvusService::Stub::experimental_async::SearchInFiles(::grpc::ClientContext* context, const ::milvus::grpc::SearchInFilesParam* request, ::milvus::grpc::TopKQueryResultList* response, ::grpc::experimental::ClientUnaryReactor* reactor) {
::grpc_impl::internal::ClientCallbackUnaryFactory::Create(stub_->channel_.get(), stub_->rpcmethod_SearchInFiles_, context, request, response, reactor);
}
void MilvusService::Stub::experimental_async::SearchInFiles(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::TopKQueryResultList* response, ::grpc::experimental::ClientUnaryReactor* reactor) {
::grpc_impl::internal::ClientCallbackUnaryFactory::Create(stub_->channel_.get(), stub_->rpcmethod_SearchInFiles_, context, request, response, reactor);
}
::grpc::ClientAsyncResponseReader< ::milvus::grpc::TopKQueryResultList>* MilvusService::Stub::AsyncSearchInFilesRaw(::grpc::ClientContext* context, const ::milvus::grpc::SearchInFilesParam& request, ::grpc::CompletionQueue* cq) {
return ::grpc_impl::internal::ClientAsyncResponseReaderFactory< ::milvus::grpc::TopKQueryResultList>::Create(channel_.get(), cq, rpcmethod_SearchInFiles_, context, request, true);
}
::grpc::ClientAsyncResponseReader< ::milvus::grpc::TopKQueryResultList>* MilvusService::Stub::PrepareAsyncSearchInFilesRaw(::grpc::ClientContext* context, const ::milvus::grpc::SearchInFilesParam& request, ::grpc::CompletionQueue* cq) {
return ::grpc_impl::internal::ClientAsyncResponseReaderFactory< ::milvus::grpc::TopKQueryResultList>::Create(channel_.get(), cq, rpcmethod_SearchInFiles_, context, request, false);
}
::grpc::Status MilvusService::Stub::DescribeTable(::grpc::ClientContext* context, const ::milvus::grpc::TableName& request, ::milvus::grpc::TableSchema* response) {
return ::grpc::internal::BlockingUnaryCall(channel_.get(), rpcmethod_DescribeTable_, context, request, response);
}
@ -341,88 +207,60 @@ void MilvusService::Stub::experimental_async::ShowTables(::grpc::ClientContext*
return ::grpc_impl::internal::ClientAsyncResponseReaderFactory< ::milvus::grpc::TableNameList>::Create(channel_.get(), cq, rpcmethod_ShowTables_, context, request, false);
}
::grpc::Status MilvusService::Stub::Cmd(::grpc::ClientContext* context, const ::milvus::grpc::Command& request, ::milvus::grpc::StringReply* response) {
return ::grpc::internal::BlockingUnaryCall(channel_.get(), rpcmethod_Cmd_, context, request, response);
::grpc::Status MilvusService::Stub::DropTable(::grpc::ClientContext* context, const ::milvus::grpc::TableName& request, ::milvus::grpc::Status* response) {
return ::grpc::internal::BlockingUnaryCall(channel_.get(), rpcmethod_DropTable_, context, request, response);
}
void MilvusService::Stub::experimental_async::Cmd(::grpc::ClientContext* context, const ::milvus::grpc::Command* request, ::milvus::grpc::StringReply* response, std::function<void(::grpc::Status)> f) {
::grpc_impl::internal::CallbackUnaryCall(stub_->channel_.get(), stub_->rpcmethod_Cmd_, context, request, response, std::move(f));
void MilvusService::Stub::experimental_async::DropTable(::grpc::ClientContext* context, const ::milvus::grpc::TableName* request, ::milvus::grpc::Status* response, std::function<void(::grpc::Status)> f) {
::grpc_impl::internal::CallbackUnaryCall(stub_->channel_.get(), stub_->rpcmethod_DropTable_, context, request, response, std::move(f));
}
void MilvusService::Stub::experimental_async::Cmd(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::StringReply* response, std::function<void(::grpc::Status)> f) {
::grpc_impl::internal::CallbackUnaryCall(stub_->channel_.get(), stub_->rpcmethod_Cmd_, context, request, response, std::move(f));
void MilvusService::Stub::experimental_async::DropTable(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::Status* response, std::function<void(::grpc::Status)> f) {
::grpc_impl::internal::CallbackUnaryCall(stub_->channel_.get(), stub_->rpcmethod_DropTable_, context, request, response, std::move(f));
}
void MilvusService::Stub::experimental_async::Cmd(::grpc::ClientContext* context, const ::milvus::grpc::Command* request, ::milvus::grpc::StringReply* response, ::grpc::experimental::ClientUnaryReactor* reactor) {
::grpc_impl::internal::ClientCallbackUnaryFactory::Create(stub_->channel_.get(), stub_->rpcmethod_Cmd_, context, request, response, reactor);
void MilvusService::Stub::experimental_async::DropTable(::grpc::ClientContext* context, const ::milvus::grpc::TableName* request, ::milvus::grpc::Status* response, ::grpc::experimental::ClientUnaryReactor* reactor) {
::grpc_impl::internal::ClientCallbackUnaryFactory::Create(stub_->channel_.get(), stub_->rpcmethod_DropTable_, context, request, response, reactor);
}
void MilvusService::Stub::experimental_async::Cmd(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::StringReply* response, ::grpc::experimental::ClientUnaryReactor* reactor) {
::grpc_impl::internal::ClientCallbackUnaryFactory::Create(stub_->channel_.get(), stub_->rpcmethod_Cmd_, context, request, response, reactor);
void MilvusService::Stub::experimental_async::DropTable(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::Status* response, ::grpc::experimental::ClientUnaryReactor* reactor) {
::grpc_impl::internal::ClientCallbackUnaryFactory::Create(stub_->channel_.get(), stub_->rpcmethod_DropTable_, context, request, response, reactor);
}
::grpc::ClientAsyncResponseReader< ::milvus::grpc::StringReply>* MilvusService::Stub::AsyncCmdRaw(::grpc::ClientContext* context, const ::milvus::grpc::Command& request, ::grpc::CompletionQueue* cq) {
return ::grpc_impl::internal::ClientAsyncResponseReaderFactory< ::milvus::grpc::StringReply>::Create(channel_.get(), cq, rpcmethod_Cmd_, context, request, true);
::grpc::ClientAsyncResponseReader< ::milvus::grpc::Status>* MilvusService::Stub::AsyncDropTableRaw(::grpc::ClientContext* context, const ::milvus::grpc::TableName& request, ::grpc::CompletionQueue* cq) {
return ::grpc_impl::internal::ClientAsyncResponseReaderFactory< ::milvus::grpc::Status>::Create(channel_.get(), cq, rpcmethod_DropTable_, context, request, true);
}
::grpc::ClientAsyncResponseReader< ::milvus::grpc::StringReply>* MilvusService::Stub::PrepareAsyncCmdRaw(::grpc::ClientContext* context, const ::milvus::grpc::Command& request, ::grpc::CompletionQueue* cq) {
return ::grpc_impl::internal::ClientAsyncResponseReaderFactory< ::milvus::grpc::StringReply>::Create(channel_.get(), cq, rpcmethod_Cmd_, context, request, false);
::grpc::ClientAsyncResponseReader< ::milvus::grpc::Status>* MilvusService::Stub::PrepareAsyncDropTableRaw(::grpc::ClientContext* context, const ::milvus::grpc::TableName& request, ::grpc::CompletionQueue* cq) {
return ::grpc_impl::internal::ClientAsyncResponseReaderFactory< ::milvus::grpc::Status>::Create(channel_.get(), cq, rpcmethod_DropTable_, context, request, false);
}
::grpc::Status MilvusService::Stub::DeleteByRange(::grpc::ClientContext* context, const ::milvus::grpc::DeleteByRangeParam& request, ::milvus::grpc::Status* response) {
return ::grpc::internal::BlockingUnaryCall(channel_.get(), rpcmethod_DeleteByRange_, context, request, response);
::grpc::Status MilvusService::Stub::CreateIndex(::grpc::ClientContext* context, const ::milvus::grpc::IndexParam& request, ::milvus::grpc::Status* response) {
return ::grpc::internal::BlockingUnaryCall(channel_.get(), rpcmethod_CreateIndex_, context, request, response);
}
void MilvusService::Stub::experimental_async::DeleteByRange(::grpc::ClientContext* context, const ::milvus::grpc::DeleteByRangeParam* request, ::milvus::grpc::Status* response, std::function<void(::grpc::Status)> f) {
::grpc_impl::internal::CallbackUnaryCall(stub_->channel_.get(), stub_->rpcmethod_DeleteByRange_, context, request, response, std::move(f));
void MilvusService::Stub::experimental_async::CreateIndex(::grpc::ClientContext* context, const ::milvus::grpc::IndexParam* request, ::milvus::grpc::Status* response, std::function<void(::grpc::Status)> f) {
::grpc_impl::internal::CallbackUnaryCall(stub_->channel_.get(), stub_->rpcmethod_CreateIndex_, context, request, response, std::move(f));
}
void MilvusService::Stub::experimental_async::DeleteByRange(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::Status* response, std::function<void(::grpc::Status)> f) {
::grpc_impl::internal::CallbackUnaryCall(stub_->channel_.get(), stub_->rpcmethod_DeleteByRange_, context, request, response, std::move(f));
void MilvusService::Stub::experimental_async::CreateIndex(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::Status* response, std::function<void(::grpc::Status)> f) {
::grpc_impl::internal::CallbackUnaryCall(stub_->channel_.get(), stub_->rpcmethod_CreateIndex_, context, request, response, std::move(f));
}
void MilvusService::Stub::experimental_async::DeleteByRange(::grpc::ClientContext* context, const ::milvus::grpc::DeleteByRangeParam* request, ::milvus::grpc::Status* response, ::grpc::experimental::ClientUnaryReactor* reactor) {
::grpc_impl::internal::ClientCallbackUnaryFactory::Create(stub_->channel_.get(), stub_->rpcmethod_DeleteByRange_, context, request, response, reactor);
void MilvusService::Stub::experimental_async::CreateIndex(::grpc::ClientContext* context, const ::milvus::grpc::IndexParam* request, ::milvus::grpc::Status* response, ::grpc::experimental::ClientUnaryReactor* reactor) {
::grpc_impl::internal::ClientCallbackUnaryFactory::Create(stub_->channel_.get(), stub_->rpcmethod_CreateIndex_, context, request, response, reactor);
}
void MilvusService::Stub::experimental_async::DeleteByRange(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::Status* response, ::grpc::experimental::ClientUnaryReactor* reactor) {
::grpc_impl::internal::ClientCallbackUnaryFactory::Create(stub_->channel_.get(), stub_->rpcmethod_DeleteByRange_, context, request, response, reactor);
void MilvusService::Stub::experimental_async::CreateIndex(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::Status* response, ::grpc::experimental::ClientUnaryReactor* reactor) {
::grpc_impl::internal::ClientCallbackUnaryFactory::Create(stub_->channel_.get(), stub_->rpcmethod_CreateIndex_, context, request, response, reactor);
}
::grpc::ClientAsyncResponseReader< ::milvus::grpc::Status>* MilvusService::Stub::AsyncDeleteByRangeRaw(::grpc::ClientContext* context, const ::milvus::grpc::DeleteByRangeParam& request, ::grpc::CompletionQueue* cq) {
return ::grpc_impl::internal::ClientAsyncResponseReaderFactory< ::milvus::grpc::Status>::Create(channel_.get(), cq, rpcmethod_DeleteByRange_, context, request, true);
::grpc::ClientAsyncResponseReader< ::milvus::grpc::Status>* MilvusService::Stub::AsyncCreateIndexRaw(::grpc::ClientContext* context, const ::milvus::grpc::IndexParam& request, ::grpc::CompletionQueue* cq) {
return ::grpc_impl::internal::ClientAsyncResponseReaderFactory< ::milvus::grpc::Status>::Create(channel_.get(), cq, rpcmethod_CreateIndex_, context, request, true);
}
::grpc::ClientAsyncResponseReader< ::milvus::grpc::Status>* MilvusService::Stub::PrepareAsyncDeleteByRangeRaw(::grpc::ClientContext* context, const ::milvus::grpc::DeleteByRangeParam& request, ::grpc::CompletionQueue* cq) {
return ::grpc_impl::internal::ClientAsyncResponseReaderFactory< ::milvus::grpc::Status>::Create(channel_.get(), cq, rpcmethod_DeleteByRange_, context, request, false);
}
::grpc::Status MilvusService::Stub::PreloadTable(::grpc::ClientContext* context, const ::milvus::grpc::TableName& request, ::milvus::grpc::Status* response) {
return ::grpc::internal::BlockingUnaryCall(channel_.get(), rpcmethod_PreloadTable_, context, request, response);
}
void MilvusService::Stub::experimental_async::PreloadTable(::grpc::ClientContext* context, const ::milvus::grpc::TableName* request, ::milvus::grpc::Status* response, std::function<void(::grpc::Status)> f) {
::grpc_impl::internal::CallbackUnaryCall(stub_->channel_.get(), stub_->rpcmethod_PreloadTable_, context, request, response, std::move(f));
}
void MilvusService::Stub::experimental_async::PreloadTable(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::Status* response, std::function<void(::grpc::Status)> f) {
::grpc_impl::internal::CallbackUnaryCall(stub_->channel_.get(), stub_->rpcmethod_PreloadTable_, context, request, response, std::move(f));
}
void MilvusService::Stub::experimental_async::PreloadTable(::grpc::ClientContext* context, const ::milvus::grpc::TableName* request, ::milvus::grpc::Status* response, ::grpc::experimental::ClientUnaryReactor* reactor) {
::grpc_impl::internal::ClientCallbackUnaryFactory::Create(stub_->channel_.get(), stub_->rpcmethod_PreloadTable_, context, request, response, reactor);
}
void MilvusService::Stub::experimental_async::PreloadTable(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::Status* response, ::grpc::experimental::ClientUnaryReactor* reactor) {
::grpc_impl::internal::ClientCallbackUnaryFactory::Create(stub_->channel_.get(), stub_->rpcmethod_PreloadTable_, context, request, response, reactor);
}
::grpc::ClientAsyncResponseReader< ::milvus::grpc::Status>* MilvusService::Stub::AsyncPreloadTableRaw(::grpc::ClientContext* context, const ::milvus::grpc::TableName& request, ::grpc::CompletionQueue* cq) {
return ::grpc_impl::internal::ClientAsyncResponseReaderFactory< ::milvus::grpc::Status>::Create(channel_.get(), cq, rpcmethod_PreloadTable_, context, request, true);
}
::grpc::ClientAsyncResponseReader< ::milvus::grpc::Status>* MilvusService::Stub::PrepareAsyncPreloadTableRaw(::grpc::ClientContext* context, const ::milvus::grpc::TableName& request, ::grpc::CompletionQueue* cq) {
return ::grpc_impl::internal::ClientAsyncResponseReaderFactory< ::milvus::grpc::Status>::Create(channel_.get(), cq, rpcmethod_PreloadTable_, context, request, false);
::grpc::ClientAsyncResponseReader< ::milvus::grpc::Status>* MilvusService::Stub::PrepareAsyncCreateIndexRaw(::grpc::ClientContext* context, const ::milvus::grpc::IndexParam& request, ::grpc::CompletionQueue* cq) {
return ::grpc_impl::internal::ClientAsyncResponseReaderFactory< ::milvus::grpc::Status>::Create(channel_.get(), cq, rpcmethod_CreateIndex_, context, request, false);
}
::grpc::Status MilvusService::Stub::DescribeIndex(::grpc::ClientContext* context, const ::milvus::grpc::TableName& request, ::milvus::grpc::IndexParam* response) {
@ -481,6 +319,258 @@ void MilvusService::Stub::experimental_async::DropIndex(::grpc::ClientContext* c
return ::grpc_impl::internal::ClientAsyncResponseReaderFactory< ::milvus::grpc::Status>::Create(channel_.get(), cq, rpcmethod_DropIndex_, context, request, false);
}
::grpc::Status MilvusService::Stub::CreatePartition(::grpc::ClientContext* context, const ::milvus::grpc::PartitionParam& request, ::milvus::grpc::Status* response) {
return ::grpc::internal::BlockingUnaryCall(channel_.get(), rpcmethod_CreatePartition_, context, request, response);
}
void MilvusService::Stub::experimental_async::CreatePartition(::grpc::ClientContext* context, const ::milvus::grpc::PartitionParam* request, ::milvus::grpc::Status* response, std::function<void(::grpc::Status)> f) {
::grpc_impl::internal::CallbackUnaryCall(stub_->channel_.get(), stub_->rpcmethod_CreatePartition_, context, request, response, std::move(f));
}
void MilvusService::Stub::experimental_async::CreatePartition(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::Status* response, std::function<void(::grpc::Status)> f) {
::grpc_impl::internal::CallbackUnaryCall(stub_->channel_.get(), stub_->rpcmethod_CreatePartition_, context, request, response, std::move(f));
}
void MilvusService::Stub::experimental_async::CreatePartition(::grpc::ClientContext* context, const ::milvus::grpc::PartitionParam* request, ::milvus::grpc::Status* response, ::grpc::experimental::ClientUnaryReactor* reactor) {
::grpc_impl::internal::ClientCallbackUnaryFactory::Create(stub_->channel_.get(), stub_->rpcmethod_CreatePartition_, context, request, response, reactor);
}
void MilvusService::Stub::experimental_async::CreatePartition(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::Status* response, ::grpc::experimental::ClientUnaryReactor* reactor) {
::grpc_impl::internal::ClientCallbackUnaryFactory::Create(stub_->channel_.get(), stub_->rpcmethod_CreatePartition_, context, request, response, reactor);
}
::grpc::ClientAsyncResponseReader< ::milvus::grpc::Status>* MilvusService::Stub::AsyncCreatePartitionRaw(::grpc::ClientContext* context, const ::milvus::grpc::PartitionParam& request, ::grpc::CompletionQueue* cq) {
return ::grpc_impl::internal::ClientAsyncResponseReaderFactory< ::milvus::grpc::Status>::Create(channel_.get(), cq, rpcmethod_CreatePartition_, context, request, true);
}
::grpc::ClientAsyncResponseReader< ::milvus::grpc::Status>* MilvusService::Stub::PrepareAsyncCreatePartitionRaw(::grpc::ClientContext* context, const ::milvus::grpc::PartitionParam& request, ::grpc::CompletionQueue* cq) {
return ::grpc_impl::internal::ClientAsyncResponseReaderFactory< ::milvus::grpc::Status>::Create(channel_.get(), cq, rpcmethod_CreatePartition_, context, request, false);
}
::grpc::Status MilvusService::Stub::ShowPartitions(::grpc::ClientContext* context, const ::milvus::grpc::TableName& request, ::milvus::grpc::PartitionList* response) {
return ::grpc::internal::BlockingUnaryCall(channel_.get(), rpcmethod_ShowPartitions_, context, request, response);
}
void MilvusService::Stub::experimental_async::ShowPartitions(::grpc::ClientContext* context, const ::milvus::grpc::TableName* request, ::milvus::grpc::PartitionList* response, std::function<void(::grpc::Status)> f) {
::grpc_impl::internal::CallbackUnaryCall(stub_->channel_.get(), stub_->rpcmethod_ShowPartitions_, context, request, response, std::move(f));
}
void MilvusService::Stub::experimental_async::ShowPartitions(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::PartitionList* response, std::function<void(::grpc::Status)> f) {
::grpc_impl::internal::CallbackUnaryCall(stub_->channel_.get(), stub_->rpcmethod_ShowPartitions_, context, request, response, std::move(f));
}
void MilvusService::Stub::experimental_async::ShowPartitions(::grpc::ClientContext* context, const ::milvus::grpc::TableName* request, ::milvus::grpc::PartitionList* response, ::grpc::experimental::ClientUnaryReactor* reactor) {
::grpc_impl::internal::ClientCallbackUnaryFactory::Create(stub_->channel_.get(), stub_->rpcmethod_ShowPartitions_, context, request, response, reactor);
}
void MilvusService::Stub::experimental_async::ShowPartitions(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::PartitionList* response, ::grpc::experimental::ClientUnaryReactor* reactor) {
::grpc_impl::internal::ClientCallbackUnaryFactory::Create(stub_->channel_.get(), stub_->rpcmethod_ShowPartitions_, context, request, response, reactor);
}
::grpc::ClientAsyncResponseReader< ::milvus::grpc::PartitionList>* MilvusService::Stub::AsyncShowPartitionsRaw(::grpc::ClientContext* context, const ::milvus::grpc::TableName& request, ::grpc::CompletionQueue* cq) {
return ::grpc_impl::internal::ClientAsyncResponseReaderFactory< ::milvus::grpc::PartitionList>::Create(channel_.get(), cq, rpcmethod_ShowPartitions_, context, request, true);
}
::grpc::ClientAsyncResponseReader< ::milvus::grpc::PartitionList>* MilvusService::Stub::PrepareAsyncShowPartitionsRaw(::grpc::ClientContext* context, const ::milvus::grpc::TableName& request, ::grpc::CompletionQueue* cq) {
return ::grpc_impl::internal::ClientAsyncResponseReaderFactory< ::milvus::grpc::PartitionList>::Create(channel_.get(), cq, rpcmethod_ShowPartitions_, context, request, false);
}
::grpc::Status MilvusService::Stub::DropPartition(::grpc::ClientContext* context, const ::milvus::grpc::PartitionParam& request, ::milvus::grpc::Status* response) {
return ::grpc::internal::BlockingUnaryCall(channel_.get(), rpcmethod_DropPartition_, context, request, response);
}
void MilvusService::Stub::experimental_async::DropPartition(::grpc::ClientContext* context, const ::milvus::grpc::PartitionParam* request, ::milvus::grpc::Status* response, std::function<void(::grpc::Status)> f) {
::grpc_impl::internal::CallbackUnaryCall(stub_->channel_.get(), stub_->rpcmethod_DropPartition_, context, request, response, std::move(f));
}
void MilvusService::Stub::experimental_async::DropPartition(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::Status* response, std::function<void(::grpc::Status)> f) {
::grpc_impl::internal::CallbackUnaryCall(stub_->channel_.get(), stub_->rpcmethod_DropPartition_, context, request, response, std::move(f));
}
void MilvusService::Stub::experimental_async::DropPartition(::grpc::ClientContext* context, const ::milvus::grpc::PartitionParam* request, ::milvus::grpc::Status* response, ::grpc::experimental::ClientUnaryReactor* reactor) {
::grpc_impl::internal::ClientCallbackUnaryFactory::Create(stub_->channel_.get(), stub_->rpcmethod_DropPartition_, context, request, response, reactor);
}
void MilvusService::Stub::experimental_async::DropPartition(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::Status* response, ::grpc::experimental::ClientUnaryReactor* reactor) {
::grpc_impl::internal::ClientCallbackUnaryFactory::Create(stub_->channel_.get(), stub_->rpcmethod_DropPartition_, context, request, response, reactor);
}
::grpc::ClientAsyncResponseReader< ::milvus::grpc::Status>* MilvusService::Stub::AsyncDropPartitionRaw(::grpc::ClientContext* context, const ::milvus::grpc::PartitionParam& request, ::grpc::CompletionQueue* cq) {
return ::grpc_impl::internal::ClientAsyncResponseReaderFactory< ::milvus::grpc::Status>::Create(channel_.get(), cq, rpcmethod_DropPartition_, context, request, true);
}
::grpc::ClientAsyncResponseReader< ::milvus::grpc::Status>* MilvusService::Stub::PrepareAsyncDropPartitionRaw(::grpc::ClientContext* context, const ::milvus::grpc::PartitionParam& request, ::grpc::CompletionQueue* cq) {
return ::grpc_impl::internal::ClientAsyncResponseReaderFactory< ::milvus::grpc::Status>::Create(channel_.get(), cq, rpcmethod_DropPartition_, context, request, false);
}
::grpc::Status MilvusService::Stub::Insert(::grpc::ClientContext* context, const ::milvus::grpc::InsertParam& request, ::milvus::grpc::VectorIds* response) {
return ::grpc::internal::BlockingUnaryCall(channel_.get(), rpcmethod_Insert_, context, request, response);
}
void MilvusService::Stub::experimental_async::Insert(::grpc::ClientContext* context, const ::milvus::grpc::InsertParam* request, ::milvus::grpc::VectorIds* response, std::function<void(::grpc::Status)> f) {
::grpc_impl::internal::CallbackUnaryCall(stub_->channel_.get(), stub_->rpcmethod_Insert_, context, request, response, std::move(f));
}
void MilvusService::Stub::experimental_async::Insert(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::VectorIds* response, std::function<void(::grpc::Status)> f) {
::grpc_impl::internal::CallbackUnaryCall(stub_->channel_.get(), stub_->rpcmethod_Insert_, context, request, response, std::move(f));
}
void MilvusService::Stub::experimental_async::Insert(::grpc::ClientContext* context, const ::milvus::grpc::InsertParam* request, ::milvus::grpc::VectorIds* response, ::grpc::experimental::ClientUnaryReactor* reactor) {
::grpc_impl::internal::ClientCallbackUnaryFactory::Create(stub_->channel_.get(), stub_->rpcmethod_Insert_, context, request, response, reactor);
}
void MilvusService::Stub::experimental_async::Insert(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::VectorIds* response, ::grpc::experimental::ClientUnaryReactor* reactor) {
::grpc_impl::internal::ClientCallbackUnaryFactory::Create(stub_->channel_.get(), stub_->rpcmethod_Insert_, context, request, response, reactor);
}
::grpc::ClientAsyncResponseReader< ::milvus::grpc::VectorIds>* MilvusService::Stub::AsyncInsertRaw(::grpc::ClientContext* context, const ::milvus::grpc::InsertParam& request, ::grpc::CompletionQueue* cq) {
return ::grpc_impl::internal::ClientAsyncResponseReaderFactory< ::milvus::grpc::VectorIds>::Create(channel_.get(), cq, rpcmethod_Insert_, context, request, true);
}
::grpc::ClientAsyncResponseReader< ::milvus::grpc::VectorIds>* MilvusService::Stub::PrepareAsyncInsertRaw(::grpc::ClientContext* context, const ::milvus::grpc::InsertParam& request, ::grpc::CompletionQueue* cq) {
return ::grpc_impl::internal::ClientAsyncResponseReaderFactory< ::milvus::grpc::VectorIds>::Create(channel_.get(), cq, rpcmethod_Insert_, context, request, false);
}
::grpc::Status MilvusService::Stub::Search(::grpc::ClientContext* context, const ::milvus::grpc::SearchParam& request, ::milvus::grpc::TopKQueryResultList* response) {
return ::grpc::internal::BlockingUnaryCall(channel_.get(), rpcmethod_Search_, context, request, response);
}
void MilvusService::Stub::experimental_async::Search(::grpc::ClientContext* context, const ::milvus::grpc::SearchParam* request, ::milvus::grpc::TopKQueryResultList* response, std::function<void(::grpc::Status)> f) {
::grpc_impl::internal::CallbackUnaryCall(stub_->channel_.get(), stub_->rpcmethod_Search_, context, request, response, std::move(f));
}
void MilvusService::Stub::experimental_async::Search(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::TopKQueryResultList* response, std::function<void(::grpc::Status)> f) {
::grpc_impl::internal::CallbackUnaryCall(stub_->channel_.get(), stub_->rpcmethod_Search_, context, request, response, std::move(f));
}
void MilvusService::Stub::experimental_async::Search(::grpc::ClientContext* context, const ::milvus::grpc::SearchParam* request, ::milvus::grpc::TopKQueryResultList* response, ::grpc::experimental::ClientUnaryReactor* reactor) {
::grpc_impl::internal::ClientCallbackUnaryFactory::Create(stub_->channel_.get(), stub_->rpcmethod_Search_, context, request, response, reactor);
}
void MilvusService::Stub::experimental_async::Search(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::TopKQueryResultList* response, ::grpc::experimental::ClientUnaryReactor* reactor) {
::grpc_impl::internal::ClientCallbackUnaryFactory::Create(stub_->channel_.get(), stub_->rpcmethod_Search_, context, request, response, reactor);
}
::grpc::ClientAsyncResponseReader< ::milvus::grpc::TopKQueryResultList>* MilvusService::Stub::AsyncSearchRaw(::grpc::ClientContext* context, const ::milvus::grpc::SearchParam& request, ::grpc::CompletionQueue* cq) {
return ::grpc_impl::internal::ClientAsyncResponseReaderFactory< ::milvus::grpc::TopKQueryResultList>::Create(channel_.get(), cq, rpcmethod_Search_, context, request, true);
}
::grpc::ClientAsyncResponseReader< ::milvus::grpc::TopKQueryResultList>* MilvusService::Stub::PrepareAsyncSearchRaw(::grpc::ClientContext* context, const ::milvus::grpc::SearchParam& request, ::grpc::CompletionQueue* cq) {
return ::grpc_impl::internal::ClientAsyncResponseReaderFactory< ::milvus::grpc::TopKQueryResultList>::Create(channel_.get(), cq, rpcmethod_Search_, context, request, false);
}
::grpc::Status MilvusService::Stub::SearchInFiles(::grpc::ClientContext* context, const ::milvus::grpc::SearchInFilesParam& request, ::milvus::grpc::TopKQueryResultList* response) {
return ::grpc::internal::BlockingUnaryCall(channel_.get(), rpcmethod_SearchInFiles_, context, request, response);
}
void MilvusService::Stub::experimental_async::SearchInFiles(::grpc::ClientContext* context, const ::milvus::grpc::SearchInFilesParam* request, ::milvus::grpc::TopKQueryResultList* response, std::function<void(::grpc::Status)> f) {
::grpc_impl::internal::CallbackUnaryCall(stub_->channel_.get(), stub_->rpcmethod_SearchInFiles_, context, request, response, std::move(f));
}
void MilvusService::Stub::experimental_async::SearchInFiles(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::TopKQueryResultList* response, std::function<void(::grpc::Status)> f) {
::grpc_impl::internal::CallbackUnaryCall(stub_->channel_.get(), stub_->rpcmethod_SearchInFiles_, context, request, response, std::move(f));
}
void MilvusService::Stub::experimental_async::SearchInFiles(::grpc::ClientContext* context, const ::milvus::grpc::SearchInFilesParam* request, ::milvus::grpc::TopKQueryResultList* response, ::grpc::experimental::ClientUnaryReactor* reactor) {
::grpc_impl::internal::ClientCallbackUnaryFactory::Create(stub_->channel_.get(), stub_->rpcmethod_SearchInFiles_, context, request, response, reactor);
}
void MilvusService::Stub::experimental_async::SearchInFiles(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::TopKQueryResultList* response, ::grpc::experimental::ClientUnaryReactor* reactor) {
::grpc_impl::internal::ClientCallbackUnaryFactory::Create(stub_->channel_.get(), stub_->rpcmethod_SearchInFiles_, context, request, response, reactor);
}
::grpc::ClientAsyncResponseReader< ::milvus::grpc::TopKQueryResultList>* MilvusService::Stub::AsyncSearchInFilesRaw(::grpc::ClientContext* context, const ::milvus::grpc::SearchInFilesParam& request, ::grpc::CompletionQueue* cq) {
return ::grpc_impl::internal::ClientAsyncResponseReaderFactory< ::milvus::grpc::TopKQueryResultList>::Create(channel_.get(), cq, rpcmethod_SearchInFiles_, context, request, true);
}
::grpc::ClientAsyncResponseReader< ::milvus::grpc::TopKQueryResultList>* MilvusService::Stub::PrepareAsyncSearchInFilesRaw(::grpc::ClientContext* context, const ::milvus::grpc::SearchInFilesParam& request, ::grpc::CompletionQueue* cq) {
return ::grpc_impl::internal::ClientAsyncResponseReaderFactory< ::milvus::grpc::TopKQueryResultList>::Create(channel_.get(), cq, rpcmethod_SearchInFiles_, context, request, false);
}
::grpc::Status MilvusService::Stub::Cmd(::grpc::ClientContext* context, const ::milvus::grpc::Command& request, ::milvus::grpc::StringReply* response) {
return ::grpc::internal::BlockingUnaryCall(channel_.get(), rpcmethod_Cmd_, context, request, response);
}
void MilvusService::Stub::experimental_async::Cmd(::grpc::ClientContext* context, const ::milvus::grpc::Command* request, ::milvus::grpc::StringReply* response, std::function<void(::grpc::Status)> f) {
::grpc_impl::internal::CallbackUnaryCall(stub_->channel_.get(), stub_->rpcmethod_Cmd_, context, request, response, std::move(f));
}
void MilvusService::Stub::experimental_async::Cmd(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::StringReply* response, std::function<void(::grpc::Status)> f) {
::grpc_impl::internal::CallbackUnaryCall(stub_->channel_.get(), stub_->rpcmethod_Cmd_, context, request, response, std::move(f));
}
void MilvusService::Stub::experimental_async::Cmd(::grpc::ClientContext* context, const ::milvus::grpc::Command* request, ::milvus::grpc::StringReply* response, ::grpc::experimental::ClientUnaryReactor* reactor) {
::grpc_impl::internal::ClientCallbackUnaryFactory::Create(stub_->channel_.get(), stub_->rpcmethod_Cmd_, context, request, response, reactor);
}
void MilvusService::Stub::experimental_async::Cmd(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::StringReply* response, ::grpc::experimental::ClientUnaryReactor* reactor) {
::grpc_impl::internal::ClientCallbackUnaryFactory::Create(stub_->channel_.get(), stub_->rpcmethod_Cmd_, context, request, response, reactor);
}
::grpc::ClientAsyncResponseReader< ::milvus::grpc::StringReply>* MilvusService::Stub::AsyncCmdRaw(::grpc::ClientContext* context, const ::milvus::grpc::Command& request, ::grpc::CompletionQueue* cq) {
return ::grpc_impl::internal::ClientAsyncResponseReaderFactory< ::milvus::grpc::StringReply>::Create(channel_.get(), cq, rpcmethod_Cmd_, context, request, true);
}
::grpc::ClientAsyncResponseReader< ::milvus::grpc::StringReply>* MilvusService::Stub::PrepareAsyncCmdRaw(::grpc::ClientContext* context, const ::milvus::grpc::Command& request, ::grpc::CompletionQueue* cq) {
return ::grpc_impl::internal::ClientAsyncResponseReaderFactory< ::milvus::grpc::StringReply>::Create(channel_.get(), cq, rpcmethod_Cmd_, context, request, false);
}
::grpc::Status MilvusService::Stub::DeleteByDate(::grpc::ClientContext* context, const ::milvus::grpc::DeleteByDateParam& request, ::milvus::grpc::Status* response) {
return ::grpc::internal::BlockingUnaryCall(channel_.get(), rpcmethod_DeleteByDate_, context, request, response);
}
void MilvusService::Stub::experimental_async::DeleteByDate(::grpc::ClientContext* context, const ::milvus::grpc::DeleteByDateParam* request, ::milvus::grpc::Status* response, std::function<void(::grpc::Status)> f) {
::grpc_impl::internal::CallbackUnaryCall(stub_->channel_.get(), stub_->rpcmethod_DeleteByDate_, context, request, response, std::move(f));
}
void MilvusService::Stub::experimental_async::DeleteByDate(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::Status* response, std::function<void(::grpc::Status)> f) {
::grpc_impl::internal::CallbackUnaryCall(stub_->channel_.get(), stub_->rpcmethod_DeleteByDate_, context, request, response, std::move(f));
}
void MilvusService::Stub::experimental_async::DeleteByDate(::grpc::ClientContext* context, const ::milvus::grpc::DeleteByDateParam* request, ::milvus::grpc::Status* response, ::grpc::experimental::ClientUnaryReactor* reactor) {
::grpc_impl::internal::ClientCallbackUnaryFactory::Create(stub_->channel_.get(), stub_->rpcmethod_DeleteByDate_, context, request, response, reactor);
}
void MilvusService::Stub::experimental_async::DeleteByDate(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::Status* response, ::grpc::experimental::ClientUnaryReactor* reactor) {
::grpc_impl::internal::ClientCallbackUnaryFactory::Create(stub_->channel_.get(), stub_->rpcmethod_DeleteByDate_, context, request, response, reactor);
}
::grpc::ClientAsyncResponseReader< ::milvus::grpc::Status>* MilvusService::Stub::AsyncDeleteByDateRaw(::grpc::ClientContext* context, const ::milvus::grpc::DeleteByDateParam& request, ::grpc::CompletionQueue* cq) {
return ::grpc_impl::internal::ClientAsyncResponseReaderFactory< ::milvus::grpc::Status>::Create(channel_.get(), cq, rpcmethod_DeleteByDate_, context, request, true);
}
::grpc::ClientAsyncResponseReader< ::milvus::grpc::Status>* MilvusService::Stub::PrepareAsyncDeleteByDateRaw(::grpc::ClientContext* context, const ::milvus::grpc::DeleteByDateParam& request, ::grpc::CompletionQueue* cq) {
return ::grpc_impl::internal::ClientAsyncResponseReaderFactory< ::milvus::grpc::Status>::Create(channel_.get(), cq, rpcmethod_DeleteByDate_, context, request, false);
}
::grpc::Status MilvusService::Stub::PreloadTable(::grpc::ClientContext* context, const ::milvus::grpc::TableName& request, ::milvus::grpc::Status* response) {
return ::grpc::internal::BlockingUnaryCall(channel_.get(), rpcmethod_PreloadTable_, context, request, response);
}
void MilvusService::Stub::experimental_async::PreloadTable(::grpc::ClientContext* context, const ::milvus::grpc::TableName* request, ::milvus::grpc::Status* response, std::function<void(::grpc::Status)> f) {
::grpc_impl::internal::CallbackUnaryCall(stub_->channel_.get(), stub_->rpcmethod_PreloadTable_, context, request, response, std::move(f));
}
void MilvusService::Stub::experimental_async::PreloadTable(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::Status* response, std::function<void(::grpc::Status)> f) {
::grpc_impl::internal::CallbackUnaryCall(stub_->channel_.get(), stub_->rpcmethod_PreloadTable_, context, request, response, std::move(f));
}
void MilvusService::Stub::experimental_async::PreloadTable(::grpc::ClientContext* context, const ::milvus::grpc::TableName* request, ::milvus::grpc::Status* response, ::grpc::experimental::ClientUnaryReactor* reactor) {
::grpc_impl::internal::ClientCallbackUnaryFactory::Create(stub_->channel_.get(), stub_->rpcmethod_PreloadTable_, context, request, response, reactor);
}
void MilvusService::Stub::experimental_async::PreloadTable(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::Status* response, ::grpc::experimental::ClientUnaryReactor* reactor) {
::grpc_impl::internal::ClientCallbackUnaryFactory::Create(stub_->channel_.get(), stub_->rpcmethod_PreloadTable_, context, request, response, reactor);
}
::grpc::ClientAsyncResponseReader< ::milvus::grpc::Status>* MilvusService::Stub::AsyncPreloadTableRaw(::grpc::ClientContext* context, const ::milvus::grpc::TableName& request, ::grpc::CompletionQueue* cq) {
return ::grpc_impl::internal::ClientAsyncResponseReaderFactory< ::milvus::grpc::Status>::Create(channel_.get(), cq, rpcmethod_PreloadTable_, context, request, true);
}
::grpc::ClientAsyncResponseReader< ::milvus::grpc::Status>* MilvusService::Stub::PrepareAsyncPreloadTableRaw(::grpc::ClientContext* context, const ::milvus::grpc::TableName& request, ::grpc::CompletionQueue* cq) {
return ::grpc_impl::internal::ClientAsyncResponseReaderFactory< ::milvus::grpc::Status>::Create(channel_.get(), cq, rpcmethod_PreloadTable_, context, request, false);
}
MilvusService::Service::Service() {
AddMethod(new ::grpc::internal::RpcServiceMethod(
MilvusService_method_names[0],
@ -495,68 +585,83 @@ MilvusService::Service::Service() {
AddMethod(new ::grpc::internal::RpcServiceMethod(
MilvusService_method_names[2],
::grpc::internal::RpcMethod::NORMAL_RPC,
new ::grpc::internal::RpcMethodHandler< MilvusService::Service, ::milvus::grpc::TableName, ::milvus::grpc::Status>(
std::mem_fn(&MilvusService::Service::DropTable), this)));
AddMethod(new ::grpc::internal::RpcServiceMethod(
MilvusService_method_names[3],
::grpc::internal::RpcMethod::NORMAL_RPC,
new ::grpc::internal::RpcMethodHandler< MilvusService::Service, ::milvus::grpc::IndexParam, ::milvus::grpc::Status>(
std::mem_fn(&MilvusService::Service::CreateIndex), this)));
AddMethod(new ::grpc::internal::RpcServiceMethod(
MilvusService_method_names[4],
::grpc::internal::RpcMethod::NORMAL_RPC,
new ::grpc::internal::RpcMethodHandler< MilvusService::Service, ::milvus::grpc::InsertParam, ::milvus::grpc::VectorIds>(
std::mem_fn(&MilvusService::Service::Insert), this)));
AddMethod(new ::grpc::internal::RpcServiceMethod(
MilvusService_method_names[5],
::grpc::internal::RpcMethod::NORMAL_RPC,
new ::grpc::internal::RpcMethodHandler< MilvusService::Service, ::milvus::grpc::SearchParam, ::milvus::grpc::TopKQueryResultList>(
std::mem_fn(&MilvusService::Service::Search), this)));
AddMethod(new ::grpc::internal::RpcServiceMethod(
MilvusService_method_names[6],
::grpc::internal::RpcMethod::NORMAL_RPC,
new ::grpc::internal::RpcMethodHandler< MilvusService::Service, ::milvus::grpc::SearchInFilesParam, ::milvus::grpc::TopKQueryResultList>(
std::mem_fn(&MilvusService::Service::SearchInFiles), this)));
AddMethod(new ::grpc::internal::RpcServiceMethod(
MilvusService_method_names[7],
::grpc::internal::RpcMethod::NORMAL_RPC,
new ::grpc::internal::RpcMethodHandler< MilvusService::Service, ::milvus::grpc::TableName, ::milvus::grpc::TableSchema>(
std::mem_fn(&MilvusService::Service::DescribeTable), this)));
AddMethod(new ::grpc::internal::RpcServiceMethod(
MilvusService_method_names[8],
MilvusService_method_names[3],
::grpc::internal::RpcMethod::NORMAL_RPC,
new ::grpc::internal::RpcMethodHandler< MilvusService::Service, ::milvus::grpc::TableName, ::milvus::grpc::TableRowCount>(
std::mem_fn(&MilvusService::Service::CountTable), this)));
AddMethod(new ::grpc::internal::RpcServiceMethod(
MilvusService_method_names[9],
MilvusService_method_names[4],
::grpc::internal::RpcMethod::NORMAL_RPC,
new ::grpc::internal::RpcMethodHandler< MilvusService::Service, ::milvus::grpc::Command, ::milvus::grpc::TableNameList>(
std::mem_fn(&MilvusService::Service::ShowTables), this)));
AddMethod(new ::grpc::internal::RpcServiceMethod(
MilvusService_method_names[10],
::grpc::internal::RpcMethod::NORMAL_RPC,
new ::grpc::internal::RpcMethodHandler< MilvusService::Service, ::milvus::grpc::Command, ::milvus::grpc::StringReply>(
std::mem_fn(&MilvusService::Service::Cmd), this)));
AddMethod(new ::grpc::internal::RpcServiceMethod(
MilvusService_method_names[11],
::grpc::internal::RpcMethod::NORMAL_RPC,
new ::grpc::internal::RpcMethodHandler< MilvusService::Service, ::milvus::grpc::DeleteByRangeParam, ::milvus::grpc::Status>(
std::mem_fn(&MilvusService::Service::DeleteByRange), this)));
AddMethod(new ::grpc::internal::RpcServiceMethod(
MilvusService_method_names[12],
MilvusService_method_names[5],
::grpc::internal::RpcMethod::NORMAL_RPC,
new ::grpc::internal::RpcMethodHandler< MilvusService::Service, ::milvus::grpc::TableName, ::milvus::grpc::Status>(
std::mem_fn(&MilvusService::Service::PreloadTable), this)));
std::mem_fn(&MilvusService::Service::DropTable), this)));
AddMethod(new ::grpc::internal::RpcServiceMethod(
MilvusService_method_names[13],
MilvusService_method_names[6],
::grpc::internal::RpcMethod::NORMAL_RPC,
new ::grpc::internal::RpcMethodHandler< MilvusService::Service, ::milvus::grpc::IndexParam, ::milvus::grpc::Status>(
std::mem_fn(&MilvusService::Service::CreateIndex), this)));
AddMethod(new ::grpc::internal::RpcServiceMethod(
MilvusService_method_names[7],
::grpc::internal::RpcMethod::NORMAL_RPC,
new ::grpc::internal::RpcMethodHandler< MilvusService::Service, ::milvus::grpc::TableName, ::milvus::grpc::IndexParam>(
std::mem_fn(&MilvusService::Service::DescribeIndex), this)));
AddMethod(new ::grpc::internal::RpcServiceMethod(
MilvusService_method_names[14],
MilvusService_method_names[8],
::grpc::internal::RpcMethod::NORMAL_RPC,
new ::grpc::internal::RpcMethodHandler< MilvusService::Service, ::milvus::grpc::TableName, ::milvus::grpc::Status>(
std::mem_fn(&MilvusService::Service::DropIndex), this)));
AddMethod(new ::grpc::internal::RpcServiceMethod(
MilvusService_method_names[9],
::grpc::internal::RpcMethod::NORMAL_RPC,
new ::grpc::internal::RpcMethodHandler< MilvusService::Service, ::milvus::grpc::PartitionParam, ::milvus::grpc::Status>(
std::mem_fn(&MilvusService::Service::CreatePartition), this)));
AddMethod(new ::grpc::internal::RpcServiceMethod(
MilvusService_method_names[10],
::grpc::internal::RpcMethod::NORMAL_RPC,
new ::grpc::internal::RpcMethodHandler< MilvusService::Service, ::milvus::grpc::TableName, ::milvus::grpc::PartitionList>(
std::mem_fn(&MilvusService::Service::ShowPartitions), this)));
AddMethod(new ::grpc::internal::RpcServiceMethod(
MilvusService_method_names[11],
::grpc::internal::RpcMethod::NORMAL_RPC,
new ::grpc::internal::RpcMethodHandler< MilvusService::Service, ::milvus::grpc::PartitionParam, ::milvus::grpc::Status>(
std::mem_fn(&MilvusService::Service::DropPartition), this)));
AddMethod(new ::grpc::internal::RpcServiceMethod(
MilvusService_method_names[12],
::grpc::internal::RpcMethod::NORMAL_RPC,
new ::grpc::internal::RpcMethodHandler< MilvusService::Service, ::milvus::grpc::InsertParam, ::milvus::grpc::VectorIds>(
std::mem_fn(&MilvusService::Service::Insert), this)));
AddMethod(new ::grpc::internal::RpcServiceMethod(
MilvusService_method_names[13],
::grpc::internal::RpcMethod::NORMAL_RPC,
new ::grpc::internal::RpcMethodHandler< MilvusService::Service, ::milvus::grpc::SearchParam, ::milvus::grpc::TopKQueryResultList>(
std::mem_fn(&MilvusService::Service::Search), this)));
AddMethod(new ::grpc::internal::RpcServiceMethod(
MilvusService_method_names[14],
::grpc::internal::RpcMethod::NORMAL_RPC,
new ::grpc::internal::RpcMethodHandler< MilvusService::Service, ::milvus::grpc::SearchInFilesParam, ::milvus::grpc::TopKQueryResultList>(
std::mem_fn(&MilvusService::Service::SearchInFiles), this)));
AddMethod(new ::grpc::internal::RpcServiceMethod(
MilvusService_method_names[15],
::grpc::internal::RpcMethod::NORMAL_RPC,
new ::grpc::internal::RpcMethodHandler< MilvusService::Service, ::milvus::grpc::Command, ::milvus::grpc::StringReply>(
std::mem_fn(&MilvusService::Service::Cmd), this)));
AddMethod(new ::grpc::internal::RpcServiceMethod(
MilvusService_method_names[16],
::grpc::internal::RpcMethod::NORMAL_RPC,
new ::grpc::internal::RpcMethodHandler< MilvusService::Service, ::milvus::grpc::DeleteByDateParam, ::milvus::grpc::Status>(
std::mem_fn(&MilvusService::Service::DeleteByDate), this)));
AddMethod(new ::grpc::internal::RpcServiceMethod(
MilvusService_method_names[17],
::grpc::internal::RpcMethod::NORMAL_RPC,
new ::grpc::internal::RpcMethodHandler< MilvusService::Service, ::milvus::grpc::TableName, ::milvus::grpc::Status>(
std::mem_fn(&MilvusService::Service::PreloadTable), this)));
}
MilvusService::Service::~Service() {
@ -576,41 +681,6 @@ MilvusService::Service::~Service() {
return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, "");
}
::grpc::Status MilvusService::Service::DropTable(::grpc::ServerContext* context, const ::milvus::grpc::TableName* request, ::milvus::grpc::Status* response) {
(void) context;
(void) request;
(void) response;
return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, "");
}
::grpc::Status MilvusService::Service::CreateIndex(::grpc::ServerContext* context, const ::milvus::grpc::IndexParam* request, ::milvus::grpc::Status* response) {
(void) context;
(void) request;
(void) response;
return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, "");
}
::grpc::Status MilvusService::Service::Insert(::grpc::ServerContext* context, const ::milvus::grpc::InsertParam* request, ::milvus::grpc::VectorIds* response) {
(void) context;
(void) request;
(void) response;
return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, "");
}
::grpc::Status MilvusService::Service::Search(::grpc::ServerContext* context, const ::milvus::grpc::SearchParam* request, ::milvus::grpc::TopKQueryResultList* response) {
(void) context;
(void) request;
(void) response;
return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, "");
}
::grpc::Status MilvusService::Service::SearchInFiles(::grpc::ServerContext* context, const ::milvus::grpc::SearchInFilesParam* request, ::milvus::grpc::TopKQueryResultList* response) {
(void) context;
(void) request;
(void) response;
return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, "");
}
::grpc::Status MilvusService::Service::DescribeTable(::grpc::ServerContext* context, const ::milvus::grpc::TableName* request, ::milvus::grpc::TableSchema* response) {
(void) context;
(void) request;
@ -632,21 +702,14 @@ MilvusService::Service::~Service() {
return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, "");
}
::grpc::Status MilvusService::Service::Cmd(::grpc::ServerContext* context, const ::milvus::grpc::Command* request, ::milvus::grpc::StringReply* response) {
::grpc::Status MilvusService::Service::DropTable(::grpc::ServerContext* context, const ::milvus::grpc::TableName* request, ::milvus::grpc::Status* response) {
(void) context;
(void) request;
(void) response;
return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, "");
}
::grpc::Status MilvusService::Service::DeleteByRange(::grpc::ServerContext* context, const ::milvus::grpc::DeleteByRangeParam* request, ::milvus::grpc::Status* response) {
(void) context;
(void) request;
(void) response;
return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, "");
}
::grpc::Status MilvusService::Service::PreloadTable(::grpc::ServerContext* context, const ::milvus::grpc::TableName* request, ::milvus::grpc::Status* response) {
::grpc::Status MilvusService::Service::CreateIndex(::grpc::ServerContext* context, const ::milvus::grpc::IndexParam* request, ::milvus::grpc::Status* response) {
(void) context;
(void) request;
(void) response;
@ -667,6 +730,69 @@ MilvusService::Service::~Service() {
return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, "");
}
::grpc::Status MilvusService::Service::CreatePartition(::grpc::ServerContext* context, const ::milvus::grpc::PartitionParam* request, ::milvus::grpc::Status* response) {
(void) context;
(void) request;
(void) response;
return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, "");
}
::grpc::Status MilvusService::Service::ShowPartitions(::grpc::ServerContext* context, const ::milvus::grpc::TableName* request, ::milvus::grpc::PartitionList* response) {
(void) context;
(void) request;
(void) response;
return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, "");
}
::grpc::Status MilvusService::Service::DropPartition(::grpc::ServerContext* context, const ::milvus::grpc::PartitionParam* request, ::milvus::grpc::Status* response) {
(void) context;
(void) request;
(void) response;
return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, "");
}
::grpc::Status MilvusService::Service::Insert(::grpc::ServerContext* context, const ::milvus::grpc::InsertParam* request, ::milvus::grpc::VectorIds* response) {
(void) context;
(void) request;
(void) response;
return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, "");
}
::grpc::Status MilvusService::Service::Search(::grpc::ServerContext* context, const ::milvus::grpc::SearchParam* request, ::milvus::grpc::TopKQueryResultList* response) {
(void) context;
(void) request;
(void) response;
return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, "");
}
::grpc::Status MilvusService::Service::SearchInFiles(::grpc::ServerContext* context, const ::milvus::grpc::SearchInFilesParam* request, ::milvus::grpc::TopKQueryResultList* response) {
(void) context;
(void) request;
(void) response;
return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, "");
}
::grpc::Status MilvusService::Service::Cmd(::grpc::ServerContext* context, const ::milvus::grpc::Command* request, ::milvus::grpc::StringReply* response) {
(void) context;
(void) request;
(void) response;
return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, "");
}
::grpc::Status MilvusService::Service::DeleteByDate(::grpc::ServerContext* context, const ::milvus::grpc::DeleteByDateParam* request, ::milvus::grpc::Status* response) {
(void) context;
(void) request;
(void) response;
return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, "");
}
::grpc::Status MilvusService::Service::PreloadTable(::grpc::ServerContext* context, const ::milvus::grpc::TableName* request, ::milvus::grpc::Status* response) {
(void) context;
(void) request;
(void) response;
return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, "");
}
} // namespace milvus
} // namespace grpc

3533
core/src/grpc/gen-milvus/milvus.grpc.pb.h
View File

File diff suppressed because it is too large Load Diff

1528
core/src/grpc/gen-milvus/milvus.pb.cc
View File

File diff suppressed because it is too large Load Diff

1054
core/src/grpc/gen-milvus/milvus.pb.h
View File

File diff suppressed because it is too large Load Diff

274
core/src/grpc/milvus.proto
View File

@ -5,14 +5,21 @@ import "status.proto";
package milvus.grpc;
/**
* @brief Table Name
* @brief Table name
*/
message TableName {
string table_name = 1;
}
/**
* @brief Table Name List
* @brief Partition name
*/
message PartitionName {
string partition_name = 1;
}
/**
* @brief Table name list
*/
message TableNameList {
Status status = 1;
@ -20,7 +27,7 @@ message TableNameList {
}
/**
* @brief Table Schema
* @brief Table schema
*/
message TableSchema {
Status status = 1;
@ -31,7 +38,24 @@ message TableSchema {
}
/**
* @brief Range Schema
* @brief Params of partition
*/
message PartitionParam {
string table_name = 1;
string partition_name = 2;
string tag = 3;
}
/**
* @brief Partition list
*/
message PartitionList {
Status status = 1;
repeated PartitionParam partition_array = 2;
}
/**
* @brief Range schema
*/
message Range {
string start_value = 1;
@ -46,12 +70,13 @@ message RowRecord {
}
/**
* @brief params to be inserted
* @brief Params to be inserted
*/
message InsertParam {
string table_name = 1;
repeated RowRecord row_record_array = 2;
repeated int64 row_id_array = 3; //optional
string partition_tag = 4;
}
/**
@ -63,7 +88,7 @@ message VectorIds {
}
/**
* @brief params for searching vector
* @brief Params for searching vector
*/
message SearchParam {
string table_name = 1;
@ -71,10 +96,11 @@ message SearchParam {
repeated Range query_range_array = 3;
int64 topk = 4;
int64 nprobe = 5;
repeated string partition_tag_array = 6;
}
/**
* @brief params for searching vector in files
* @brief Params for searching vector in files
*/
message SearchInFilesParam {
repeated string file_id_array = 1;
@ -105,7 +131,7 @@ message TopKQueryResultList {
}
/**
* @brief Server String Reply
* @brief Server string Reply
*/
message StringReply {
Status status = 1;
@ -129,7 +155,7 @@ message TableRowCount {
}
/**
* @brief Give Server Command
* @brief Give server Command
*/
message Command {
string cmd = 1;
@ -155,169 +181,173 @@ message IndexParam {
}
/**
* @brief table name and range for DeleteByRange
* @brief table name and range for DeleteByDate
*/
message DeleteByRangeParam {
message DeleteByDateParam {
Range range = 1;
string table_name = 2;
}
service MilvusService {
/**
* @brief Create table method
* @brief This method is used to create table
*
* This method is used to create table
*
* @param param, use to provide table information to be created.
* @param TableSchema, use to provide table information to be created.
*
* @return Status
*/
rpc CreateTable(TableSchema) returns (Status){}
/**
* @brief Test table existence method
* @brief This method is used to test table existence.
*
* This method is used to test table existence.
*
* @param table_name, table name is going to be tested.
* @param TableName, table name is going to be tested.
*
* @return BoolReply
*/
rpc HasTable(TableName) returns (BoolReply) {}
/**
* @brief Delete table method
* @brief This method is used to get table schema.
*
* This method is used to delete table.
* @param TableName, target table name.
*
* @param table_name, table name is going to be deleted.
*
*/
rpc DropTable(TableName) returns (Status) {}
/**
* @brief Build index by table method
*
* This method is used to build index by table in sync mode.
*
* @param table_name, table is going to be built index.
*
*/
rpc CreateIndex(IndexParam) returns (Status) {}
/**
* @brief Add vector array to table
*
* This method is used to add vector array to table.
*
* @param table_name, table_name is inserted.
* @param record_array, vector array is inserted.
*
* @return vector id array
*/
rpc Insert(InsertParam) returns (VectorIds) {}
/**
* @brief Query vector
*
* This method is used to query vector in table.
*
* @param table_name, table_name is queried.
* @param query_record_array, all vector are going to be queried.
* @param query_range_array, optional ranges for conditional search. If not specified, search whole table
* @param topk, how many similarity vectors will be searched.
*
* @return query result array.
*/
rpc Search(SearchParam) returns (TopKQueryResultList) {}
/**
* @brief Internal use query interface
*
* This method is used to query vector in specified files.
*
* @param file_id_array, specified files id array, queried.
* @param query_record_array, all vector are going to be queried.
* @param query_range_array, optional ranges for conditional search. If not specified, search whole table
* @param topk, how many similarity vectors will be searched.
*
* @return query result array.
*/
rpc SearchInFiles(SearchInFilesParam) returns (TopKQueryResultList) {}
/**
* @brief Get table schema
*
* This method is used to get table schema.
*
* @param table_name, target table name.
*
* @return table schema
* @return TableSchema
*/
rpc DescribeTable(TableName) returns (TableSchema) {}
/**
* @brief Get table schema
* @brief This method is used to get table schema.
*
* This method is used to get table schema.
* @param TableName, target table name.
*
* @param table_name, target table name.
*
* @return table schema
* @return TableRowCount
*/
rpc CountTable(TableName) returns (TableRowCount) {}
/**
* @brief List all tables in database
* @brief This method is used to list all tables.
*
* This method is used to list all tables.
* @param Command, dummy parameter.
*
*
* @return table names.
* @return TableNameList
*/
rpc ShowTables(Command) returns (TableNameList) {}
/**
* @brief Give the server status
* @brief This method is used to delete table.
*
* This method is used to give the server status.
* @param TableName, table name is going to be deleted.
*
* @return Server status.
* @return TableNameList
*/
rpc DropTable(TableName) returns (Status) {}
/**
* @brief This method is used to build index by table in sync mode.
*
* @param IndexParam, index paramters.
*
* @return Status
*/
rpc CreateIndex(IndexParam) returns (Status) {}
/**
* @brief This method is used to describe index
*
* @param TableName, target table name.
*
* @return IndexParam
*/
rpc DescribeIndex(TableName) returns (IndexParam) {}
/**
* @brief This method is used to drop index
*
* @param TableName, target table name.
*
* @return Status
*/
rpc DropIndex(TableName) returns (Status) {}
/**
* @brief This method is used to create partition
*
* @param PartitionParam, partition parameters.
*
* @return Status
*/
rpc CreatePartition(PartitionParam) returns (Status) {}
/**
* @brief This method is used to show partition information
*
* @param TableName, target table name.
*
* @return PartitionList
*/
rpc ShowPartitions(TableName) returns (PartitionList) {}
/**
* @brief This method is used to drop partition
*
* @param PartitionParam, target partition.
*
* @return Status
*/
rpc DropPartition(PartitionParam) returns (Status) {}
/**
* @brief This method is used to add vector array to table.
*
* @param InsertParam, insert parameters.
*
* @return VectorIds
*/
rpc Insert(InsertParam) returns (VectorIds) {}
/**
* @brief This method is used to query vector in table.
*
* @param SearchParam, search parameters.
*
* @return TopKQueryResultList
*/
rpc Search(SearchParam) returns (TopKQueryResultList) {}
/**
* @brief This method is used to query vector in specified files.
*
* @param SearchInFilesParam, search in files paremeters.
*
* @return TopKQueryResultList
*/
rpc SearchInFiles(SearchInFilesParam) returns (TopKQueryResultList) {}
/**
* @brief This method is used to give the server status.
*
* @param Command, command string
*
* @return StringReply
*/
rpc Cmd(Command) returns (StringReply) {}
/**
* @brief delete table by range
* @brief This method is used to delete vector by date range
*
* This method is used to delete vector by range
* @param DeleteByDateParam, delete parameters.
*
* @return rpc status.
* @return status
*/
rpc DeleteByRange(DeleteByRangeParam) returns (Status) {}
rpc DeleteByDate(DeleteByDateParam) returns (Status) {}
/**
* @brief preload table
* @brief This method is used to preload table
*
* This method is used to preload table
* @param TableName, target table name.
*
* @return Status.
* @return Status
*/
rpc PreloadTable(TableName) returns (Status) {}
/**
* @brief describe index
*
* This method is used to describe index
*
* @return Status.
*/
rpc DescribeIndex(TableName) returns (IndexParam) {}
/**
* @brief drop index
*
* This method is used to drop index
*
* @return Status.
*/
rpc DropIndex(TableName) returns (Status) {}
}

16
core/src/index/CMakeLists.txt
View File

@ -18,11 +18,11 @@
#-------------------------------------------------------------------------------
cmake_minimum_required(VERSION 3.14)
cmake_minimum_required(VERSION 3.12)
message(STATUS "------------------------------KNOWHERE-----------------------------------")
message(STATUS "Building using CMake version: ${CMAKE_VERSION}")
set(KNOWHERE_VERSION "0.5.0")
set(KNOWHERE_VERSION "0.6.0")
string(REGEX MATCH "^[0-9]+\\.[0-9]+\\.[0-9]+" KNOWHERE_BASE_VERSION "${KNOWHERE_VERSION}")
project(knowhere VERSION "${KNOWHERE_BASE_VERSION}" LANGUAGES C CXX)
set(CMAKE_CXX_STANDARD 14)
@ -72,17 +72,17 @@ include(ExternalProject)
include(DefineOptionsCore)
include(BuildUtilsCore)
set(KNOWHERE_GPU_VERSION false)
if (MILVUS_CPU_VERSION OR KNOWHERE_CPU_VERSION)
message(STATUS "Building Knowhere CPU version")
add_compile_definitions("MILVUS_CPU_VERSION")
else ()
set(KNOWHERE_CPU_VERSION false)
if (MILVUS_GPU_VERSION OR KNOWHERE_GPU_VERSION)
message(STATUS "Building Knowhere GPU version")
add_compile_definitions("MILVUS_GPU_VERSION")
set(KNOWHERE_GPU_VERSION true)
enable_language(CUDA)
find_package(CUDA 10 REQUIRED)
set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -Xcompiler -fPIC -std=c++11 -D_FORCE_INLINES --expt-extended-lambda")
else ()
message(STATUS "Building Knowhere CPU version")
set(KNOWHERE_CPU_VERSION true)
add_compile_definitions("MILVUS_CPU_VERSION")
endif ()
include(ThirdPartyPackagesCore)

6
core/src/index/build.sh
View File

@ -5,7 +5,7 @@ BUILD_UNITTEST="OFF"
INSTALL_PREFIX=$(pwd)/cmake_build
MAKE_CLEAN="OFF"
PROFILING="OFF"
BUILD_FAISS_WITH_MKL="OFF"
FAISS_WITH_MKL="OFF"
USE_JFROG_CACHE="OFF"
while getopts "p:d:t:uhrcgmj" arg
@ -31,7 +31,7 @@ do
PROFILING="ON"
;;
m)
BUILD_FAISS_WITH_MKL="ON"
FAISS_WITH_MKL="ON"
;;
j)
USE_JFROG_CACHE="ON"
@ -74,7 +74,7 @@ if [[ ${MAKE_CLEAN} == "ON" ]]; then
-DCMAKE_BUILD_TYPE=${BUILD_TYPE} \
-DCMAKE_CUDA_COMPILER=${CUDA_COMPILER} \
-DMILVUS_ENABLE_PROFILING=${PROFILING} \
-DBUILD_FAISS_WITH_MKL=${BUILD_FAISS_WITH_MKL} \
-DFAISS_WITH_MKL=${FAISS_WITH_MKL} \
-DUSE_JFROG_CACHE=${USE_JFROG_CACHE} \
../"
echo ${CMAKE_CMD}

23
core/src/index/cmake/DefineOptionsCore.cmake
View File

@ -41,18 +41,18 @@ macro(define_option_string name description default)
endmacro()
#----------------------------------------------------------------------
set_option_category("CPU version")
set_option_category("GPU version")
if (MILVUS_CPU_VERSION)
define_option(KNOWHERE_CPU_VERSION "Build CPU version only" ON)
if (MILVUS_GPU_VERSION)
define_option(KNOWHERE_GPU_VERSION "Build GPU version" ON)
else ()
define_option(KNOWHERE_CPU_VERSION "Build CPU version only" OFF)
define_option(KNOWHERE_GPU_VERSION "Build GPU version" OFF)
endif ()
#----------------------------------------------------------------------
set_option_category("Thirdparty")
set(KNOWHERE_DEPENDENCY_SOURCE_DEFAULT "AUTO")
set(KNOWHERE_DEPENDENCY_SOURCE_DEFAULT "BUNDLED")
define_option_string(KNOWHERE_DEPENDENCY_SOURCE
"Method to use for acquiring KNOWHERE's build dependencies"
@ -79,18 +79,7 @@ define_option(KNOWHERE_WITH_FAISS "Build with FAISS library" ON)
define_option(KNOWHERE_WITH_FAISS_GPU_VERSION "Build with FAISS GPU version" ON)
define_option(BUILD_FAISS_WITH_MKL "Build FAISS with MKL" OFF)
#----------------------------------------------------------------------
if (MSVC)
set_option_category("MSVC")
define_option(MSVC_LINK_VERBOSE
"Pass verbose linking options when linking libraries and executables"
OFF)
define_option(KNOWHERE_USE_STATIC_CRT "Build KNOWHERE with statically linked CRT" OFF)
endif ()
define_option(FAISS_WITH_MKL "Build FAISS with MKL" OFF)
#----------------------------------------------------------------------
set_option_category("Test and benchmark")

431
core/src/index/cmake/FindArrow.cmake Normal file
View File

@ -0,0 +1,431 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
# - Find Arrow (arrow/api.h, libarrow.a, libarrow.so)
# This module defines
# ARROW_FOUND, whether Arrow has been found
# ARROW_FULL_SO_VERSION, full shared object version of found Arrow "100.0.0"
# ARROW_IMPORT_LIB, path to libarrow's import library (Windows only)
# ARROW_INCLUDE_DIR, directory containing headers
# ARROW_LIBS, deprecated. Use ARROW_LIB_DIR instead
# ARROW_LIB_DIR, directory containing Arrow libraries
# ARROW_SHARED_IMP_LIB, deprecated. Use ARROW_IMPORT_LIB instead
# ARROW_SHARED_LIB, path to libarrow's shared library
# ARROW_SO_VERSION, shared object version of found Arrow such as "100"
# ARROW_STATIC_LIB, path to libarrow.a
# ARROW_VERSION, version of found Arrow
# ARROW_VERSION_MAJOR, major version of found Arrow
# ARROW_VERSION_MINOR, minor version of found Arrow
# ARROW_VERSION_PATCH, patch version of found Arrow
include(FindPkgConfig)
include(FindPackageHandleStandardArgs)
set(ARROW_SEARCH_LIB_PATH_SUFFIXES)
if(CMAKE_LIBRARY_ARCHITECTURE)
list(APPEND ARROW_SEARCH_LIB_PATH_SUFFIXES "lib/${CMAKE_LIBRARY_ARCHITECTURE}")
endif()
list(APPEND ARROW_SEARCH_LIB_PATH_SUFFIXES
"lib64"
"lib32"
"lib"
"bin")
set(ARROW_CONFIG_SUFFIXES
"_RELEASE"
"_RELWITHDEBINFO"
"_MINSIZEREL"
"_DEBUG"
"")
if(CMAKE_BUILD_TYPE)
string(TOUPPER ${CMAKE_BUILD_TYPE} ARROW_CONFIG_SUFFIX_PREFERRED)
set(ARROW_CONFIG_SUFFIX_PREFERRED "_${ARROW_CONFIG_SUFFIX_PREFERRED}")
list(INSERT ARROW_CONFIG_SUFFIXES 0 "${ARROW_CONFIG_SUFFIX_PREFERRED}")
endif()
if(NOT DEFINED ARROW_MSVC_STATIC_LIB_SUFFIX)
if(MSVC)
set(ARROW_MSVC_STATIC_LIB_SUFFIX "_static")
else()
set(ARROW_MSVC_STATIC_LIB_SUFFIX "")
endif()
endif()
# Internal function.
#
# Set shared library name for ${base_name} to ${output_variable}.
#
# Example:
# arrow_build_shared_library_name(ARROW_SHARED_LIBRARY_NAME arrow)
# # -> ARROW_SHARED_LIBRARY_NAME=libarrow.so on Linux
# # -> ARROW_SHARED_LIBRARY_NAME=libarrow.dylib on macOS
# # -> ARROW_SHARED_LIBRARY_NAME=arrow.dll with MSVC on Windows
# # -> ARROW_SHARED_LIBRARY_NAME=libarrow.dll with MinGW on Windows
function(arrow_build_shared_library_name output_variable base_name)
set(${output_variable}
"${CMAKE_SHARED_LIBRARY_PREFIX}${base_name}${CMAKE_SHARED_LIBRARY_SUFFIX}"
PARENT_SCOPE)
endfunction()
# Internal function.
#
# Set import library name for ${base_name} to ${output_variable}.
# This is useful only for MSVC build. Import library is used only
# with MSVC build.
#
# Example:
# arrow_build_import_library_name(ARROW_IMPORT_LIBRARY_NAME arrow)
# # -> ARROW_IMPORT_LIBRARY_NAME=arrow on Linux (meaningless)
# # -> ARROW_IMPORT_LIBRARY_NAME=arrow on macOS (meaningless)
# # -> ARROW_IMPORT_LIBRARY_NAME=arrow.lib with MSVC on Windows
# # -> ARROW_IMPORT_LIBRARY_NAME=libarrow.dll.a with MinGW on Windows
function(arrow_build_import_library_name output_variable base_name)
set(${output_variable}
"${CMAKE_IMPORT_LIBRARY_PREFIX}${base_name}${CMAKE_IMPORT_LIBRARY_SUFFIX}"
PARENT_SCOPE)
endfunction()
# Internal function.
#
# Set static library name for ${base_name} to ${output_variable}.
#
# Example:
# arrow_build_static_library_name(ARROW_STATIC_LIBRARY_NAME arrow)
# # -> ARROW_STATIC_LIBRARY_NAME=libarrow.a on Linux
# # -> ARROW_STATIC_LIBRARY_NAME=libarrow.a on macOS
# # -> ARROW_STATIC_LIBRARY_NAME=arrow.lib with MSVC on Windows
# # -> ARROW_STATIC_LIBRARY_NAME=libarrow.dll.a with MinGW on Windows
function(arrow_build_static_library_name output_variable base_name)
set(
${output_variable}
"${CMAKE_STATIC_LIBRARY_PREFIX}${base_name}${ARROW_MSVC_STATIC_LIB_SUFFIX}${CMAKE_STATIC_LIBRARY_SUFFIX}"
PARENT_SCOPE)
endfunction()
# Internal function.
#
# Set macro value for ${macro_name} in ${header_content} to ${output_variable}.
#
# Example:
# arrow_extract_macro_value(version_major
# "ARROW_VERSION_MAJOR"
# "#define ARROW_VERSION_MAJOR 1.0.0")
# # -> version_major=1.0.0
function(arrow_extract_macro_value output_variable macro_name header_content)
string(REGEX MATCH "#define +${macro_name} +[^\r\n]+" macro_definition
"${header_content}")
string(REGEX
REPLACE "^#define +${macro_name} +(.+)$" "\\1" macro_value "${macro_definition}")
set(${output_variable} "${macro_value}" PARENT_SCOPE)
endfunction()
# Internal macro only for arrow_find_package.
#
# Find package in HOME.
macro(arrow_find_package_home)
find_path(${prefix}_include_dir "${header_path}"
PATHS "${home}"
PATH_SUFFIXES "include"
NO_DEFAULT_PATH)
set(include_dir "${${prefix}_include_dir}")
set(${prefix}_INCLUDE_DIR "${include_dir}" PARENT_SCOPE)
if(MSVC)
set(CMAKE_SHARED_LIBRARY_SUFFIXES_ORIGINAL ${CMAKE_FIND_LIBRARY_SUFFIXES})
# .dll isn't found by find_library with MSVC because .dll isn't included in
# CMAKE_FIND_LIBRARY_SUFFIXES.
list(APPEND CMAKE_FIND_LIBRARY_SUFFIXES "${CMAKE_SHARED_LIBRARY_SUFFIX}")
endif()
find_library(${prefix}_shared_lib
NAMES "${shared_lib_name}"
PATHS "${home}"
PATH_SUFFIXES ${ARROW_SEARCH_LIB_PATH_SUFFIXES}
NO_DEFAULT_PATH)
if(MSVC)
set(CMAKE_SHARED_LIBRARY_SUFFIXES ${CMAKE_FIND_LIBRARY_SUFFIXES_ORIGINAL})
endif()
set(shared_lib "${${prefix}_shared_lib}")
set(${prefix}_SHARED_LIB "${shared_lib}" PARENT_SCOPE)
if(shared_lib)
add_library(${target_shared} SHARED IMPORTED)
set_target_properties(${target_shared} PROPERTIES IMPORTED_LOCATION "${shared_lib}")
if(include_dir)
set_target_properties(${target_shared}
PROPERTIES INTERFACE_INCLUDE_DIRECTORIES "${include_dir}")
endif()
find_library(${prefix}_import_lib
NAMES "${import_lib_name}"
PATHS "${home}"
PATH_SUFFIXES ${ARROW_SEARCH_LIB_PATH_SUFFIXES}
NO_DEFAULT_PATH)
set(import_lib "${${prefix}_import_lib}")
set(${prefix}_IMPORT_LIB "${import_lib}" PARENT_SCOPE)
if(import_lib)
set_target_properties(${target_shared} PROPERTIES IMPORTED_IMPLIB "${import_lib}")
endif()
endif()
find_library(${prefix}_static_lib
NAMES "${static_lib_name}"
PATHS "${home}"
PATH_SUFFIXES ${ARROW_SEARCH_LIB_PATH_SUFFIXES}
NO_DEFAULT_PATH)
set(static_lib "${${prefix}_static_lib}")
set(${prefix}_STATIC_LIB "${static_lib}" PARENT_SCOPE)
if(static_lib)
add_library(${target_static} STATIC IMPORTED)
set_target_properties(${target_static} PROPERTIES IMPORTED_LOCATION "${static_lib}")
if(include_dir)
set_target_properties(${target_static}
PROPERTIES INTERFACE_INCLUDE_DIRECTORIES "${include_dir}")
endif()
endif()
endmacro()
# Internal macro only for arrow_find_package.
#
# Find package by CMake package configuration.
macro(arrow_find_package_cmake_package_configuration)
# ARROW-5575: We need to split target files for each component
if(TARGET ${target_shared} OR TARGET ${target_static})
set(${cmake_package_name}_FOUND TRUE)
else()
find_package(${cmake_package_name} CONFIG)
endif()
if(${cmake_package_name}_FOUND)
set(${prefix}_USE_CMAKE_PACKAGE_CONFIG TRUE PARENT_SCOPE)
if(TARGET ${target_shared})
foreach(suffix ${ARROW_CONFIG_SUFFIXES})
get_target_property(shared_lib ${target_shared} IMPORTED_LOCATION${suffix})
if(shared_lib)
# Remove shared library version:
# libarrow.so.100.0.0 -> libarrow.so
# Because ARROW_HOME and pkg-config approaches don't add
# shared library version.
string(REGEX
REPLACE "(${CMAKE_SHARED_LIBRARY_SUFFIX})[.0-9]+$" "\\1" shared_lib
"${shared_lib}")
set(${prefix}_SHARED_LIB "${shared_lib}" PARENT_SCOPE)
break()
endif()
endforeach()
endif()
if(TARGET ${target_static})
foreach(suffix ${ARROW_CONFIG_SUFFIXES})
get_target_property(static_lib ${target_static} IMPORTED_LOCATION${suffix})
if(static_lib)
set(${prefix}_STATIC_LIB "${static_lib}" PARENT_SCOPE)
break()
endif()
endforeach()
endif()
endif()
endmacro()
# Internal macro only for arrow_find_package.
#
# Find package by pkg-config.
macro(arrow_find_package_pkg_config)
pkg_check_modules(${prefix}_PC ${pkg_config_name})
if(${prefix}_PC_FOUND)
set(${prefix}_USE_PKG_CONFIG TRUE PARENT_SCOPE)
set(include_dir "${${prefix}_PC_INCLUDEDIR}")
set(lib_dir "${${prefix}_PC_LIBDIR}")
set(shared_lib_paths "${${prefix}_PC_LINK_LIBRARIES}")
# Use the first shared library path as the IMPORTED_LOCATION
# for ${target_shared}. This assumes that the first shared library
# path is the shared library path for this module.
list(GET shared_lib_paths 0 first_shared_lib_path)
# Use the rest shared library paths as the INTERFACE_LINK_LIBRARIES
# for ${target_shared}. This assumes that the rest shared library
# paths are dependency library paths for this module.
list(LENGTH shared_lib_paths n_shared_lib_paths)
if(n_shared_lib_paths LESS_EQUAL 1)
set(rest_shared_lib_paths)
else()
list(SUBLIST
shared_lib_paths
1
-1
rest_shared_lib_paths)
endif()
set(${prefix}_VERSION "${${prefix}_PC_VERSION}" PARENT_SCOPE)
set(${prefix}_INCLUDE_DIR "${include_dir}" PARENT_SCOPE)
set(${prefix}_SHARED_LIB "${first_shared_lib_path}" PARENT_SCOPE)
add_library(${target_shared} SHARED IMPORTED)
set_target_properties(${target_shared}
PROPERTIES INTERFACE_INCLUDE_DIRECTORIES
"${include_dir}"
INTERFACE_LINK_LIBRARIES
"${rest_shared_lib_paths}"
IMPORTED_LOCATION
"${first_shared_lib_path}")
find_library(${prefix}_static_lib
NAMES "${static_lib_name}"
PATHS "${lib_dir}"
NO_DEFAULT_PATH)
set(static_lib "${${prefix}_static_lib}")
set(${prefix}_STATIC_LIB "${static_lib}" PARENT_SCOPE)
if(static_lib)
add_library(${target_static} STATIC IMPORTED)
set_target_properties(${target_static}
PROPERTIES INTERFACE_INCLUDE_DIRECTORIES "${include_dir}"
IMPORTED_LOCATION "${static_lib}")
endif()
endif()
endmacro()
function(arrow_find_package
prefix
home
base_name
header_path
cmake_package_name
pkg_config_name)
arrow_build_shared_library_name(shared_lib_name ${base_name})
arrow_build_import_library_name(import_lib_name ${base_name})
arrow_build_static_library_name(static_lib_name ${base_name})
set(target_shared ${base_name}_shared)
set(target_static ${base_name}_static)
if(home)
arrow_find_package_home()
set(${prefix}_FIND_APPROACH "HOME: ${home}" PARENT_SCOPE)
else()
arrow_find_package_cmake_package_configuration()
if(${cmake_package_name}_FOUND)
set(${prefix}_FIND_APPROACH
"CMake package configuration: ${cmake_package_name}"
PARENT_SCOPE)
else()
arrow_find_package_pkg_config()
set(${prefix}_FIND_APPROACH "pkg-config: ${pkg_config_name}" PARENT_SCOPE)
endif()
endif()
if(NOT include_dir)
if(TARGET ${target_shared})
get_target_property(include_dir ${target_shared} INTERFACE_INCLUDE_DIRECTORIES)
elseif(TARGET ${target_static})
get_target_property(include_dir ${target_static} INTERFACE_INCLUDE_DIRECTORIES)
endif()
endif()
if(include_dir)
set(${prefix}_INCLUDE_DIR "${include_dir}" PARENT_SCOPE)
endif()
if(shared_lib)
get_filename_component(lib_dir "${shared_lib}" DIRECTORY)
elseif(static_lib)
get_filename_component(lib_dir "${static_lib}" DIRECTORY)
else()
set(lib_dir NOTFOUND)
endif()
set(${prefix}_LIB_DIR "${lib_dir}" PARENT_SCOPE)
# For backward compatibility
set(${prefix}_LIBS "${lib_dir}" PARENT_SCOPE)
endfunction()
if(NOT "$ENV{ARROW_HOME}" STREQUAL "")
file(TO_CMAKE_PATH "$ENV{ARROW_HOME}" ARROW_HOME)
endif()
arrow_find_package(ARROW
"${ARROW_HOME}"
arrow
arrow/api.h
Arrow
arrow)
if(ARROW_HOME)
if(ARROW_INCLUDE_DIR)
file(READ "${ARROW_INCLUDE_DIR}/arrow/util/config.h" ARROW_CONFIG_H_CONTENT)
arrow_extract_macro_value(ARROW_VERSION_MAJOR "ARROW_VERSION_MAJOR"
"${ARROW_CONFIG_H_CONTENT}")
arrow_extract_macro_value(ARROW_VERSION_MINOR "ARROW_VERSION_MINOR"
"${ARROW_CONFIG_H_CONTENT}")
arrow_extract_macro_value(ARROW_VERSION_PATCH "ARROW_VERSION_PATCH"
"${ARROW_CONFIG_H_CONTENT}")
if("${ARROW_VERSION_MAJOR}" STREQUAL ""
OR "${ARROW_VERSION_MINOR}" STREQUAL ""
OR "${ARROW_VERSION_PATCH}" STREQUAL "")
set(ARROW_VERSION "0.0.0")
else()
set(ARROW_VERSION
"${ARROW_VERSION_MAJOR}.${ARROW_VERSION_MINOR}.${ARROW_VERSION_PATCH}")
endif()
arrow_extract_macro_value(ARROW_SO_VERSION_QUOTED "ARROW_SO_VERSION"
"${ARROW_CONFIG_H_CONTENT}")
string(REGEX REPLACE "^\"(.+)\"$" "\\1" ARROW_SO_VERSION "${ARROW_SO_VERSION_QUOTED}")
arrow_extract_macro_value(ARROW_FULL_SO_VERSION_QUOTED "ARROW_FULL_SO_VERSION"
"${ARROW_CONFIG_H_CONTENT}")
string(REGEX
REPLACE "^\"(.+)\"$" "\\1" ARROW_FULL_SO_VERSION
"${ARROW_FULL_SO_VERSION_QUOTED}")
endif()
else()
if(ARROW_USE_CMAKE_PACKAGE_CONFIG)
find_package(Arrow CONFIG)
elseif(ARROW_USE_PKG_CONFIG)
pkg_get_variable(ARROW_SO_VERSION arrow so_version)
pkg_get_variable(ARROW_FULL_SO_VERSION arrow full_so_version)
endif()
endif()
set(ARROW_ABI_VERSION ${ARROW_SO_VERSION})
mark_as_advanced(ARROW_ABI_VERSION
ARROW_CONFIG_SUFFIXES
ARROW_FULL_SO_VERSION
ARROW_IMPORT_LIB
ARROW_INCLUDE_DIR
ARROW_LIBS
ARROW_LIB_DIR
ARROW_SEARCH_LIB_PATH_SUFFIXES
ARROW_SHARED_IMP_LIB
ARROW_SHARED_LIB
ARROW_SO_VERSION
ARROW_STATIC_LIB
ARROW_VERSION
ARROW_VERSION_MAJOR
ARROW_VERSION_MINOR
ARROW_VERSION_PATCH)
find_package_handle_standard_args(Arrow REQUIRED_VARS
# The first required variable is shown
# in the found message. So this list is
# not sorted alphabetically.
ARROW_INCLUDE_DIR
ARROW_LIB_DIR
ARROW_FULL_SO_VERSION
ARROW_SO_VERSION
VERSION_VAR
ARROW_VERSION)
set(ARROW_FOUND ${Arrow_FOUND})
if(Arrow_FOUND AND NOT Arrow_FIND_QUIETLY)
message(STATUS "Arrow version: ${ARROW_VERSION} (${ARROW_FIND_APPROACH})")
message(STATUS "Arrow SO and ABI version: ${ARROW_SO_VERSION}")
message(STATUS "Arrow full SO version: ${ARROW_FULL_SO_VERSION}")
message(STATUS "Found the Arrow core shared library: ${ARROW_SHARED_LIB}")
message(STATUS "Found the Arrow core import library: ${ARROW_IMPORT_LIB}")
message(STATUS "Found the Arrow core static library: ${ARROW_STATIC_LIB}")
endif()

44
core/src/index/cmake/FindFAISS.cmake Normal file
View File

@ -0,0 +1,44 @@
set(FAISS_STATIC_LIB_NAME ${CMAKE_STATIC_LIBRARY_PREFIX}faiss${CMAKE_STATIC_LIBRARY_SUFFIX})
# First, find via if specified FAISS_ROOT
if (FAISS_ROOT)
find_library(FAISS_STATIC_LIB
NAMES ${FAISS_STATIC_LIB_NAME}
PATHS ${FAISS_ROOT}
PATH_SUFFIXES "lib"
NO_DEFAULT_PATH
)
find_path(FAISS_INCLUDE_DIR
NAMES "faiss/Index.h"
PATHS ${FAISS_ROOT}
NO_DEFAULT_PATH
PATH_SUFFIXES "include"
)
endif ()
find_package_handle_standard_args(FAISS REQUIRED_VARS FAISS_STATIC_LIB FAISS_INCLUDE_DIR)
if (FAISS_FOUND)
if (NOT TARGET faiss)
add_library(faiss STATIC IMPORTED)
set_target_properties(
faiss
PROPERTIES
IMPORTED_LOCATION "${FAISS_STATIC_LIB}"
INTERFACE_INCLUDE_DIRECTORIES "${FAISS_INCLUDE_DIR}"
)
if (FAISS_WITH_MKL)
set_target_properties(
faiss
PROPERTIES
INTERFACE_LINK_LIBRARIES "${MKL_LIBS}")
else ()
set_target_properties(
faiss
PROPERTIES
INTERFACE_LINK_LIBRARIES "openblas;lapack")
endif ()
endif ()
endif ()

143
core/src/index/cmake/ThirdPartyPackagesCore.cmake
View File

@ -16,11 +16,12 @@
set(KNOWHERE_THIRDPARTY_DEPENDENCIES
ARROW
Arrow
FAISS
GTest
LAPACK
OpenBLAS
MKL
)
message(STATUS "Using ${KNOWHERE_DEPENDENCY_SOURCE} approach to find dependencies")
@ -33,7 +34,7 @@ foreach (DEPENDENCY ${KNOWHERE_THIRDPARTY_DEPENDENCIES})
endforeach ()
macro(build_dependency DEPENDENCY_NAME)
if ("${DEPENDENCY_NAME}" STREQUAL "ARROW")
if ("${DEPENDENCY_NAME}" STREQUAL "Arrow")
build_arrow()
elseif ("${DEPENDENCY_NAME}" STREQUAL "LAPACK")
build_lapack()
@ -43,6 +44,8 @@ macro(build_dependency DEPENDENCY_NAME)
build_openblas()
elseif ("${DEPENDENCY_NAME}" STREQUAL "FAISS")
build_faiss()
elseif ("${DEPENDENCY_NAME}" STREQUAL "MKL")
build_mkl()
else ()
message(FATAL_ERROR "Unknown thirdparty dependency to build: ${DEPENDENCY_NAME}")
endif ()
@ -50,9 +53,10 @@ endmacro()
macro(resolve_dependency DEPENDENCY_NAME)
if (${DEPENDENCY_NAME}_SOURCE STREQUAL "AUTO")
#message(STATUS "Finding ${DEPENDENCY_NAME} package")
#message(STATUS "${DEPENDENCY_NAME} package not found")
build_dependency(${DEPENDENCY_NAME})
find_package(${DEPENDENCY_NAME} MODULE)
if (NOT ${${DEPENDENCY_NAME}_FOUND})
build_dependency(${DEPENDENCY_NAME})
endif ()
elseif (${DEPENDENCY_NAME}_SOURCE STREQUAL "BUNDLED")
build_dependency(${DEPENDENCY_NAME})
elseif (${DEPENDENCY_NAME}_SOURCE STREQUAL "SYSTEM")
@ -131,17 +135,6 @@ if (USE_JFROG_CACHE STREQUAL "ON")
endif ()
endif ()
macro(resolve_dependency DEPENDENCY_NAME)
if (${DEPENDENCY_NAME}_SOURCE STREQUAL "AUTO")
#disable find_package for now
build_dependency(${DEPENDENCY_NAME})
elseif (${DEPENDENCY_NAME}_SOURCE STREQUAL "BUNDLED")
build_dependency(${DEPENDENCY_NAME})
elseif (${DEPENDENCY_NAME}_SOURCE STREQUAL "SYSTEM")
find_package(${DEPENDENCY_NAME} REQUIRED)
endif ()
endmacro()
# ----------------------------------------------------------------------
# ExternalProject options
@ -232,28 +225,12 @@ foreach (_VERSION_ENTRY ${TOOLCHAIN_VERSIONS_TXT})
set(${_LIB_NAME} "${_LIB_VERSION}")
endforeach ()
if (CUSTOMIZATION)
execute_process(COMMAND wget -q --method HEAD ${FAISS_URL} RESULT_VARIABLE return_code)
message(STATUS "Check the remote cache file ${FAISS_URL}. return code = ${return_code}")
if (NOT return_code EQUAL 0)
MESSAGE(FATAL_ERROR "Can't access to ${FAISS_URL}")
else ()
set(FAISS_SOURCE_URL ${FAISS_URL})
# set(FAISS_MD5 "a589663865a8558205533c8ac414278c")
# set(FAISS_MD5 "57da9c4f599cc8fa4260488b1c96e1cc") # commit-id 6dbdf75987c34a2c853bd172ea0d384feea8358c branch-0.2.0
# set(FAISS_MD5 "21deb1c708490ca40ecb899122c01403") # commit-id 643e48f479637fd947e7b93fa4ca72b38ecc9a39 branch-0.2.0
# set(FAISS_MD5 "072db398351cca6e88f52d743bbb9fa0") # commit-id 3a2344d04744166af41ef1a74449d68a315bfe17 branch-0.2.1
# set(FAISS_MD5 "c89ea8e655f5cdf58f42486f13614714") # commit-id 9c28a1cbb88f41fa03b03d7204106201ad33276b branch-0.2.1
# set(FAISS_MD5 "87fdd86351ffcaf3f80dc26ade63c44b") # commit-id 841a156e67e8e22cd8088e1b58c00afbf2efc30b branch-0.2.1
# set(FAISS_MD5 "f3b2ce3364c3fa7febd3aa7fdd0fe380") # commit-id 694e03458e6b69ce8a62502f71f69a614af5af8f branch-0.3.0
# set(FAISS_MD5 "bb30722c22390ce5f6759ccb216c1b2a") # commit-id d324db297475286afe107847c7fb7a0f9dc7e90e branch-0.3.0
set(FAISS_MD5 "2293cdb209c3718e3b19f3edae8b32b3") # commit-id a13c1205dc52977a9ad3b33a14efa958604a8bff branch-0.3.0
endif()
else()
set(FAISS_SOURCE_URL "https://github.com/JinHai-CN/faiss/archive/1.6.0.tar.gz")
if (DEFINED ENV{FAISS_SOURCE_URL})
set(FAISS_SOURCE_URL "$ENV{FAISS_SOURCE_URL}")
else ()
set(FAISS_SOURCE_URL "https://github.com/JinHai-CN/faiss/archive/${FAISS_VERSION}.tar.gz")
set(FAISS_MD5 "b02c1a53234f5acc9bea1b0c55524f50")
endif()
message(STATUS "FAISS URL = ${FAISS_SOURCE_URL}")
endif ()
if (DEFINED ENV{KNOWHERE_ARROW_URL})
set(ARROW_SOURCE_URL "$ENV{KNOWHERE_ARROW_URL}")
@ -296,6 +273,7 @@ macro(build_arrow)
set(ARROW_STATIC_LIB
"${ARROW_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}${ARROW_STATIC_LIB_NAME}${CMAKE_STATIC_LIBRARY_SUFFIX}"
)
set(ARROW_LIB_DIR "${ARROW_PREFIX}/lib")
set(ARROW_INCLUDE_DIR "${ARROW_PREFIX}/include")
set(ARROW_CMAKE_ARGS
@ -392,7 +370,7 @@ macro(build_arrow)
)
endif ()
file(MAKE_DIRECTORY "${ARROW_PREFIX}/include")
file(MAKE_DIRECTORY "${ARROW_INCLUDE_DIR}")
add_library(arrow STATIC IMPORTED)
set_target_properties(arrow
PROPERTIES IMPORTED_LOCATION "${ARROW_STATIC_LIB}"
@ -402,17 +380,17 @@ macro(build_arrow)
set(JEMALLOC_PREFIX "${INDEX_BINARY_DIR}/arrow_ep-prefix/src/arrow_ep-build/jemalloc_ep-prefix/src/jemalloc_ep")
add_custom_command(TARGET arrow_ep POST_BUILD
COMMAND ${CMAKE_COMMAND} -E make_directory ${ARROW_PREFIX}/lib/
COMMAND ${CMAKE_COMMAND} -E copy ${JEMALLOC_PREFIX}/lib/libjemalloc_pic.a ${ARROW_PREFIX}/lib/
COMMAND ${CMAKE_COMMAND} -E make_directory ${ARROW_LIB_DIR}
COMMAND ${CMAKE_COMMAND} -E copy ${JEMALLOC_PREFIX}/lib/libjemalloc_pic.a ${ARROW_LIB_DIR}
DEPENDS ${JEMALLOC_PREFIX}/lib/libjemalloc_pic.a)
endmacro()
if (KNOWHERE_WITH_ARROW AND NOT TARGET arrow_ep)
resolve_dependency(ARROW)
resolve_dependency(Arrow)
link_directories(SYSTEM ${ARROW_PREFIX}/lib/)
link_directories(SYSTEM ${ARROW_LIB_DIR})
include_directories(SYSTEM ${ARROW_INCLUDE_DIR})
endif ()
@ -682,28 +660,46 @@ if (KNOWHERE_BUILD_TESTS AND NOT TARGET googletest_ep)
include_directories(SYSTEM ${GTEST_INCLUDE_DIR})
endif ()
# ----------------------------------------------------------------------
# MKL
macro(build_mkl)
if (FAISS_WITH_MKL)
if (EXISTS "/proc/cpuinfo")
FILE(READ /proc/cpuinfo PROC_CPUINFO)
SET(VENDOR_ID_RX "vendor_id[ \t]*:[ \t]*([a-zA-Z]+)\n")
STRING(REGEX MATCH "${VENDOR_ID_RX}" VENDOR_ID "${PROC_CPUINFO}")
STRING(REGEX REPLACE "${VENDOR_ID_RX}" "\\1" VENDOR_ID "${VENDOR_ID}")
if (NOT ${VENDOR_ID} STREQUAL "GenuineIntel")
set(FAISS_WITH_MKL OFF)
endif ()
endif ()
find_path(MKL_LIB_PATH
NAMES "libmkl_intel_ilp64.a" "libmkl_gnu_thread.a" "libmkl_core.a"
PATH_SUFFIXES "intel/compilers_and_libraries_${MKL_VERSION}/linux/mkl/lib/intel64/")
if (${MKL_LIB_PATH} STREQUAL "MKL_LIB_PATH-NOTFOUND")
message(FATAL_ERROR "Could not find MKL libraries")
endif ()
message(STATUS "MKL lib path = ${MKL_LIB_PATH}")
set(MKL_LIBS
${MKL_LIB_PATH}/libmkl_intel_ilp64.a
${MKL_LIB_PATH}/libmkl_gnu_thread.a
${MKL_LIB_PATH}/libmkl_core.a
)
endif ()
endmacro()
# ----------------------------------------------------------------------
# FAISS
macro(build_faiss)
message(STATUS "Building FAISS-${FAISS_VERSION} from source")
if (NOT DEFINED BUILD_FAISS_WITH_MKL)
set(BUILD_FAISS_WITH_MKL OFF)
endif ()
if (EXISTS "/proc/cpuinfo")
FILE(READ /proc/cpuinfo PROC_CPUINFO)
SET(VENDOR_ID_RX "vendor_id[ \t]*:[ \t]*([a-zA-Z]+)\n")
STRING(REGEX MATCH "${VENDOR_ID_RX}" VENDOR_ID "${PROC_CPUINFO}")
STRING(REGEX REPLACE "${VENDOR_ID_RX}" "\\1" VENDOR_ID "${VENDOR_ID}")
if (NOT ${VENDOR_ID} STREQUAL "GenuineIntel")
set(BUILD_FAISS_WITH_MKL OFF)
endif ()
endif ()
set(FAISS_PREFIX "${INDEX_BINARY_DIR}/faiss_ep-prefix/src/faiss_ep")
set(FAISS_INCLUDE_DIR "${FAISS_PREFIX}/include")
set(FAISS_STATIC_LIB
@ -715,30 +711,11 @@ macro(build_faiss)
"CXXFLAGS=${EP_CXX_FLAGS}"
--without-python)
set(FAISS_CFLAGS ${EP_C_FLAGS})
set(FAISS_CXXFLAGS ${EP_CXX_FLAGS})
if (BUILD_FAISS_WITH_MKL)
find_path(MKL_LIB_PATH
NAMES "libmkl_intel_ilp64.a" "libmkl_gnu_thread.a" "libmkl_core.a"
PATH_SUFFIXES "intel/compilers_and_libraries_${MKL_VERSION}/linux/mkl/lib/intel64/")
if (${MKL_LIB_PATH} STREQUAL "MKL_LIB_PATH-NOTFOUND")
message(FATAL_ERROR "Could not find MKL libraries")
endif ()
message(STATUS "Build Faiss with MKL. MKL lib path = ${MKL_LIB_PATH}")
set(MKL_LIBS
${MKL_LIB_PATH}/libmkl_intel_ilp64.a
${MKL_LIB_PATH}/libmkl_gnu_thread.a
${MKL_LIB_PATH}/libmkl_core.a
)
if (FAISS_WITH_MKL)
set(FAISS_CONFIGURE_ARGS ${FAISS_CONFIGURE_ARGS}
"CPPFLAGS=-DFINTEGER=long -DMKL_ILP64 -m64 -I${MKL_LIB_PATH}/../../include"
"LDFLAGS=-L${MKL_LIB_PATH}"
)
else ()
message(STATUS "Build Faiss with OpenBlas/LAPACK")
set(FAISS_CONFIGURE_ARGS ${FAISS_CONFIGURE_ARGS}
@ -779,7 +756,7 @@ macro(build_faiss)
BUILD_BYPRODUCTS
${FAISS_STATIC_LIB})
if (NOT BUILD_FAISS_WITH_MKL)
if (NOT FAISS_WITH_MKL)
ExternalProject_Add_StepDependencies(faiss_ep build openblas_ep lapack_ep)
endif ()
@ -809,7 +786,7 @@ macro(build_faiss)
BUILD_BYPRODUCTS
${FAISS_STATIC_LIB})
if (NOT BUILD_FAISS_WITH_MKL)
if (NOT FAISS_WITH_MKL)
ExternalProject_Add_StepDependencies(faiss_ep build openblas_ep lapack_ep)
endif ()
@ -824,7 +801,7 @@ macro(build_faiss)
IMPORTED_LOCATION "${FAISS_STATIC_LIB}"
INTERFACE_INCLUDE_DIRECTORIES "${FAISS_INCLUDE_DIR}"
)
if (BUILD_FAISS_WITH_MKL)
if (FAISS_WITH_MKL)
set_target_properties(
faiss
PROPERTIES
@ -843,7 +820,9 @@ endmacro()
if (KNOWHERE_WITH_FAISS AND NOT TARGET faiss_ep)
if (NOT BUILD_FAISS_WITH_MKL)
if (FAISS_WITH_MKL)
resolve_dependency(MKL)
else ()
resolve_dependency(OpenBLAS)
get_target_property(OPENBLAS_INCLUDE_DIR openblas INTERFACE_INCLUDE_DIRECTORIES)
include_directories(SYSTEM "${OPENBLAS_INCLUDE_DIR}")

4
core/src/index/knowhere/CMakeLists.txt
View File

@ -49,12 +49,12 @@ set(depend_libs
SPTAGLibStatic
faiss
arrow
${ARROW_PREFIX}/lib/libjemalloc_pic.a
${ARROW_LIB_DIR}/libjemalloc_pic.a
gomp
gfortran
pthread
)
if (BUILD_FAISS_WITH_MKL)
if (FAISS_WITH_MKL)
set(depend_libs ${depend_libs}
"-Wl,--start-group \
${MKL_LIB_PATH}/libmkl_intel_ilp64.a \

2
core/src/index/knowhere/knowhere/common/Log.h
View File

@ -17,7 +17,7 @@
#pragma once
#include "external/easyloggingpp/easylogging++.h"
#include "easyloggingpp/easylogging++.h"
namespace knowhere {

8
core/src/index/knowhere/knowhere/index/vector_index/IndexIVFPQ.cpp
View File

@ -17,15 +17,19 @@
#include <faiss/IndexFlat.h>
#include <faiss/IndexIVFPQ.h>
#ifdef MILVUS_GPU_VERSION
#include <faiss/gpu/GpuCloner.h>
#endif
#include <memory>
#include <utility>
#include "knowhere/adapter/VectorAdapter.h"
#include "knowhere/common/Exception.h"
#ifdef MILVUS_GPU_VERSION
#include "knowhere/index/vector_index/IndexGPUIVF.h"
#include "knowhere/index/vector_index/IndexGPUIVFPQ.h"
#endif
#include "knowhere/index/vector_index/IndexIVFPQ.h"
namespace knowhere {
@ -66,6 +70,7 @@ IVFPQ::Clone_impl(const std::shared_ptr<faiss::Index>& index) {
VectorIndexPtr
IVFPQ::CopyCpuToGpu(const int64_t& device_id, const Config& config) {
#ifdef MILVUS_GPU_VERSION
if (auto res = FaissGpuResourceMgr::GetInstance().GetRes(device_id)) {
ResScope rs(res, device_id, false);
auto gpu_index = faiss::gpu::index_cpu_to_gpu(res->faiss_res.get(), device_id, index_.get());
@ -76,6 +81,9 @@ IVFPQ::CopyCpuToGpu(const int64_t& device_id, const Config& config) {
} else {
KNOWHERE_THROW_MSG("CopyCpuToGpu Error, can't get gpu_resource");
}
#else
KNOWHERE_THROW_MSG("Calling IVFPQ::CopyCpuToGpu when we are using CPU version");
#endif
}
} // namespace knowhere

5
core/src/index/thirdparty/versions.txt vendored
View File

@ -1,6 +1,7 @@
ARROW_VERSION=apache-arrow-0.14.0
ARROW_VERSION=apache-arrow-0.15.1
BOOST_VERSION=1.70.0
GTEST_VERSION=1.8.1
LAPACK_VERSION=v3.8.0
OPENBLAS_VERSION=v0.3.6
FAISS_VERSION=branch-0.3.0
FAISS_VERSION=1.6.0
MKL_VERSION=2019.5.281

6
core/src/index/unittest/CMakeLists.txt
View File

@ -6,9 +6,9 @@ include_directories(${INDEX_SOURCE_DIR})
set(depend_libs
gtest gmock gtest_main gmock_main
faiss
arrow "${ARROW_PREFIX}/lib/libjemalloc_pic.a"
arrow "${ARROW_LIB_DIR}/libjemalloc_pic.a"
)
if (BUILD_FAISS_WITH_MKL)
if (FAISS_WITH_MKL)
set(depend_libs ${depend_libs}
"-Wl,--start-group \
${MKL_LIB_PATH}/libmkl_intel_ilp64.a \
@ -27,7 +27,7 @@ set(basic_libs
)
set(util_srcs
${MILVUS_ENGINE_SRC}/external/easyloggingpp/easylogging++.cc
${MILVUS_THIRDPARTY_SRC}/easyloggingpp/easylogging++.cc
${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_index/helpers/FaissIO.cpp
${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_index/helpers/IndexParameter.cpp
${INDEX_SOURCE_DIR}/knowhere/knowhere/adapter/Structure.cpp

4
core/src/index/unittest/faiss_benchmark/CMakeLists.txt
View File

@ -13,9 +13,9 @@ if (KNOWHERE_GPU_VERSION)
set(depend_libs
faiss hdf5
arrow ${ARROW_PREFIX}/lib/libjemalloc_pic.a
arrow ${ARROW_LIB_DIR}/libjemalloc_pic.a
)
if (BUILD_FAISS_WITH_MKL)
if (FAISS_WITH_MKL)
set(depend_libs ${depend_libs}
"-Wl,--start-group \
${MKL_LIB_PATH}/libmkl_intel_ilp64.a \

4
core/src/index/unittest/faiss_ori/CMakeLists.txt
View File

@ -8,9 +8,9 @@ if (KNOWHERE_GPU_VERSION)
set(depend_libs
faiss
arrow ${ARROW_PREFIX}/lib/libjemalloc_pic.a
arrow ${ARROW_LIB_DIR}/libjemalloc_pic.a
)
if (BUILD_FAISS_WITH_MKL)
if (FAISS_WITH_MKL)
set(depend_libs ${depend_libs}
"-Wl,--start-group \
${MKL_LIB_PATH}/libmkl_intel_ilp64.a \

2
core/src/index/unittest/test_nsg/CMakeLists.txt
View File

@ -5,7 +5,7 @@
add_definitions(-std=c++11 -O3 -lboost -march=native -Wall -DINFO)
find_package(OpenMP REQUIRED)
if (OPENMP_FOUND)
if (OpenMP_FOUND)
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${OpenMP_C_FLAGS}")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS}")
else ()

12
core/src/main.cpp
View File

@ -22,7 +22,7 @@
#include <cstring>
#include <string>
#include "external/easyloggingpp/easylogging++.h"
#include "easyloggingpp/easylogging++.h"
#include "metrics/Metrics.h"
#include "server/Server.h"
#include "src/config.h"
@ -51,7 +51,13 @@ print_banner() {
std::cout << " /_/ /_/___/____/___/\\____/___/ " << std::endl;
std::cout << std::endl;
std::cout << "Welcome to Milvus!" << std::endl;
std::cout << "Milvus " << BUILD_TYPE << " version: v" << MILVUS_VERSION << ", built at " << BUILD_TIME << std::endl;
std::cout << "Milvus " << BUILD_TYPE << " version: v" << MILVUS_VERSION << ", built at " << BUILD_TIME << ", with "
#ifdef WITH_MKL
<< "MKL"
#else
<< "OpenBLAS"
#endif
<< " library." << std::endl;
#ifdef MILVUS_CPU_VERSION
std::cout << "You are using Milvus CPU version" << std::endl;
#else
@ -137,7 +143,7 @@ main(int argc, char* argv[]) {
s = server.Start();
if (s.ok()) {
std::cout << "Milvus server start successfully." << std::endl;
std::cout << "Milvus server started successfully!" << std::endl;
} else {
goto FAIL;
}

16
core/src/scheduler/job/SearchJob.cpp
View File

@ -49,13 +49,21 @@ void
SearchJob::SearchDone(size_t index_id) {
std::unique_lock<std::mutex> lock(mutex_);
index_files_.erase(index_id);
cv_.notify_all();
if (index_files_.empty()) {
cv_.notify_all();
}
SERVER_LOG_DEBUG << "SearchJob " << id() << " finish index file: " << index_id;
}
ResultSet&
SearchJob::GetResult() {
return result_;
ResultIds&
SearchJob::GetResultIds() {
return result_ids_;
}
ResultDistances&
SearchJob::GetResultDistances() {
return result_distances_;
}
Status&

22
core/src/scheduler/job/SearchJob.h
View File

@ -29,6 +29,7 @@
#include <vector>
#include "Job.h"
#include "db/Types.h"
#include "db/meta/MetaTypes.h"
namespace milvus {
@ -37,9 +38,9 @@ namespace scheduler {
using engine::meta::TableFileSchemaPtr;
using Id2IndexMap = std::unordered_map<size_t, TableFileSchemaPtr>;
using IdDistPair = std::pair<int64_t, double>;
using Id2DistVec = std::vector<IdDistPair>;
using ResultSet = std::vector<Id2DistVec>;
using ResultIds = engine::ResultIds;
using ResultDistances = engine::ResultDistances;
class SearchJob : public Job {
public:
@ -55,8 +56,11 @@ class SearchJob : public Job {
void
SearchDone(size_t index_id);
ResultSet&
GetResult();
ResultIds&
GetResultIds();
ResultDistances&
GetResultDistances();
Status&
GetStatus();
@ -90,6 +94,11 @@ class SearchJob : public Job {
return index_files_;
}
std::mutex&
mutex() {
return mutex_;
}
private:
uint64_t topk_ = 0;
uint64_t nq_ = 0;
@ -99,7 +108,8 @@ class SearchJob : public Job {
Id2IndexMap index_files_;
// TODO: column-base better ?
ResultSet result_;
ResultIds result_ids_;
ResultDistances result_distances_;
Status status_;
std::mutex mutex_;

115
core/src/scheduler/task/SearchTask.cpp
View File

@ -219,8 +219,11 @@ XSearchTask::Execute() {
// step 3: pick up topk result
auto spec_k = index_engine_->Count() < topk ? index_engine_->Count() : topk;
XSearchTask::MergeTopkToResultSet(output_ids, output_distance, spec_k, nq, topk, metric_l2,
search_job->GetResult());
{
std::unique_lock<std::mutex> lock(search_job->mutex());
XSearchTask::MergeTopkToResultSet(output_ids, output_distance, spec_k, nq, topk, metric_l2,
search_job->GetResultIds(), search_job->GetResultDistances());
}
span = rc.RecordSection(hdr + ", reduce topk");
// search_job->AccumReduceCost(span);
@ -240,71 +243,69 @@ XSearchTask::Execute() {
}
void
XSearchTask::MergeTopkToResultSet(const std::vector<int64_t>& input_ids, const std::vector<float>& input_distance,
uint64_t input_k, uint64_t nq, uint64_t topk, bool ascending,
scheduler::ResultSet& result) {
if (result.empty()) {
result.resize(nq);
XSearchTask::MergeTopkToResultSet(const scheduler::ResultIds& src_ids, const scheduler::ResultDistances& src_distances,
size_t src_k, size_t nq, size_t topk, bool ascending, scheduler::ResultIds& tar_ids,
scheduler::ResultDistances& tar_distances) {
if (src_ids.empty()) {
return;
}
size_t tar_k = tar_ids.size() / nq;
size_t buf_k = std::min(topk, src_k + tar_k);
scheduler::ResultIds buf_ids(nq * buf_k, -1);
scheduler::ResultDistances buf_distances(nq * buf_k, 0.0);
for (uint64_t i = 0; i < nq; i++) {
scheduler::Id2DistVec result_buf;
auto& result_i = result[i];
size_t buf_k_j = 0, src_k_j = 0, tar_k_j = 0;
size_t buf_idx, src_idx, tar_idx;
if (result[i].empty()) {
result_buf.resize(input_k, scheduler::IdDistPair(-1, 0.0));
uint64_t input_k_multi_i = topk * i;
for (auto k = 0; k < input_k; ++k) {
uint64_t idx = input_k_multi_i + k;
auto& result_buf_item = result_buf[k];
result_buf_item.first = input_ids[idx];
result_buf_item.second = input_distance[idx];
size_t buf_k_multi_i = buf_k * i;
size_t src_k_multi_i = topk * i;
size_t tar_k_multi_i = tar_k * i;
while (buf_k_j < buf_k && src_k_j < src_k && tar_k_j < tar_k) {
src_idx = src_k_multi_i + src_k_j;
tar_idx = tar_k_multi_i + tar_k_j;
buf_idx = buf_k_multi_i + buf_k_j;
if ((ascending && src_distances[src_idx] < tar_distances[tar_idx]) ||
(!ascending && src_distances[src_idx] > tar_distances[tar_idx])) {
buf_ids[buf_idx] = src_ids[src_idx];
buf_distances[buf_idx] = src_distances[src_idx];
src_k_j++;
} else {
buf_ids[buf_idx] = tar_ids[tar_idx];
buf_distances[buf_idx] = tar_distances[tar_idx];
tar_k_j++;
}
} else {
size_t tar_size = result_i.size();
uint64_t output_k = std::min(topk, input_k + tar_size);
result_buf.resize(output_k, scheduler::IdDistPair(-1, 0.0));
size_t buf_k = 0, src_k = 0, tar_k = 0;
uint64_t src_idx;
uint64_t input_k_multi_i = topk * i;
while (buf_k < output_k && src_k < input_k && tar_k < tar_size) {
src_idx = input_k_multi_i + src_k;
auto& result_buf_item = result_buf[buf_k];
auto& result_item = result_i[tar_k];
if ((ascending && input_distance[src_idx] < result_item.second) ||
(!ascending && input_distance[src_idx] > result_item.second)) {
result_buf_item.first = input_ids[src_idx];
result_buf_item.second = input_distance[src_idx];
src_k++;
} else {
result_buf_item = result_item;
tar_k++;
buf_k_j++;
}
if (buf_k_j < buf_k) {
if (src_k_j < src_k) {
while (buf_k_j < buf_k && src_k_j < src_k) {
buf_idx = buf_k_multi_i + buf_k_j;
src_idx = src_k_multi_i + src_k_j;
buf_ids[buf_idx] = src_ids[src_idx];
buf_distances[buf_idx] = src_distances[src_idx];
src_k_j++;
buf_k_j++;
}
buf_k++;
}
if (buf_k < output_k) {
if (src_k < input_k) {
while (buf_k < output_k && src_k < input_k) {
src_idx = input_k_multi_i + src_k;
auto& result_buf_item = result_buf[buf_k];
result_buf_item.first = input_ids[src_idx];
result_buf_item.second = input_distance[src_idx];
src_k++;
buf_k++;
}
} else {
while (buf_k < output_k && tar_k < tar_size) {
result_buf[buf_k] = result_i[tar_k];
tar_k++;
buf_k++;
}
} else {
while (buf_k_j < buf_k && tar_k_j < tar_k) {
buf_idx = buf_k_multi_i + buf_k_j;
tar_idx = tar_k_multi_i + tar_k_j;
buf_ids[buf_idx] = tar_ids[tar_idx];
buf_distances[buf_idx] = tar_distances[tar_idx];
tar_k_j++;
buf_k_j++;
}
}
}
result_i.swap(result_buf);
}
tar_ids.swap(buf_ids);
tar_distances.swap(buf_distances);
}
// void

5
core/src/scheduler/task/SearchTask.h
View File

@ -39,8 +39,9 @@ class XSearchTask : public Task {
public:
static void
MergeTopkToResultSet(const std::vector<int64_t>& input_ids, const std::vector<float>& input_distance,
uint64_t input_k, uint64_t nq, uint64_t topk, bool ascending, scheduler::ResultSet& result);
MergeTopkToResultSet(const scheduler::ResultIds& src_ids, const scheduler::ResultDistances& src_distances,
size_t src_k, size_t nq, size_t topk, bool ascending, scheduler::ResultIds& tar_ids,
scheduler::ResultDistances& tar_distances);
// static void
// MergeTopkArray(std::vector<int64_t>& tar_ids, std::vector<float>& tar_distance, uint64_t& tar_input_k,

2
core/src/sdk/CMakeLists.txt
View File

@ -22,7 +22,7 @@ include_directories(include)
aux_source_directory(interface interface_files)
aux_source_directory(grpc grpc_client_files)
add_library(milvus_sdk STATIC
add_library(milvus_sdk SHARED
${interface_files}
${grpc_client_files}
${grpc_service_files}

103
core/src/sdk/README.md Normal file
View File

@ -0,0 +1,103 @@
### Build C++ SDK
The C++ SDK source code is under milvus/core/src/sdk. Build entire milvus project will also build the C++ SDK project.
If you don't want to build the entire milvus project, follow below steps:
```shell
# generate make files
$ cd [Milvus root path]/core
$ ./build.sh -l
# build C++ SDK project
$ cd [Milvus root path]/core/cmake_build
$ make -C src/sdk
```
### Try C++ example
Firstly, you need to start a Milvus server.
If you've already built the entire milvus project, just start Milvus server with the following command:
```shell
# start milvus server
$ cd [Milvus root path]/core
$ ./start_server.sh
```
You can also use Docker to start Milvus server:
```shell
# pull milvus docker image and start milvus server
$ docker pull milvusdb/milvus:latest
$ docker run --runtime=nvidia -p 19530:19530 -d milvusdb/milvus:latest
```
Run C++ example:
```shell
# run milvus C++ example
$ cd [Milvus root path]/core/cmake_build/src/sdk/examples/simple
$ ./sdk_simple
```
### Make your own C++ client project
Create a folder for the project, and copy C++ SDK header and library files into it.
```shell
# create project folder
$ mkdir MyMilvusClient
$ cd MyMilvusClient
# copy necessary files
$ cp [Milvus root path]/core/cmake_build/src/sdk/libmilvus_sdk.so .
$ cp [Milvus root path]/core/src/sdk/include/MilvusApi.h .
$ cp [Milvus root path]/core/src/sdk/include/Status.h .
```
Create file main.cpp in the project folder, and copy the following code into it:
```shell
#include "./MilvusApi.h"
#include "./Status.h"
int main() {
// connect to milvus server
std::shared_ptr<milvus::Connection> conn = milvus::Connection::Create();
milvus::ConnectParam param = {"127.0.0.1", "19530"};
conn->Connect(param);
// put your client code here
milvus::Connection::Destroy(conn);
return 0;
}
```
Create file CMakeList.txt in the project folder, and copy the following code into it:
```shell
cmake_minimum_required(VERSION 3.14)
project(test)
set(CMAKE_CXX_STANDARD 14)
add_executable(milvus_client main.cpp)
target_link_libraries(milvus_client
${PROJECT_SOURCE_DIR}/libmilvus_sdk.so)
```
Now there are 5 files in your project:
```shell
MyMilvusClient
|-CMakeList.txt
|-main.cpp
|-libmilvus_sdk.so
|-MilvusApi.h
|-Status.h
```
Build the project:
```shell
$ mkdir cmake_build
$ cd cmake_build
$ cmake ..
$ make
```
Run your client program:
```shell
$ ./milvus_client
```

4
core/src/sdk/examples/CMakeLists.txt
View File

@ -17,5 +17,7 @@
# under the License.
#-------------------------------------------------------------------------------
aux_source_directory(${MILVUS_SOURCE_DIR}/src/sdk/examples/utils util_files)
add_subdirectory(grpcsimple)
add_subdirectory(simple)
add_subdirectory(partition)

371
core/src/sdk/examples/grpcsimple/src/ClientTest.cpp
View File

@ -1,371 +0,0 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#include "sdk/examples/grpcsimple/src/ClientTest.h"
#include "MilvusApi.h"
#include <time.h>
#include <unistd.h>
#include <chrono>
#include <iostream>
#include <memory>
#include <thread>
#include <utility>
#include <vector>
//#define SET_VECTOR_IDS;
namespace {
const std::string&
GetTableName();
const char* TABLE_NAME = GetTableName().c_str();
constexpr int64_t TABLE_DIMENSION = 512;
constexpr int64_t TABLE_INDEX_FILE_SIZE = 1024;
constexpr int64_t BATCH_ROW_COUNT = 100000;
constexpr int64_t NQ = 5;
constexpr int64_t TOP_K = 10;
constexpr int64_t SEARCH_TARGET = 5000; // change this value, result is different
constexpr int64_t ADD_VECTOR_LOOP = 5;
constexpr int64_t SECONDS_EACH_HOUR = 3600;
constexpr milvus::IndexType INDEX_TYPE = milvus::IndexType::gpu_ivfsq8;
constexpr int32_t N_LIST = 15000;
#define BLOCK_SPLITER std::cout << "===========================================" << std::endl;
void
PrintTableSchema(const milvus::TableSchema& tb_schema) {
BLOCK_SPLITER
std::cout << "Table name: " << tb_schema.table_name << std::endl;
std::cout << "Table dimension: " << tb_schema.dimension << std::endl;
BLOCK_SPLITER
}
void
PrintSearchResult(const std::vector<std::pair<int64_t, milvus::RowRecord>>& search_record_array,
const std::vector<milvus::TopKQueryResult>& topk_query_result_array) {
BLOCK_SPLITER
std::cout << "Returned result count: " << topk_query_result_array.size() << std::endl;
int32_t index = 0;
for (auto& result : topk_query_result_array) {
auto search_id = search_record_array[index].first;
index++;
std::cout << "No." << std::to_string(index) << " vector " << std::to_string(search_id) << " top "
<< std::to_string(result.query_result_arrays.size()) << " search result:" << std::endl;
for (auto& item : result.query_result_arrays) {
std::cout << "\t" << std::to_string(item.id) << "\tdistance:" << std::to_string(item.distance);
std::cout << std::endl;
}
}
BLOCK_SPLITER
}
std::string
CurrentTime() {
time_t tt;
time(&tt);
tt = tt + 8 * SECONDS_EACH_HOUR;
tm t;
gmtime_r(&tt, &t);
std::string str = std::to_string(t.tm_year + 1900) + "_" + std::to_string(t.tm_mon + 1) + "_" +
std::to_string(t.tm_mday) + "_" + std::to_string(t.tm_hour) + "_" + std::to_string(t.tm_min) +
"_" + std::to_string(t.tm_sec);
return str;
}
std::string
CurrentTmDate(int64_t offset_day = 0) {
time_t tt;
time(&tt);
tt = tt + 8 * SECONDS_EACH_HOUR;
tt = tt + 24 * SECONDS_EACH_HOUR * offset_day;
tm t;
gmtime_r(&tt, &t);
std::string str =
std::to_string(t.tm_year + 1900) + "-" + std::to_string(t.tm_mon + 1) + "-" + std::to_string(t.tm_mday);
return str;
}
const std::string&
GetTableName() {
static std::string s_id("tbl_" + CurrentTime());
return s_id;
}
milvus::TableSchema
BuildTableSchema() {
milvus::TableSchema tb_schema;
tb_schema.table_name = TABLE_NAME;
tb_schema.dimension = TABLE_DIMENSION;
tb_schema.index_file_size = TABLE_INDEX_FILE_SIZE;
tb_schema.metric_type = milvus::MetricType::L2;
return tb_schema;
}
void
BuildVectors(int64_t from, int64_t to, std::vector<milvus::RowRecord>& vector_record_array) {
if (to <= from) {
return;
}
vector_record_array.clear();
for (int64_t k = from; k < to; k++) {
milvus::RowRecord record;
record.data.resize(TABLE_DIMENSION);
for (int64_t i = 0; i < TABLE_DIMENSION; i++) {
record.data[i] = (float)(k % (i + 1));
}
vector_record_array.emplace_back(record);
}
}
void
Sleep(int seconds) {
std::cout << "Waiting " << seconds << " seconds ..." << std::endl;
sleep(seconds);
}
class TimeRecorder {
public:
explicit TimeRecorder(const std::string& title) : title_(title) {
start_ = std::chrono::system_clock::now();
}
~TimeRecorder() {
std::chrono::system_clock::time_point end = std::chrono::system_clock::now();
int64_t span = (std::chrono::duration_cast<std::chrono::milliseconds>(end - start_)).count();
std::cout << title_ << " totally cost: " << span << " ms" << std::endl;
}
private:
std::string title_;
std::chrono::system_clock::time_point start_;
};
void
CheckResult(const std::vector<std::pair<int64_t, milvus::RowRecord>>& search_record_array,
const std::vector<milvus::TopKQueryResult>& topk_query_result_array) {
BLOCK_SPLITER
int64_t index = 0;
for (auto& result : topk_query_result_array) {
auto result_id = result.query_result_arrays[0].id;
auto search_id = search_record_array[index++].first;
if (result_id != search_id) {
std::cout << "The top 1 result is wrong: " << result_id << " vs. " << search_id << std::endl;
} else {
std::cout << "Check result sucessfully" << std::endl;
}
}
BLOCK_SPLITER
}
void
DoSearch(std::shared_ptr<milvus::Connection> conn,
const std::vector<std::pair<int64_t, milvus::RowRecord>>& search_record_array, const std::string& phase_name) {
std::vector<milvus::Range> query_range_array;
milvus::Range rg;
rg.start_value = CurrentTmDate();
rg.end_value = CurrentTmDate(1);
query_range_array.emplace_back(rg);
std::vector<milvus::RowRecord> record_array;
for (auto& pair : search_record_array) {
record_array.push_back(pair.second);
}
auto start = std::chrono::high_resolution_clock::now();
std::vector<milvus::TopKQueryResult> topk_query_result_array;
{
TimeRecorder rc(phase_name);
milvus::Status stat =
conn->Search(TABLE_NAME, record_array, query_range_array, TOP_K, 32, topk_query_result_array);
std::cout << "SearchVector function call status: " << stat.message() << std::endl;
}
auto finish = std::chrono::high_resolution_clock::now();
std::cout << "SEARCHVECTOR COST: "
<< std::chrono::duration_cast<std::chrono::duration<double>>(finish - start).count() << "s\n";
PrintSearchResult(search_record_array, topk_query_result_array);
CheckResult(search_record_array, topk_query_result_array);
}
} // namespace
void
ClientTest::Test(const std::string& address, const std::string& port) {
std::shared_ptr<milvus::Connection> conn = milvus::Connection::Create();
{ // connect server
milvus::ConnectParam param = {address, port};
milvus::Status stat = conn->Connect(param);
std::cout << "Connect function call status: " << stat.message() << std::endl;
}
{ // server version
std::string version = conn->ServerVersion();
std::cout << "Server version: " << version << std::endl;
}
{ // sdk version
std::string version = conn->ClientVersion();
std::cout << "SDK version: " << version << std::endl;
}
{
std::vector<std::string> tables;
milvus::Status stat = conn->ShowTables(tables);
std::cout << "ShowTables function call status: " << stat.message() << std::endl;
std::cout << "All tables: " << std::endl;
for (auto& table : tables) {
int64_t row_count = 0;
// conn->DropTable(table);
stat = conn->CountTable(table, row_count);
std::cout << "\t" << table << "(" << row_count << " rows)" << std::endl;
}
}
{ // create table
milvus::TableSchema tb_schema = BuildTableSchema();
milvus::Status stat = conn->CreateTable(tb_schema);
std::cout << "CreateTable function call status: " << stat.message() << std::endl;
PrintTableSchema(tb_schema);
bool has_table = conn->HasTable(tb_schema.table_name);
if (has_table) {
std::cout << "Table is created" << std::endl;
}
}
{ // describe table
milvus::TableSchema tb_schema;
milvus::Status stat = conn->DescribeTable(TABLE_NAME, tb_schema);
std::cout << "DescribeTable function call status: " << stat.message() << std::endl;
PrintTableSchema(tb_schema);
}
std::vector<std::pair<int64_t, milvus::RowRecord>> search_record_array;
{ // insert vectors
for (int i = 0; i < ADD_VECTOR_LOOP; i++) { // add vectors
std::vector<milvus::RowRecord> record_array;
int64_t begin_index = i * BATCH_ROW_COUNT;
BuildVectors(begin_index, begin_index + BATCH_ROW_COUNT, record_array);
#ifdef SET_VECTOR_IDS
record_ids.resize(ADD_VECTOR_LOOP * BATCH_ROW_COUNT);
for (auto j = begin_index; j < begin_index + BATCH_ROW_COUNT; j++) {
record_ids[i * BATCH_ROW_COUNT + j] = i * BATCH_ROW_COUNT + j;
}
#endif
std::vector<int64_t> record_ids;
// generate user defined ids
for (int k = 0; k < BATCH_ROW_COUNT; k++) {
record_ids.push_back(i * BATCH_ROW_COUNT + k);
}
auto start = std::chrono::high_resolution_clock::now();
milvus::Status stat = conn->Insert(TABLE_NAME, record_array, record_ids);
auto finish = std::chrono::high_resolution_clock::now();
std::cout << "InsertVector cost: "
<< std::chrono::duration_cast<std::chrono::duration<double>>(finish - start).count() << "s\n";
std::cout << "InsertVector function call status: " << stat.message() << std::endl;
std::cout << "Returned id array count: " << record_ids.size() << std::endl;
if (search_record_array.size() < NQ) {
search_record_array.push_back(std::make_pair(record_ids[SEARCH_TARGET], record_array[SEARCH_TARGET]));
}
}
}
{ // search vectors without index
Sleep(2);
int64_t row_count = 0;
milvus::Status stat = conn->CountTable(TABLE_NAME, row_count);
std::cout << TABLE_NAME << "(" << row_count << " rows)" << std::endl;
// DoSearch(conn, search_record_array, "Search without index");
}
{ // wait unit build index finish
std::cout << "Wait until create all index done" << std::endl;
milvus::IndexParam index;
index.table_name = TABLE_NAME;
index.index_type = INDEX_TYPE;
index.nlist = N_LIST;
milvus::Status stat = conn->CreateIndex(index);
std::cout << "CreateIndex function call status: " << stat.message() << std::endl;
milvus::IndexParam index2;
stat = conn->DescribeIndex(TABLE_NAME, index2);
std::cout << "DescribeIndex function call status: " << stat.message() << std::endl;
}
{ // preload table
milvus::Status stat = conn->PreloadTable(TABLE_NAME);
std::cout << "PreloadTable function call status: " << stat.message() << std::endl;
}
{ // search vectors after build index finish
for (uint64_t i = 0; i < 5; ++i) {
DoSearch(conn, search_record_array, "Search after build index finish");
}
// std::cout << conn->DumpTaskTables() << std::endl;
}
{ // delete index
milvus::Status stat = conn->DropIndex(TABLE_NAME);
std::cout << "DropIndex function call status: " << stat.message() << std::endl;
int64_t row_count = 0;
stat = conn->CountTable(TABLE_NAME, row_count);
std::cout << TABLE_NAME << "(" << row_count << " rows)" << std::endl;
}
{ // delete by range
milvus::Range rg;
rg.start_value = CurrentTmDate(-3);
rg.end_value = CurrentTmDate(-2);
milvus::Status stat = conn->DeleteByRange(rg, TABLE_NAME);
std::cout << "DeleteByRange function call status: " << stat.message() << std::endl;
}
{
// delete table
// Status stat = conn->DropTable(TABLE_NAME);
// std::cout << "DeleteTable function call status: " << stat.message() << std::endl;
}
{ // server status
std::string status = conn->ServerStatus();
std::cout << "Server status before disconnect: " << status << std::endl;
}
milvus::Connection::Destroy(conn);
{ // server status
std::string status = conn->ServerStatus();
std::cout << "Server status after disconnect: " << status << std::endl;
}
}

34
core/src/sdk/examples/partition/CMakeLists.txt Normal file
View File

@ -0,0 +1,34 @@
#-------------------------------------------------------------------------------
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http:#www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#-------------------------------------------------------------------------------
aux_source_directory(src src_files)
add_executable(sdk_partition
main.cpp
${src_files}
${util_files}
)
target_link_libraries(sdk_partition
milvus_sdk
pthread
)
install(TARGETS sdk_partition DESTINATION bin)

79
core/src/sdk/examples/partition/main.cpp Normal file
View File

@ -0,0 +1,79 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#include <getopt.h>
#include <libgen.h>
#include <cstring>
#include <string>
#include "sdk/examples/partition/src/ClientTest.h"
void
print_help(const std::string& app_name);
int
main(int argc, char* argv[]) {
printf("Client start...\n");
std::string app_name = basename(argv[0]);
static struct option long_options[] = {{"server", optional_argument, nullptr, 's'},
{"port", optional_argument, nullptr, 'p'},
{"help", no_argument, nullptr, 'h'},
{nullptr, 0, nullptr, 0}};
int option_index = 0;
std::string address = "127.0.0.1", port = "19530";
app_name = argv[0];
int value;
while ((value = getopt_long(argc, argv, "s:p:h", long_options, &option_index)) != -1) {
switch (value) {
case 's': {
char* address_ptr = strdup(optarg);
address = address_ptr;
free(address_ptr);
break;
}
case 'p': {
char* port_ptr = strdup(optarg);
port = port_ptr;
free(port_ptr);
break;
}
case 'h':
default:
print_help(app_name);
return EXIT_SUCCESS;
}
}
ClientTest test;
test.Test(address, port);
printf("Client stop...\n");
return 0;
}
void
print_help(const std::string& app_name) {
printf("\n Usage: %s [OPTIONS]\n\n", app_name.c_str());
printf(" Options:\n");
printf(" -s --server Server address, default 127.0.0.1\n");
printf(" -p --port Server port, default 19530\n");
printf(" -h --help Print help information\n");
printf("\n");
}

205
core/src/sdk/examples/partition/src/ClientTest.cpp Normal file
View File

@ -0,0 +1,205 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#include "sdk/examples/partition/src/ClientTest.h"
#include "MilvusApi.h"
#include "sdk/examples/utils/Utils.h"
#include <src/sdk/examples/utils/TimeRecorder.h>
#include <time.h>
#include <unistd.h>
#include <chrono>
#include <iostream>
#include <memory>
#include <thread>
#include <utility>
#include <vector>
namespace {
const char* TABLE_NAME = milvus_sdk::Utils::GenTableName().c_str();
constexpr int64_t TABLE_DIMENSION = 512;
constexpr int64_t TABLE_INDEX_FILE_SIZE = 1024;
constexpr milvus::MetricType TABLE_METRIC_TYPE = milvus::MetricType::L2;
constexpr int64_t BATCH_ROW_COUNT = 10000;
constexpr int64_t NQ = 5;
constexpr int64_t TOP_K = 10;
constexpr int64_t NPROBE = 32;
constexpr int64_t SEARCH_TARGET = 5000; // change this value, result is different
constexpr milvus::IndexType INDEX_TYPE = milvus::IndexType::IVFSQ8;
constexpr int32_t N_LIST = 15000;
constexpr int32_t PARTITION_COUNT = 5;
constexpr int32_t TARGET_PARTITION = 3;
milvus::TableSchema
BuildTableSchema() {
milvus::TableSchema tb_schema = {TABLE_NAME, TABLE_DIMENSION, TABLE_INDEX_FILE_SIZE, TABLE_METRIC_TYPE};
return tb_schema;
}
milvus::PartitionParam
BuildPartitionParam(int32_t index) {
std::string tag = std::to_string(index);
std::string partition_name = std::string(TABLE_NAME) + "_" + tag;
milvus::PartitionParam partition_param = {TABLE_NAME, partition_name, tag};
return partition_param;
}
milvus::IndexParam
BuildIndexParam() {
milvus::IndexParam index_param = {TABLE_NAME, INDEX_TYPE, N_LIST};
return index_param;
}
} // namespace
void
ClientTest::Test(const std::string& address, const std::string& port) {
std::shared_ptr<milvus::Connection> conn = milvus::Connection::Create();
milvus::Status stat;
{ // connect server
milvus::ConnectParam param = {address, port};
stat = conn->Connect(param);
std::cout << "Connect function call status: " << stat.message() << std::endl;
}
{ // create table
milvus::TableSchema tb_schema = BuildTableSchema();
stat = conn->CreateTable(tb_schema);
std::cout << "CreateTable function call status: " << stat.message() << std::endl;
milvus_sdk::Utils::PrintTableSchema(tb_schema);
}
{ // create partition
for (int32_t i = 0; i < PARTITION_COUNT; i++) {
milvus::PartitionParam partition_param = BuildPartitionParam(i);
stat = conn->CreatePartition(partition_param);
std::cout << "CreatePartition function call status: " << stat.message() << std::endl;
milvus_sdk::Utils::PrintPartitionParam(partition_param);
}
}
{ // insert vectors
milvus_sdk::TimeRecorder rc("All vectors");
for (int i = 0; i < PARTITION_COUNT * 5; i++) {
std::vector<milvus::RowRecord> record_array;
std::vector<int64_t> record_ids;
int64_t begin_index = i * BATCH_ROW_COUNT;
{ // generate vectors
milvus_sdk::TimeRecorder rc("Build vectors No." + std::to_string(i));
milvus_sdk::Utils::BuildVectors(begin_index, begin_index + BATCH_ROW_COUNT, record_array, record_ids,
TABLE_DIMENSION);
}
std::string title = "Insert " + std::to_string(record_array.size()) + " vectors No." + std::to_string(i);
milvus_sdk::TimeRecorder rc(title);
stat = conn->Insert(TABLE_NAME, std::to_string(i % PARTITION_COUNT), record_array, record_ids);
}
}
std::vector<std::pair<int64_t, milvus::RowRecord>> search_record_array;
{ // build search vectors
std::vector<milvus::RowRecord> record_array;
std::vector<int64_t> record_ids;
int64_t index = TARGET_PARTITION * BATCH_ROW_COUNT + SEARCH_TARGET;
milvus_sdk::Utils::BuildVectors(index, index + 1, record_array, record_ids, TABLE_DIMENSION);
search_record_array.push_back(std::make_pair(record_ids[0], record_array[0]));
}
milvus_sdk::Utils::Sleep(3);
{ // table row count
int64_t row_count = 0;
stat = conn->CountTable(TABLE_NAME, row_count);
std::cout << TABLE_NAME << "(" << row_count << " rows)" << std::endl;
}
{ // search vectors
std::cout << "Search in correct partition" << std::endl;
std::vector<std::string> partiton_tags = {std::to_string(TARGET_PARTITION)};
std::vector<milvus::TopKQueryResult> topk_query_result_array;
milvus_sdk::Utils::DoSearch(conn, TABLE_NAME, partiton_tags, TOP_K, NPROBE, search_record_array,
topk_query_result_array);
std::cout << "Search in wrong partition" << std::endl;
partiton_tags = {"0"};
milvus_sdk::Utils::DoSearch(conn, TABLE_NAME, partiton_tags, TOP_K, NPROBE, search_record_array,
topk_query_result_array);
std::cout << "Search by regex matched partition tag" << std::endl;
partiton_tags = {"\\d"};
milvus_sdk::Utils::DoSearch(conn, TABLE_NAME, partiton_tags, TOP_K, NPROBE, search_record_array,
topk_query_result_array);
}
{ // wait unit build index finish
std::cout << "Wait until create all index done" << std::endl;
milvus::IndexParam index1 = BuildIndexParam();
milvus_sdk::Utils::PrintIndexParam(index1);
stat = conn->CreateIndex(index1);
std::cout << "CreateIndex function call status: " << stat.message() << std::endl;
milvus::IndexParam index2;
stat = conn->DescribeIndex(TABLE_NAME, index2);
std::cout << "DescribeIndex function call status: " << stat.message() << std::endl;
milvus_sdk::Utils::PrintIndexParam(index2);
}
{ // table row count
int64_t row_count = 0;
stat = conn->CountTable(TABLE_NAME, row_count);
std::cout << TABLE_NAME << "(" << row_count << " rows)" << std::endl;
}
{ // drop partition
milvus::PartitionParam param1 = {TABLE_NAME, "", std::to_string(TARGET_PARTITION)};
milvus_sdk::Utils::PrintPartitionParam(param1);
stat = conn->DropPartition(param1);
std::cout << "DropPartition function call status: " << stat.message() << std::endl;
}
{ // table row count
int64_t row_count = 0;
stat = conn->CountTable(TABLE_NAME, row_count);
std::cout << TABLE_NAME << "(" << row_count << " rows)" << std::endl;
}
{ // search vectors
std::cout << "Search in whole table" << std::endl;
std::vector<std::string> partiton_tags;
std::vector<milvus::TopKQueryResult> topk_query_result_array;
milvus_sdk::Utils::DoSearch(conn, TABLE_NAME, partiton_tags, TOP_K, NPROBE, search_record_array,
topk_query_result_array);
}
{ // drop index
stat = conn->DropIndex(TABLE_NAME);
std::cout << "DropIndex function call status: " << stat.message() << std::endl;
int64_t row_count = 0;
stat = conn->CountTable(TABLE_NAME, row_count);
std::cout << TABLE_NAME << "(" << row_count << " rows)" << std::endl;
}
{ // drop table
stat = conn->DropTable(TABLE_NAME);
std::cout << "DropTable function call status: " << stat.message() << std::endl;
}
milvus::Connection::Destroy(conn);
}

0
core/src/sdk/examples/grpcsimple/src/ClientTest.h → core/src/sdk/examples/partition/src/ClientTest.h
View File

2
core/src/sdk/examples/grpcsimple/CMakeLists.txt → core/src/sdk/examples/simple/CMakeLists.txt
View File

@ -17,12 +17,12 @@
# under the License.
#-------------------------------------------------------------------------------
aux_source_directory(src src_files)
add_executable(sdk_simple
main.cpp
${src_files}
${util_files}
)
target_link_libraries(sdk_simple

2
core/src/sdk/examples/grpcsimple/main.cpp → core/src/sdk/examples/simple/main.cpp
View File

@ -20,7 +20,7 @@
#include <cstring>
#include <string>
#include "src/ClientTest.h"
#include "sdk/examples/simple/src/ClientTest.h"
void
print_help(const std::string& app_name);

209
core/src/sdk/examples/simple/src/ClientTest.cpp Normal file
View File

@ -0,0 +1,209 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#include "sdk/examples/simple/src/ClientTest.h"
#include "MilvusApi.h"
#include "sdk/examples/utils/TimeRecorder.h"
#include "sdk/examples/utils/Utils.h"
#include <time.h>
#include <unistd.h>
#include <iostream>
#include <memory>
#include <thread>
#include <utility>
#include <vector>
namespace {
const char* TABLE_NAME = milvus_sdk::Utils::GenTableName().c_str();
constexpr int64_t TABLE_DIMENSION = 512;
constexpr int64_t TABLE_INDEX_FILE_SIZE = 1024;
constexpr milvus::MetricType TABLE_METRIC_TYPE = milvus::MetricType::L2;
constexpr int64_t BATCH_ROW_COUNT = 100000;
constexpr int64_t NQ = 5;
constexpr int64_t TOP_K = 10;
constexpr int64_t NPROBE = 32;
constexpr int64_t SEARCH_TARGET = 5000; // change this value, result is different
constexpr int64_t ADD_VECTOR_LOOP = 5;
constexpr milvus::IndexType INDEX_TYPE = milvus::IndexType::IVFSQ8;
constexpr int32_t N_LIST = 15000;
milvus::TableSchema
BuildTableSchema() {
milvus::TableSchema tb_schema = {TABLE_NAME, TABLE_DIMENSION, TABLE_INDEX_FILE_SIZE, TABLE_METRIC_TYPE};
return tb_schema;
}
milvus::IndexParam
BuildIndexParam() {
milvus::IndexParam index_param = {TABLE_NAME, INDEX_TYPE, N_LIST};
return index_param;
}
} // namespace
void
ClientTest::Test(const std::string& address, const std::string& port) {
std::shared_ptr<milvus::Connection> conn = milvus::Connection::Create();
milvus::Status stat;
{ // connect server
milvus::ConnectParam param = {address, port};
stat = conn->Connect(param);
std::cout << "Connect function call status: " << stat.message() << std::endl;
}
{ // server version
std::string version = conn->ServerVersion();
std::cout << "Server version: " << version << std::endl;
}
{ // sdk version
std::string version = conn->ClientVersion();
std::cout << "SDK version: " << version << std::endl;
}
{ // show tables
std::vector<std::string> tables;
stat = conn->ShowTables(tables);
std::cout << "ShowTables function call status: " << stat.message() << std::endl;
std::cout << "All tables: " << std::endl;
for (auto& table : tables) {
int64_t row_count = 0;
// conn->DropTable(table);
stat = conn->CountTable(table, row_count);
std::cout << "\t" << table << "(" << row_count << " rows)" << std::endl;
}
}
{ // create table
milvus::TableSchema tb_schema = BuildTableSchema();
stat = conn->CreateTable(tb_schema);
std::cout << "CreateTable function call status: " << stat.message() << std::endl;
milvus_sdk::Utils::PrintTableSchema(tb_schema);
bool has_table = conn->HasTable(tb_schema.table_name);
if (has_table) {
std::cout << "Table is created" << std::endl;
}
}
{ // describe table
milvus::TableSchema tb_schema;
stat = conn->DescribeTable(TABLE_NAME, tb_schema);
std::cout << "DescribeTable function call status: " << stat.message() << std::endl;
milvus_sdk::Utils::PrintTableSchema(tb_schema);
}
{ // insert vectors
for (int i = 0; i < ADD_VECTOR_LOOP; i++) {
std::vector<milvus::RowRecord> record_array;
std::vector<int64_t> record_ids;
int64_t begin_index = i * BATCH_ROW_COUNT;
{ // generate vectors
milvus_sdk::TimeRecorder rc("Build vectors No." + std::to_string(i));
milvus_sdk::Utils::BuildVectors(begin_index, begin_index + BATCH_ROW_COUNT, record_array, record_ids,
TABLE_DIMENSION);
}
std::string title = "Insert " + std::to_string(record_array.size()) + " vectors No." + std::to_string(i);
milvus_sdk::TimeRecorder rc(title);
stat = conn->Insert(TABLE_NAME, "", record_array, record_ids);
std::cout << "InsertVector function call status: " << stat.message() << std::endl;
std::cout << "Returned id array count: " << record_ids.size() << std::endl;
}
}
std::vector<std::pair<int64_t, milvus::RowRecord>> search_record_array;
{ // build search vectors
for (int64_t i = 0; i < NQ; i++) {
std::vector<milvus::RowRecord> record_array;
std::vector<int64_t> record_ids;
int64_t index = i * BATCH_ROW_COUNT + SEARCH_TARGET;
milvus_sdk::Utils::BuildVectors(index, index + 1, record_array, record_ids, TABLE_DIMENSION);
search_record_array.push_back(std::make_pair(record_ids[0], record_array[0]));
}
}
milvus_sdk::Utils::Sleep(3);
{ // search vectors
std::vector<std::string> partiton_tags;
std::vector<milvus::TopKQueryResult> topk_query_result_array;
milvus_sdk::Utils::DoSearch(conn, TABLE_NAME, partiton_tags, TOP_K, NPROBE, search_record_array,
topk_query_result_array);
}
{ // wait unit build index finish
std::cout << "Wait until create all index done" << std::endl;
milvus::IndexParam index1 = BuildIndexParam();
milvus_sdk::Utils::PrintIndexParam(index1);
stat = conn->CreateIndex(index1);
std::cout << "CreateIndex function call status: " << stat.message() << std::endl;
milvus::IndexParam index2;
stat = conn->DescribeIndex(TABLE_NAME, index2);
std::cout << "DescribeIndex function call status: " << stat.message() << std::endl;
milvus_sdk::Utils::PrintIndexParam(index2);
}
{ // preload table
stat = conn->PreloadTable(TABLE_NAME);
std::cout << "PreloadTable function call status: " << stat.message() << std::endl;
}
{ // search vectors
std::vector<std::string> partiton_tags;
std::vector<milvus::TopKQueryResult> topk_query_result_array;
milvus_sdk::Utils::DoSearch(conn, TABLE_NAME, partiton_tags, TOP_K, NPROBE, search_record_array,
topk_query_result_array);
}
{ // drop index
stat = conn->DropIndex(TABLE_NAME);
std::cout << "DropIndex function call status: " << stat.message() << std::endl;
int64_t row_count = 0;
stat = conn->CountTable(TABLE_NAME, row_count);
std::cout << TABLE_NAME << "(" << row_count << " rows)" << std::endl;
}
{ // delete by range
milvus::Range rg;
rg.start_value = milvus_sdk::Utils::CurrentTmDate(-3);
rg.end_value = milvus_sdk::Utils::CurrentTmDate(-2);
stat = conn->DeleteByDate(TABLE_NAME, rg);
std::cout << "DeleteByDate function call status: " << stat.message() << std::endl;
}
{ // drop table
stat = conn->DropTable(TABLE_NAME);
std::cout << "DropTable function call status: " << stat.message() << std::endl;
}
{ // server status
std::string status = conn->ServerStatus();
std::cout << "Server status before disconnect: " << status << std::endl;
}
milvus::Connection::Destroy(conn);
{ // server status
std::string status = conn->ServerStatus();
std::cout << "Server status after disconnect: " << status << std::endl;
}
}

26
core/src/sdk/examples/simple/src/ClientTest.h Normal file
View File

@ -0,0 +1,26 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#pragma once
#include <string>
class ClientTest {
public:
void
Test(const std::string& address, const std::string& port);
};

35
core/src/sdk/examples/utils/TimeRecorder.cpp Normal file
View File

@ -0,0 +1,35 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#include "sdk/examples/utils/TimeRecorder.h"
#include <iostream>
namespace milvus_sdk {
TimeRecorder::TimeRecorder(const std::string& title) : title_(title) {
start_ = std::chrono::system_clock::now();
std::cout << title_ << " begin..." << std::endl;
}
TimeRecorder::~TimeRecorder() {
std::chrono::system_clock::time_point end = std::chrono::system_clock::now();
int64_t span = (std::chrono::duration_cast<std::chrono::milliseconds>(end - start_)).count();
std::cout << title_ << " totally cost: " << span << " ms" << std::endl;
}
} // namespace milvus_sdk

36
core/src/sdk/examples/utils/TimeRecorder.h Normal file
View File

@ -0,0 +1,36 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#pragma once
#include <chrono>
#include <string>
namespace milvus_sdk {
class TimeRecorder {
public:
explicit TimeRecorder(const std::string& title);
~TimeRecorder();
private:
std::string title_;
std::chrono::system_clock::time_point start_;
};
} // namespace milvus_sdk

225
core/src/sdk/examples/utils/Utils.cpp Normal file
View File

@ -0,0 +1,225 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#include "sdk/examples/utils/Utils.h"
#include "sdk/examples/utils/TimeRecorder.h"
#include <time.h>
#include <unistd.h>
#include <iostream>
#include <memory>
#include <thread>
#include <utility>
#include <vector>
namespace milvus_sdk {
constexpr int64_t SECONDS_EACH_HOUR = 3600;
#define BLOCK_SPLITER std::cout << "===========================================" << std::endl;
std::string
Utils::CurrentTime() {
time_t tt;
time(&tt);
tt = tt + 8 * SECONDS_EACH_HOUR;
tm t;
gmtime_r(&tt, &t);
std::string str = std::to_string(t.tm_year + 1900) + "_" + std::to_string(t.tm_mon + 1) + "_" +
std::to_string(t.tm_mday) + "_" + std::to_string(t.tm_hour) + "_" + std::to_string(t.tm_min) +
"_" + std::to_string(t.tm_sec);
return str;
}
std::string
Utils::CurrentTmDate(int64_t offset_day) {
time_t tt;
time(&tt);
tt = tt + 8 * SECONDS_EACH_HOUR;
tt = tt + 24 * SECONDS_EACH_HOUR * offset_day;
tm t;
gmtime_r(&tt, &t);
std::string str =
std::to_string(t.tm_year + 1900) + "-" + std::to_string(t.tm_mon + 1) + "-" + std::to_string(t.tm_mday);
return str;
}
void
Utils::Sleep(int seconds) {
std::cout << "Waiting " << seconds << " seconds ..." << std::endl;
sleep(seconds);
}
const std::string&
Utils::GenTableName() {
static std::string s_id("tbl_" + CurrentTime());
return s_id;
}
std::string
Utils::MetricTypeName(const milvus::MetricType& metric_type) {
switch (metric_type) {
case milvus::MetricType::L2:
return "L2 distance";
case milvus::MetricType::IP:
return "Inner product";
default:
return "Unknown metric type";
}
}
std::string
Utils::IndexTypeName(const milvus::IndexType& index_type) {
switch (index_type) {
case milvus::IndexType::FLAT:
return "FLAT";
case milvus::IndexType::IVFFLAT:
return "IVFFLAT";
case milvus::IndexType::IVFSQ8:
return "IVFSQ8";
case milvus::IndexType::NSG:
return "NSG";
case milvus::IndexType::IVFSQ8H:
return "IVFSQ8H";
default:
return "Unknown index type";
}
}
void
Utils::PrintTableSchema(const milvus::TableSchema& tb_schema) {
BLOCK_SPLITER
std::cout << "Table name: " << tb_schema.table_name << std::endl;
std::cout << "Table dimension: " << tb_schema.dimension << std::endl;
std::cout << "Table index file size: " << tb_schema.index_file_size << std::endl;
std::cout << "Table metric type: " << MetricTypeName(tb_schema.metric_type) << std::endl;
BLOCK_SPLITER
}
void
Utils::PrintPartitionParam(const milvus::PartitionParam& partition_param) {
BLOCK_SPLITER
std::cout << "Table name: " << partition_param.table_name << std::endl;
std::cout << "Partition name: " << partition_param.partition_name << std::endl;
std::cout << "Partition tag: " << partition_param.partition_tag << std::endl;
BLOCK_SPLITER
}
void
Utils::PrintIndexParam(const milvus::IndexParam& index_param) {
BLOCK_SPLITER
std::cout << "Index table name: " << index_param.table_name << std::endl;
std::cout << "Index type: " << IndexTypeName(index_param.index_type) << std::endl;
std::cout << "Index nlist: " << index_param.nlist << std::endl;
BLOCK_SPLITER
}
void
Utils::BuildVectors(int64_t from, int64_t to, std::vector<milvus::RowRecord>& vector_record_array,
std::vector<int64_t>& record_ids, int64_t dimension) {
if (to <= from) {
return;
}
vector_record_array.clear();
record_ids.clear();
for (int64_t k = from; k < to; k++) {
milvus::RowRecord record;
record.data.resize(dimension);
for (int64_t i = 0; i < dimension; i++) {
record.data[i] = (float)(k % (i + 1));
}
vector_record_array.emplace_back(record);
record_ids.push_back(k);
}
}
void
Utils::PrintSearchResult(const std::vector<std::pair<int64_t, milvus::RowRecord>>& search_record_array,
const std::vector<milvus::TopKQueryResult>& topk_query_result_array) {
BLOCK_SPLITER
std::cout << "Returned result count: " << topk_query_result_array.size() << std::endl;
int32_t index = 0;
for (auto& result : topk_query_result_array) {
auto search_id = search_record_array[index].first;
index++;
std::cout << "No." << std::to_string(index) << " vector " << std::to_string(search_id) << " top "
<< std::to_string(result.query_result_arrays.size()) << " search result:" << std::endl;
for (auto& item : result.query_result_arrays) {
std::cout << "\t" << std::to_string(item.id) << "\tdistance:" << std::to_string(item.distance);
std::cout << std::endl;
}
}
BLOCK_SPLITER
}
void
Utils::CheckSearchResult(const std::vector<std::pair<int64_t, milvus::RowRecord>>& search_record_array,
const std::vector<milvus::TopKQueryResult>& topk_query_result_array) {
BLOCK_SPLITER
int64_t index = 0;
for (auto& result : topk_query_result_array) {
auto result_id = result.query_result_arrays[0].id;
auto search_id = search_record_array[index++].first;
if (result_id != search_id) {
std::cout << "The top 1 result is wrong: " << result_id << " vs. " << search_id << std::endl;
} else {
std::cout << "Check result sucessfully" << std::endl;
}
}
BLOCK_SPLITER
}
void
Utils::DoSearch(std::shared_ptr<milvus::Connection> conn, const std::string& table_name,
const std::vector<std::string>& partiton_tags, int64_t top_k, int64_t nprobe,
const std::vector<std::pair<int64_t, milvus::RowRecord>>& search_record_array,
std::vector<milvus::TopKQueryResult>& topk_query_result_array) {
topk_query_result_array.clear();
std::vector<milvus::Range> query_range_array;
milvus::Range rg;
rg.start_value = CurrentTmDate();
rg.end_value = CurrentTmDate(1);
query_range_array.emplace_back(rg);
std::vector<milvus::RowRecord> record_array;
for (auto& pair : search_record_array) {
record_array.push_back(pair.second);
}
{
BLOCK_SPLITER
milvus_sdk::TimeRecorder rc("search");
milvus::Status stat = conn->Search(table_name, partiton_tags, record_array, query_range_array, top_k, nprobe,
topk_query_result_array);
std::cout << "SearchVector function call status: " << stat.message() << std::endl;
BLOCK_SPLITER
}
PrintSearchResult(search_record_array, topk_query_result_array);
CheckSearchResult(search_record_array, topk_query_result_array);
}
} // namespace milvus_sdk

77
core/src/sdk/examples/utils/Utils.h Normal file
View File

@ -0,0 +1,77 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#pragma once
#include "MilvusApi.h"
#include <memory>
#include <string>
#include <utility>
#include <vector>
namespace milvus_sdk {
class Utils {
public:
static std::string
CurrentTime();
static std::string
CurrentTmDate(int64_t offset_day = 0);
static const std::string&
GenTableName();
static void
Sleep(int seconds);
static std::string
MetricTypeName(const milvus::MetricType& metric_type);
static std::string
IndexTypeName(const milvus::IndexType& index_type);
static void
PrintTableSchema(const milvus::TableSchema& tb_schema);
static void
PrintPartitionParam(const milvus::PartitionParam& partition_param);
static void
PrintIndexParam(const milvus::IndexParam& index_param);
static void
BuildVectors(int64_t from, int64_t to, std::vector<milvus::RowRecord>& vector_record_array,
std::vector<int64_t>& record_ids, int64_t dimension);
static void
PrintSearchResult(const std::vector<std::pair<int64_t, milvus::RowRecord>>& search_record_array,
const std::vector<milvus::TopKQueryResult>& topk_query_result_array);
static void
CheckSearchResult(const std::vector<std::pair<int64_t, milvus::RowRecord>>& search_record_array,
const std::vector<milvus::TopKQueryResult>& topk_query_result_array);
static void
DoSearch(std::shared_ptr<milvus::Connection> conn, const std::string& table_name,
const std::vector<std::string>& partiton_tags, int64_t top_k, int64_t nprobe,
const std::vector<std::pair<int64_t, milvus::RowRecord>>& search_record_array,
std::vector<milvus::TopKQueryResult>& topk_query_result_array);
};
} // namespace milvus_sdk

67
core/src/sdk/grpc/ClientProxy.cpp
View File

@ -138,8 +138,8 @@ ClientProxy::CreateIndex(const IndexParam& index_param) {
}
Status
ClientProxy::Insert(const std::string& table_name, const std::vector<RowRecord>& record_array,
std::vector<int64_t>& id_array) {
ClientProxy::Insert(const std::string& table_name, const std::string& partition_tag,
const std::vector<RowRecord>& record_array, std::vector<int64_t>& id_array) {
Status status = Status::OK();
try {
////////////////////////////////////////////////////////////////////////////
@ -185,6 +185,7 @@ ClientProxy::Insert(const std::string& table_name, const std::vector<RowRecord>&
#else
::milvus::grpc::InsertParam insert_param;
insert_param.set_table_name(table_name);
insert_param.set_partition_tag(partition_tag);
for (auto& record : record_array) {
::milvus::grpc::RowRecord* grpc_record = insert_param.add_row_record_array();
@ -215,15 +216,18 @@ ClientProxy::Insert(const std::string& table_name, const std::vector<RowRecord>&
}
Status
ClientProxy::Search(const std::string& table_name, const std::vector<RowRecord>& query_record_array,
const std::vector<Range>& query_range_array, int64_t topk, int64_t nprobe,
std::vector<TopKQueryResult>& topk_query_result_array) {
ClientProxy::Search(const std::string& table_name, const std::vector<std::string>& partiton_tags,
const std::vector<RowRecord>& query_record_array, const std::vector<Range>& query_range_array,
int64_t topk, int64_t nprobe, std::vector<TopKQueryResult>& topk_query_result_array) {
try {
// step 1: convert vectors data
::milvus::grpc::SearchParam search_param;
search_param.set_table_name(table_name);
search_param.set_topk(topk);
search_param.set_nprobe(nprobe);
for (auto& tag : partiton_tags) {
search_param.add_partition_tag_array(tag);
}
for (auto& record : query_record_array) {
::milvus::grpc::RowRecord* row_record = search_param.add_query_record_array();
for (auto& rec : record.data) {
@ -349,13 +353,13 @@ ClientProxy::DumpTaskTables() const {
}
Status
ClientProxy::DeleteByRange(milvus::Range& range, const std::string& table_name) {
ClientProxy::DeleteByDate(const std::string& table_name, const milvus::Range& range) {
try {
::milvus::grpc::DeleteByRangeParam delete_by_range_param;
::milvus::grpc::DeleteByDateParam delete_by_range_param;
delete_by_range_param.set_table_name(table_name);
delete_by_range_param.mutable_range()->set_start_value(range.start_value);
delete_by_range_param.mutable_range()->set_end_value(range.end_value);
return client_ptr_->DeleteByRange(delete_by_range_param);
return client_ptr_->DeleteByDate(delete_by_range_param);
} catch (std::exception& ex) {
return Status(StatusCode::UnknownError, "fail to delete by range: " + std::string(ex.what()));
}
@ -401,4 +405,51 @@ ClientProxy::DropIndex(const std::string& table_name) const {
}
}
Status
ClientProxy::CreatePartition(const PartitionParam& partition_param) {
try {
::milvus::grpc::PartitionParam grpc_partition_param;
grpc_partition_param.set_table_name(partition_param.table_name);
grpc_partition_param.set_partition_name(partition_param.partition_name);
grpc_partition_param.set_tag(partition_param.partition_tag);
Status status = client_ptr_->CreatePartition(grpc_partition_param);
return status;
} catch (std::exception& ex) {
return Status(StatusCode::UnknownError, "fail to create partition: " + std::string(ex.what()));
}
}
Status
ClientProxy::ShowPartitions(const std::string& table_name, PartitionList& partition_array) const {
try {
::milvus::grpc::TableName grpc_table_name;
grpc_table_name.set_table_name(table_name);
::milvus::grpc::PartitionList grpc_partition_list;
Status status = client_ptr_->ShowPartitions(grpc_table_name, grpc_partition_list);
partition_array.resize(grpc_partition_list.partition_array_size());
for (uint64_t i = 0; i < grpc_partition_list.partition_array_size(); ++i) {
partition_array[i].table_name = grpc_partition_list.partition_array(i).table_name();
partition_array[i].partition_name = grpc_partition_list.partition_array(i).partition_name();
partition_array[i].partition_tag = grpc_partition_list.partition_array(i).tag();
}
return status;
} catch (std::exception& ex) {
return Status(StatusCode::UnknownError, "fail to show partitions: " + std::string(ex.what()));
}
}
Status
ClientProxy::DropPartition(const PartitionParam& partition_param) {
try {
::milvus::grpc::PartitionParam grpc_partition_param;
grpc_partition_param.set_table_name(partition_param.table_name);
grpc_partition_param.set_partition_name(partition_param.partition_name);
grpc_partition_param.set_tag(partition_param.partition_tag);
Status status = client_ptr_->DropPartition(grpc_partition_param);
return status;
} catch (std::exception& ex) {
return Status(StatusCode::UnknownError, "fail to drop partition: " + std::string(ex.what()));
}
}
} // namespace milvus

19
core/src/sdk/grpc/ClientProxy.h
View File

@ -54,13 +54,13 @@ class ClientProxy : public Connection {
CreateIndex(const IndexParam& index_param) override;
Status
Insert(const std::string& table_name, const std::vector<RowRecord>& record_array,
Insert(const std::string& table_name, const std::string& partition_tag, const std::vector<RowRecord>& record_array,
std::vector<int64_t>& id_array) override;
Status
Search(const std::string& table_name, const std::vector<RowRecord>& query_record_array,
const std::vector<Range>& query_range_array, int64_t topk, int64_t nprobe,
std::vector<TopKQueryResult>& topk_query_result_array) override;
Search(const std::string& table_name, const std::vector<std::string>& partiton_tags,
const std::vector<RowRecord>& query_record_array, const std::vector<Range>& query_range_array, int64_t topk,
int64_t nprobe, std::vector<TopKQueryResult>& topk_query_result_array) override;
Status
DescribeTable(const std::string& table_name, TableSchema& table_schema) override;
@ -84,7 +84,7 @@ class ClientProxy : public Connection {
DumpTaskTables() const override;
Status
DeleteByRange(Range& range, const std::string& table_name) override;
DeleteByDate(const std::string& table_name, const Range& range) override;
Status
PreloadTable(const std::string& table_name) const override;
@ -95,6 +95,15 @@ class ClientProxy : public Connection {
Status
DropIndex(const std::string& table_name) const override;
Status
CreatePartition(const PartitionParam& partition_param) override;
Status
ShowPartitions(const std::string& table_name, PartitionList& partition_array) const override;
Status
DropPartition(const PartitionParam& partition_param) override;
private:
std::shared_ptr<::grpc::Channel> channel_;

59
core/src/sdk/grpc/GrpcClient.cpp
View File

@ -259,13 +259,13 @@ GrpcClient::PreloadTable(milvus::grpc::TableName& table_name) {
}
Status
GrpcClient::DeleteByRange(grpc::DeleteByRangeParam& delete_by_range_param) {
GrpcClient::DeleteByDate(grpc::DeleteByDateParam& delete_by_range_param) {
ClientContext context;
::milvus::grpc::Status response;
::grpc::Status grpc_status = stub_->DeleteByRange(&context, delete_by_range_param, &response);
::grpc::Status grpc_status = stub_->DeleteByDate(&context, delete_by_range_param, &response);
if (!grpc_status.ok()) {
std::cerr << "DeleteByRange gRPC failed!" << std::endl;
std::cerr << "DeleteByDate gRPC failed!" << std::endl;
return Status(StatusCode::RPCFailed, grpc_status.error_message());
}
@ -317,4 +317,57 @@ GrpcClient::DropIndex(grpc::TableName& table_name) {
return Status::OK();
}
Status
GrpcClient::CreatePartition(const grpc::PartitionParam& partition_param) {
ClientContext context;
::milvus::grpc::Status response;
::grpc::Status grpc_status = stub_->CreatePartition(&context, partition_param, &response);
if (!grpc_status.ok()) {
std::cerr << "CreatePartition gRPC failed!" << std::endl;
return Status(StatusCode::RPCFailed, grpc_status.error_message());
}
if (response.error_code() != grpc::SUCCESS) {
std::cerr << response.reason() << std::endl;
return Status(StatusCode::ServerFailed, response.reason());
}
return Status::OK();
}
Status
GrpcClient::ShowPartitions(const grpc::TableName& table_name, grpc::PartitionList& partition_array) const {
ClientContext context;
::grpc::Status grpc_status = stub_->ShowPartitions(&context, table_name, &partition_array);
if (!grpc_status.ok()) {
std::cerr << "ShowPartitions gRPC failed!" << std::endl;
return Status(StatusCode::RPCFailed, grpc_status.error_message());
}
if (partition_array.status().error_code() != grpc::SUCCESS) {
std::cerr << partition_array.status().reason() << std::endl;
return Status(StatusCode::ServerFailed, partition_array.status().reason());
}
return Status::OK();
}
Status
GrpcClient::DropPartition(const ::milvus::grpc::PartitionParam& partition_param) {
ClientContext context;
::milvus::grpc::Status response;
::grpc::Status grpc_status = stub_->DropPartition(&context, partition_param, &response);
if (!grpc_status.ok()) {
std::cerr << "DropPartition gRPC failed!" << std::endl;
return Status(StatusCode::RPCFailed, grpc_status.error_message());
}
if (response.error_code() != grpc::SUCCESS) {
std::cerr << response.reason() << std::endl;
return Status(StatusCode::ServerFailed, response.reason());
}
return Status::OK();
}
} // namespace milvus

11
core/src/sdk/grpc/GrpcClient.h
View File

@ -72,7 +72,7 @@ class GrpcClient {
Cmd(std::string& result, const std::string& cmd);
Status
DeleteByRange(grpc::DeleteByRangeParam& delete_by_range_param);
DeleteByDate(grpc::DeleteByDateParam& delete_by_range_param);
Status
PreloadTable(grpc::TableName& table_name);
@ -83,6 +83,15 @@ class GrpcClient {
Status
DropIndex(grpc::TableName& table_name);
Status
CreatePartition(const grpc::PartitionParam& partition_param);
Status
ShowPartitions(const grpc::TableName& table_name, grpc::PartitionList& partition_array) const;
Status
DropPartition(const ::milvus::grpc::PartitionParam& partition_param);
Status
Disconnect();

116
core/src/sdk/include/MilvusApi.h
View File

@ -31,12 +31,12 @@ namespace milvus {
* @brief Index Type
*/
enum class IndexType {
invalid = 0,
cpu_idmap,
gpu_ivfflat,
gpu_ivfsq8,
mix_nsg,
ivfsq8h,
INVALID = 0,
FLAT = 1,
IVFFLAT = 2,
IVFSQ8 = 3,
NSG = 4,
IVFSQ8H = 5,
};
enum class MetricType {
@ -64,7 +64,7 @@ struct TableSchema {
/**
* @brief Range information
* for DATE partition, the format is like: 'year-month-day'
* for DATE range, the format is like: 'year-month-day'
*/
struct Range {
std::string start_value; ///< Range start
@ -102,6 +102,17 @@ struct IndexParam {
int32_t nlist;
};
/**
* @brief partition parameters
*/
struct PartitionParam {
std::string table_name;
std::string partition_name;
std::string partition_tag;
};
using PartitionList = std::vector<PartitionParam>;
/**
* @brief SDK main class
*/
@ -195,7 +206,7 @@ class Connection {
*
* This method is used to create table
*
* @param table_name, table name is going to be tested.
* @param table_name, target table's name.
*
* @return Indicate if table is cexist
*/
@ -205,9 +216,9 @@ class Connection {
/**
* @brief Delete table method
*
* This method is used to delete table.
* This method is used to delete table(and its partitions).
*
* @param table_name, table name is going to be deleted.
* @param table_name, target table's name.
*
* @return Indicate if table is delete successfully.
*/
@ -217,7 +228,7 @@ class Connection {
/**
* @brief Create index method
*
* This method is used to create index for whole table
* This method is used to create index for whole table(and its partitions).
*
* @param IndexParam
* table_name, table name is going to be create index.
@ -235,14 +246,15 @@ class Connection {
*
* This method is used to add vector array to table.
*
* @param table_name, table_name is inserted.
* @param table_name, target table's name.
* @param partition_tag, target partition's tag, keep empty if no partition.
* @param record_array, vector array is inserted.
* @param id_array, after inserted every vector is given a id.
*
* @return Indicate if vector array are inserted successfully
*/
virtual Status
Insert(const std::string& table_name, const std::vector<RowRecord>& record_array,
Insert(const std::string& table_name, const std::string& partition_tag, const std::vector<RowRecord>& record_array,
std::vector<int64_t>& id_array) = 0;
/**
@ -250,7 +262,8 @@ class Connection {
*
* This method is used to query vector in table.
*
* @param table_name, table_name is queried.
* @param table_name, target table's name, keep empty if no partition.
* @param partition_tags, target partitions.
* @param query_record_array, all vector are going to be queried.
* @param query_range_array, time ranges, if not specified, will search in whole table
* @param topk, how many similarity vectors will be searched.
@ -259,16 +272,16 @@ class Connection {
* @return Indicate if query is successful.
*/
virtual Status
Search(const std::string& table_name, const std::vector<RowRecord>& query_record_array,
const std::vector<Range>& query_range_array, int64_t topk, int64_t nprobe,
std::vector<TopKQueryResult>& topk_query_result_array) = 0;
Search(const std::string& table_name, const std::vector<std::string>& partiton_tags,
const std::vector<RowRecord>& query_record_array, const std::vector<Range>& query_range_array, int64_t topk,
int64_t nprobe, std::vector<TopKQueryResult>& topk_query_result_array) = 0;
/**
* @brief Show table description
*
* This method is used to show table information.
*
* @param table_name, which table is show.
* @param table_name, target table's name.
* @param table_schema, table_schema is given when operation is successful.
*
* @return Indicate if this operation is successful.
@ -281,8 +294,8 @@ class Connection {
*
* This method is used to get table row count.
*
* @param table_name, table's name.
* @param row_count, table total row count.
* @param table_name, target table's name.
* @param row_count, table total row count(including partitions).
*
* @return Indicate if this operation is successful.
*/
@ -331,21 +344,28 @@ class Connection {
virtual std::string
ServerStatus() const = 0;
/**
* @brief dump server tasks information
*
* This method is internal used.
*
* @return Server status.
*/
virtual std::string
DumpTaskTables() const = 0;
/**
* @brief delete tables by range
* @brief delete tables by date range
*
* This method is used to delete tables by range.
* This method is used to delete table data by date range.
*
* @param table_name, target table's name.
* @param Range, table range to delete.
* @param table_name
*
* @return Indicate if this operation is successful.
*/
virtual Status
DeleteByRange(Range& range, const std::string& table_name) = 0;
DeleteByDate(const std::string& table_name, const Range& range) = 0;
/**
* @brief preload table
@ -364,9 +384,10 @@ class Connection {
*
* This method is used to describe index
*
* @param table_name
* @param table_name, target table's name.
* @param index_param, returned index information.
*
* @return index informations and indicate if this operation is successful.
* @return Indicate if this operation is successful.
*/
virtual Status
DescribeIndex(const std::string& table_name, IndexParam& index_param) const = 0;
@ -374,14 +395,53 @@ class Connection {
/**
* @brief drop index
*
* This method is used to drop index
* This method is used to drop index of table(and its partitions)
*
* @param table_name
* @param table_name, target table's name.
*
* @return Indicate if this operation is successful.
*/
virtual Status
DropIndex(const std::string& table_name) const = 0;
/**
* @brief Create partition method
*
* This method is used to create table partition
*
* @param param, use to provide partition information to be created.
*
* @return Indicate if partition is created successfully
*/
virtual Status
CreatePartition(const PartitionParam& param) = 0;
/**
* @brief Test table existence method
*
* This method is used to create table
*
* @param table_name, table name is going to be tested.
* @param partition_array, partition array of the table.
*
* @return Indicate if this operation is successful
*/
virtual Status
ShowPartitions(const std::string& table_name, PartitionList& partition_array) const = 0;
/**
* @brief Delete partition method
*
* This method is used to delete table partition.
*
* @param param, target partition to be deleted.
* NOTE: if param.table_name is empty, you must specify param.partition_name,
* else you can specify param.table_name and param.tag and let the param.partition_name be empty
*
* @return Indicate if partition is delete successfully.
*/
virtual Status
DropPartition(const PartitionParam& param) = 0;
};
} // namespace milvus

33
core/src/sdk/interface/ConnectionImpl.cpp
View File

@ -83,16 +83,16 @@ ConnectionImpl::CreateIndex(const IndexParam& index_param) {
}
Status
ConnectionImpl::Insert(const std::string& table_name, const std::vector<RowRecord>& record_array,
std::vector<int64_t>& id_array) {
return client_proxy_->Insert(table_name, record_array, id_array);
ConnectionImpl::Insert(const std::string& table_name, const std::string& partition_tag,
const std::vector<RowRecord>& record_array, std::vector<int64_t>& id_array) {
return client_proxy_->Insert(table_name, partition_tag, record_array, id_array);
}
Status
ConnectionImpl::Search(const std::string& table_name, const std::vector<RowRecord>& query_record_array,
const std::vector<Range>& query_range_array, int64_t topk, int64_t nprobe,
std::vector<TopKQueryResult>& topk_query_result_array) {
return client_proxy_->Search(table_name, query_record_array, query_range_array, topk, nprobe,
ConnectionImpl::Search(const std::string& table_name, const std::vector<std::string>& partiton_tags,
const std::vector<RowRecord>& query_record_array, const std::vector<Range>& query_range_array,
int64_t topk, int64_t nprobe, std::vector<TopKQueryResult>& topk_query_result_array) {
return client_proxy_->Search(table_name, partiton_tags, query_record_array, query_range_array, topk, nprobe,
topk_query_result_array);
}
@ -127,8 +127,8 @@ ConnectionImpl::DumpTaskTables() const {
}
Status
ConnectionImpl::DeleteByRange(Range& range, const std::string& table_name) {
return client_proxy_->DeleteByRange(range, table_name);
ConnectionImpl::DeleteByDate(const std::string& table_name, const Range& range) {
return client_proxy_->DeleteByDate(table_name, range);
}
Status
@ -146,4 +146,19 @@ ConnectionImpl::DropIndex(const std::string& table_name) const {
return client_proxy_->DropIndex(table_name);
}
Status
ConnectionImpl::CreatePartition(const PartitionParam& param) {
return client_proxy_->CreatePartition(param);
}
Status
ConnectionImpl::ShowPartitions(const std::string& table_name, PartitionList& partition_array) const {
return client_proxy_->ShowPartitions(table_name, partition_array);
}
Status
ConnectionImpl::DropPartition(const PartitionParam& param) {
return client_proxy_->DropPartition(param);
}
} // namespace milvus

19
core/src/sdk/interface/ConnectionImpl.h
View File

@ -56,13 +56,13 @@ class ConnectionImpl : public Connection {
CreateIndex(const IndexParam& index_param) override;
Status
Insert(const std::string& table_name, const std::vector<RowRecord>& record_array,
Insert(const std::string& table_name, const std::string& partition_tag, const std::vector<RowRecord>& record_array,
std::vector<int64_t>& id_array) override;
Status
Search(const std::string& table_name, const std::vector<RowRecord>& query_record_array,
const std::vector<Range>& query_range_array, int64_t topk, int64_t nprobe,
std::vector<TopKQueryResult>& topk_query_result_array) override;
Search(const std::string& table_name, const std::vector<std::string>& partiton_tags,
const std::vector<RowRecord>& query_record_array, const std::vector<Range>& query_range_array, int64_t topk,
int64_t nprobe, std::vector<TopKQueryResult>& topk_query_result_array) override;
Status
DescribeTable(const std::string& table_name, TableSchema& table_schema) override;
@ -86,7 +86,7 @@ class ConnectionImpl : public Connection {
DumpTaskTables() const override;
Status
DeleteByRange(Range& range, const std::string& table_name) override;
DeleteByDate(const std::string& table_name, const Range& range) override;
Status
PreloadTable(const std::string& table_name) const override;
@ -97,6 +97,15 @@ class ConnectionImpl : public Connection {
Status
DropIndex(const std::string& table_name) const override;
Status
CreatePartition(const PartitionParam& param) override;
Status
ShowPartitions(const std::string& table_name, PartitionList& partition_array) const override;
Status
DropPartition(const PartitionParam& param) override;
private:
std::shared_ptr<ClientProxy> client_proxy_;
};

145
core/src/server/Config.cpp
View File

@ -25,6 +25,7 @@
#include "config/YamlConfigMgr.h"
#include "server/Config.h"
#include "utils/CommonUtil.h"
#include "utils/StringHelpFunctions.h"
#include "utils/ValidationUtil.h"
namespace milvus {
@ -306,6 +307,7 @@ Config::ResetDefaultConfig() {
return s;
}
#ifdef MILVUS_GPU_VERSION
s = SetCacheConfigGpuCacheCapacity(CONFIG_CACHE_GPU_CACHE_CAPACITY_DEFAULT);
if (!s.ok()) {
return s;
@ -315,6 +317,7 @@ Config::ResetDefaultConfig() {
if (!s.ok()) {
return s;
}
#endif
s = SetCacheConfigCacheInsertData(CONFIG_CACHE_CACHE_INSERT_DATA_DEFAULT);
if (!s.ok()) {
@ -343,6 +346,11 @@ Config::ResetDefaultConfig() {
return s;
}
s = SetResourceConfigSearchResources(CONFIG_RESOURCE_SEARCH_RESOURCES_DEFAULT);
if (!s.ok()) {
return s;
}
s = SetResourceConfigIndexBuildDevice(CONFIG_RESOURCE_INDEX_BUILD_DEVICE_DEFAULT);
if (!s.ok()) {
return s;
@ -403,8 +411,7 @@ Status
Config::CheckServerConfigDeployMode(const std::string& value) {
if (value != "single" && value != "cluster_readonly" && value != "cluster_writable") {
return Status(SERVER_INVALID_ARGUMENT,
"server_config.deploy_mode is not one of "
"single, cluster_readonly, and cluster_writable.");
"server_config.deploy_mode is not one of single, cluster_readonly, and cluster_writable.");
}
return Status::OK();
}
@ -592,15 +599,15 @@ Config::CheckCacheConfigGpuCacheCapacity(const std::string& value) {
return Status(SERVER_INVALID_ARGUMENT, msg);
} else {
uint64_t gpu_cache_capacity = std::stoi(value) * GB;
int gpu_index;
Status s = GetResourceConfigIndexBuildDevice(gpu_index);
int device_id;
Status s = GetResourceConfigIndexBuildDevice(device_id);
if (!s.ok()) {
return s;
}
size_t gpu_memory;
if (!ValidationUtil::GetGpuMemory(gpu_index, gpu_memory).ok()) {
std::string msg = "Fail to get GPU memory for GPU device: " + std::to_string(gpu_index);
if (!ValidationUtil::GetGpuMemory(device_id, gpu_memory).ok()) {
std::string msg = "Fail to get GPU memory for GPU device: " + std::to_string(device_id);
return Status(SERVER_UNEXPECTED_ERROR, msg);
} else if (gpu_cache_capacity >= gpu_memory) {
std::string msg = "Invalid gpu cache capacity: " + value +
@ -689,29 +696,33 @@ Config::CheckResourceConfigMode(const std::string& value) {
}
Status
CheckGpuDevice(const std::string& value) {
CheckResource(const std::string& value) {
std::string s = value;
std::transform(s.begin(), s.end(), s.begin(), ::tolower);
#ifdef MILVUS_CPU_VERSION
if (s != "cpu") {
return Status(SERVER_INVALID_ARGUMENT, "Invalid CPU resource: " + s);
}
#else
const std::regex pat("gpu(\\d+)");
std::cmatch m;
if (!std::regex_match(value.c_str(), m, pat)) {
std::string msg = "Invalid gpu device: " + value +
". Possible reason: resource_config.search_resources does not match your hardware.";
const std::regex pat("cpu|gpu(\\d+)");
std::smatch m;
if (!std::regex_match(s, m, pat)) {
std::string msg = "Invalid search resource: " + value +
". Possible reason: resource_config.search_resources is not in the format of cpux or gpux";
return Status(SERVER_INVALID_ARGUMENT, msg);
}
int32_t gpu_index = std::stoi(value.substr(3));
if (!ValidationUtil::ValidateGpuIndex(gpu_index).ok()) {
std::string msg = "Invalid gpu device: " + value +
". Possible reason: resource_config.search_resources does not match your hardware.";
return Status(SERVER_INVALID_ARGUMENT, msg);
if (s.compare(0, 3, "gpu") == 0) {
int32_t gpu_index = std::stoi(s.substr(3));
if (!ValidationUtil::ValidateGpuIndex(gpu_index).ok()) {
std::string msg = "Invalid search resource: " + value +
". Possible reason: resource_config.search_resources does not match your hardware.";
return Status(SERVER_INVALID_ARGUMENT, msg);
}
}
#endif
return Status::OK();
}
@ -724,38 +735,20 @@ Config::CheckResourceConfigSearchResources(const std::vector<std::string>& value
return Status(SERVER_INVALID_ARGUMENT, msg);
}
bool cpu_found = false, gpu_found = false;
for (auto& device : value) {
if (device == "cpu") {
cpu_found = true;
continue;
for (auto& resource : value) {
auto status = CheckResource(resource);
if (!status.ok()) {
return Status(SERVER_INVALID_ARGUMENT, status.message());
}
if (CheckGpuDevice(device).ok()) {
gpu_found = true;
} else {
std::string msg = "Invalid search resource: " + device +
". Possible reason: resource_config.search_resources does not match your hardware.";
return Status(SERVER_INVALID_ARGUMENT, msg);
}
}
if (cpu_found && !gpu_found) {
std::string msg =
"Invalid search resource. Possible reason: resource_config.search_resources has only CPU resource.";
return Status(SERVER_INVALID_ARGUMENT, msg);
}
return Status::OK();
}
Status
Config::CheckResourceConfigIndexBuildDevice(const std::string& value) {
// if (value == "cpu") {
// return Status::OK();
// }
if (!CheckGpuDevice(value).ok()) {
std::string msg = "Invalid index build device: " + value +
". Possible reason: resource_config.index_build_device does not match your hardware.";
return Status(SERVER_INVALID_ARGUMENT, msg);
auto status = CheckResource(value);
if (!status.ok()) {
return Status(SERVER_INVALID_ARGUMENT, status.message());
}
return Status::OK();
}
@ -796,6 +789,22 @@ Config::GetConfigStr(const std::string& parent_key, const std::string& child_key
return value;
}
std::string
Config::GetConfigSequenceStr(const std::string& parent_key, const std::string& child_key, const std::string& delim,
const std::string& default_value) {
std::string value;
if (!GetConfigValueInMem(parent_key, child_key, value).ok()) {
std::vector<std::string> sequence = GetConfigNode(parent_key).GetSequence(child_key);
if (sequence.empty()) {
value = default_value;
} else {
server::StringHelpFunctions::MergeStringWithDelimeter(sequence, delim, value);
}
SetConfigValueInMem(parent_key, child_key, value);
}
return value;
}
Status
Config::GetServerConfigAddress(std::string& value) {
value = GetConfigStr(CONFIG_SERVER, CONFIG_SERVER_ADDRESS, CONFIG_SERVER_ADDRESS_DEFAULT);
@ -1019,8 +1028,10 @@ Config::GetResourceConfigMode(std::string& value) {
Status
Config::GetResourceConfigSearchResources(std::vector<std::string>& value) {
ConfigNode resource_config = GetConfigNode(CONFIG_RESOURCE);
value = resource_config.GetSequence(CONFIG_RESOURCE_SEARCH_RESOURCES);
std::string str =
GetConfigSequenceStr(CONFIG_RESOURCE, CONFIG_RESOURCE_SEARCH_RESOURCES,
CONFIG_RESOURCE_SEARCH_RESOURCES_DELIMITER, CONFIG_RESOURCE_SEARCH_RESOURCES_DEFAULT);
server::StringHelpFunctions::SplitStringByDelimeter(str, CONFIG_RESOURCE_SEARCH_RESOURCES_DELIMITER, value);
return CheckResourceConfigSearchResources(value);
}
@ -1033,10 +1044,10 @@ Config::GetResourceConfigIndexBuildDevice(int32_t& value) {
return s;
}
if (str != "cpu") {
value = std::stoi(str.substr(3));
if (str == "cpu") {
value = CPU_DEVICE_ID;
} else {
value = -1;
value = std::stoi(str.substr(3));
}
return Status::OK();
@ -1163,7 +1174,7 @@ Config::SetMetricConfigEnableMonitor(const std::string& value) {
return s;
}
SetConfigValueInMem(CONFIG_DB, CONFIG_METRIC_ENABLE_MONITOR, value);
SetConfigValueInMem(CONFIG_METRIC, CONFIG_METRIC_ENABLE_MONITOR, value);
return Status::OK();
}
@ -1174,7 +1185,7 @@ Config::SetMetricConfigCollector(const std::string& value) {
return s;
}
SetConfigValueInMem(CONFIG_DB, CONFIG_METRIC_COLLECTOR, value);
SetConfigValueInMem(CONFIG_METRIC, CONFIG_METRIC_COLLECTOR, value);
return Status::OK();
}
@ -1185,7 +1196,7 @@ Config::SetMetricConfigPrometheusPort(const std::string& value) {
return s;
}
SetConfigValueInMem(CONFIG_DB, CONFIG_METRIC_PROMETHEUS_PORT, value);
SetConfigValueInMem(CONFIG_METRIC, CONFIG_METRIC_PROMETHEUS_PORT, value);
return Status::OK();
}
@ -1197,7 +1208,7 @@ Config::SetCacheConfigCpuCacheCapacity(const std::string& value) {
return s;
}
SetConfigValueInMem(CONFIG_DB, CONFIG_CACHE_CPU_CACHE_CAPACITY, value);
SetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_CPU_CACHE_CAPACITY, value);
return Status::OK();
}
@ -1208,7 +1219,7 @@ Config::SetCacheConfigCpuCacheThreshold(const std::string& value) {
return s;
}
SetConfigValueInMem(CONFIG_DB, CONFIG_CACHE_CPU_CACHE_THRESHOLD, value);
SetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_CPU_CACHE_THRESHOLD, value);
return Status::OK();
}
@ -1219,7 +1230,7 @@ Config::SetCacheConfigGpuCacheCapacity(const std::string& value) {
return s;
}
SetConfigValueInMem(CONFIG_DB, CONFIG_CACHE_GPU_CACHE_CAPACITY, value);
SetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_GPU_CACHE_CAPACITY, value);
return Status::OK();
}
@ -1230,7 +1241,7 @@ Config::SetCacheConfigGpuCacheThreshold(const std::string& value) {
return s;
}
SetConfigValueInMem(CONFIG_DB, CONFIG_CACHE_GPU_CACHE_THRESHOLD, value);
SetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_GPU_CACHE_THRESHOLD, value);
return Status::OK();
}
@ -1241,7 +1252,7 @@ Config::SetCacheConfigCacheInsertData(const std::string& value) {
return s;
}
SetConfigValueInMem(CONFIG_DB, CONFIG_CACHE_CACHE_INSERT_DATA, value);
SetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_CACHE_INSERT_DATA, value);
return Status::OK();
}
@ -1253,7 +1264,7 @@ Config::SetEngineConfigUseBlasThreshold(const std::string& value) {
return s;
}
SetConfigValueInMem(CONFIG_DB, CONFIG_ENGINE_USE_BLAS_THRESHOLD, value);
SetConfigValueInMem(CONFIG_ENGINE, CONFIG_ENGINE_USE_BLAS_THRESHOLD, value);
return Status::OK();
}
@ -1264,7 +1275,7 @@ Config::SetEngineConfigOmpThreadNum(const std::string& value) {
return s;
}
SetConfigValueInMem(CONFIG_DB, CONFIG_ENGINE_OMP_THREAD_NUM, value);
SetConfigValueInMem(CONFIG_ENGINE, CONFIG_ENGINE_OMP_THREAD_NUM, value);
return Status::OK();
}
@ -1275,7 +1286,7 @@ Config::SetEngineConfigGpuSearchThreshold(const std::string& value) {
return s;
}
SetConfigValueInMem(CONFIG_DB, CONFIG_ENGINE_GPU_SEARCH_THRESHOLD, value);
SetConfigValueInMem(CONFIG_ENGINE, CONFIG_ENGINE_GPU_SEARCH_THRESHOLD, value);
return Status::OK();
}
@ -1287,7 +1298,21 @@ Config::SetResourceConfigMode(const std::string& value) {
return s;
}
SetConfigValueInMem(CONFIG_DB, CONFIG_RESOURCE_MODE, value);
SetConfigValueInMem(CONFIG_RESOURCE, CONFIG_RESOURCE_MODE, value);
return Status::OK();
}
Status
Config::SetResourceConfigSearchResources(const std::string& value) {
std::vector<std::string> res_vec;
server::StringHelpFunctions::SplitStringByDelimeter(value, CONFIG_RESOURCE_SEARCH_RESOURCES_DELIMITER, res_vec);
Status s = CheckResourceConfigSearchResources(res_vec);
if (!s.ok()) {
return s;
}
SetConfigValueInMem(CONFIG_RESOURCE, CONFIG_RESOURCE_SEARCH_RESOURCES, value);
return Status::OK();
}
@ -1298,7 +1323,7 @@ Config::SetResourceConfigIndexBuildDevice(const std::string& value) {
return s;
}
SetConfigValueInMem(CONFIG_DB, CONFIG_RESOURCE_INDEX_BUILD_DEVICE, value);
SetConfigValueInMem(CONFIG_RESOURCE, CONFIG_RESOURCE_INDEX_BUILD_DEVICE, value);
return Status::OK();
}

12
core/src/server/Config.h
View File

@ -92,12 +92,19 @@ static const char* CONFIG_RESOURCE = "resource_config";
static const char* CONFIG_RESOURCE_MODE = "mode";
static const char* CONFIG_RESOURCE_MODE_DEFAULT = "simple";
static const char* CONFIG_RESOURCE_SEARCH_RESOURCES = "search_resources";
static const char* CONFIG_RESOURCE_SEARCH_RESOURCES_DELIMITER = ",";
#ifdef MILVUS_CPU_VERSION
static const char* CONFIG_RESOURCE_SEARCH_RESOURCES_DEFAULT = "cpu";
#else
static const char* CONFIG_RESOURCE_SEARCH_RESOURCES_DEFAULT = "cpu,gpu0";
#endif
static const char* CONFIG_RESOURCE_INDEX_BUILD_DEVICE = "index_build_device";
#ifdef MILVUS_CPU_VERSION
static const char* CONFIG_RESOURCE_INDEX_BUILD_DEVICE_DEFAULT = "cpu";
#else
static const char* CONFIG_RESOURCE_INDEX_BUILD_DEVICE_DEFAULT = "gpu0";
#endif
const int32_t CPU_DEVICE_ID = -1;
class Config {
public:
@ -185,6 +192,9 @@ class Config {
std::string
GetConfigStr(const std::string& parent_key, const std::string& child_key, const std::string& default_value = "");
std::string
GetConfigSequenceStr(const std::string& parent_key, const std::string& child_key, const std::string& delim = ",",
const std::string& default_value = "");
public:
/* server config */
@ -306,6 +316,8 @@ class Config {
Status
SetResourceConfigMode(const std::string& value);
Status
SetResourceConfigSearchResources(const std::string& value);
Status
SetResourceConfigIndexBuildDevice(const std::string& value);
private:

36
core/src/server/grpc_impl/GrpcRequestHandler.cpp
View File

@ -150,9 +150,9 @@ GrpcRequestHandler::Cmd(::grpc::ServerContext* context, const ::milvus::grpc::Co
}
::grpc::Status
GrpcRequestHandler::DeleteByRange(::grpc::ServerContext* context, const ::milvus::grpc::DeleteByRangeParam* request,
::milvus::grpc::Status* response) {
BaseTaskPtr task_ptr = DeleteByRangeTask::Create(request);
GrpcRequestHandler::DeleteByDate(::grpc::ServerContext* context, const ::milvus::grpc::DeleteByDateParam* request,
::milvus::grpc::Status* response) {
BaseTaskPtr task_ptr = DeleteByDateTask::Create(request);
::milvus::grpc::Status grpc_status;
GrpcRequestScheduler::ExecTask(task_ptr, &grpc_status);
response->set_error_code(grpc_status.error_code());
@ -193,6 +193,36 @@ GrpcRequestHandler::DropIndex(::grpc::ServerContext* context, const ::milvus::gr
return ::grpc::Status::OK;
}
::grpc::Status
GrpcRequestHandler::CreatePartition(::grpc::ServerContext* context, const ::milvus::grpc::PartitionParam* request,
::milvus::grpc::Status* response) {
BaseTaskPtr task_ptr = CreatePartitionTask::Create(request);
GrpcRequestScheduler::ExecTask(task_ptr, response);
return ::grpc::Status::OK;
}
::grpc::Status
GrpcRequestHandler::ShowPartitions(::grpc::ServerContext* context, const ::milvus::grpc::TableName* request,
::milvus::grpc::PartitionList* response) {
BaseTaskPtr task_ptr = ShowPartitionsTask::Create(request->table_name(), response);
::milvus::grpc::Status grpc_status;
GrpcRequestScheduler::ExecTask(task_ptr, &grpc_status);
response->mutable_status()->set_reason(grpc_status.reason());
response->mutable_status()->set_error_code(grpc_status.error_code());
return ::grpc::Status::OK;
}
::grpc::Status
GrpcRequestHandler::DropPartition(::grpc::ServerContext* context, const ::milvus::grpc::PartitionParam* request,
::milvus::grpc::Status* response) {
BaseTaskPtr task_ptr = DropPartitionTask::Create(request);
::milvus::grpc::Status grpc_status;
GrpcRequestScheduler::ExecTask(task_ptr, &grpc_status);
response->set_reason(grpc_status.reason());
response->set_error_code(grpc_status.error_code());
return ::grpc::Status::OK;
}
} // namespace grpc
} // namespace server
} // namespace milvus

406
core/src/server/grpc_impl/GrpcRequestHandler.h
View File

@ -28,296 +28,168 @@ namespace server {
namespace grpc {
class GrpcRequestHandler final : public ::milvus::grpc::MilvusService::Service {
public:
/**
* @brief Create table method
*
* This method is used to create table
*
* @param context, add context for every RPC
* @param request, used to provide table information to be created.
* @param response, used to get the status
*
* @return status
*
* @param request
* @param response
* @param context
*/
// *
// @brief This method is used to create table
//
// @param TableSchema, use to provide table information to be created.
//
// @return Status
::grpc::Status
CreateTable(::grpc::ServerContext* context, const ::milvus::grpc::TableSchema* request,
::milvus::grpc::Status* response) override;
/**
* @brief Test table existence method
*
* This method is used to test table existence.
*
* @param context, add context for every RPC
* @param request, table name is going to be tested.
* @param response, get the bool reply of hastable
*
* @return status
*
* @param request
* @param response
* @param context
*/
// *
// @brief This method is used to test table existence.
//
// @param TableName, table name is going to be tested.
//
// @return BoolReply
::grpc::Status
HasTable(::grpc::ServerContext* context, const ::milvus::grpc::TableName* request,
::milvus::grpc::BoolReply* response) override;
/**
* @brief Drop table method
*
* This method is used to drop table.
*
* @param context, add context for every RPC
* @param request, table name is going to be deleted.
* @param response, get the status of droptable
*
* @return status
*
* @param request
* @param response
* @param context
*/
::grpc::Status
DropTable(::grpc::ServerContext* context, const ::milvus::grpc::TableName* request,
::milvus::grpc::Status* response) override;
/**
* @brief build index by table method
*
* This method is used to build index by table in sync.
*
* @param context, add context for every RPC
* @param request, table name is going to be built index.
* @param response, get the status of buildindex
*
* @return status
*
* @param request
* @param response
* @param context
*/
::grpc::Status
CreateIndex(::grpc::ServerContext* context, const ::milvus::grpc::IndexParam* request,
::milvus::grpc::Status* response) override;
/**
* @brief Insert vector array to table
*
* This method is used to insert vector array to table.
*
* @param context, add context for every RPC
* @param request, table_name is inserted.
* @param response, vector array is inserted.
*
* @return status
*
* @param context
* @param request
* @param response
*/
::grpc::Status
Insert(::grpc::ServerContext* context, const ::milvus::grpc::InsertParam* request,
::milvus::grpc::VectorIds* response) override;
/**
* @brief Query vector
*
* This method is used to query vector in table.
*
* @param context, add context for every RPC
* @param request:
* table_name, table_name is queried.
* query_record_array, all vector are going to be queried.
* query_range_array, optional ranges for conditional search. If not specified, search whole table
* topk, how many similarity vectors will be searched.
*
* @param writer, write query result array.
*
* @return status
*
* @param context
* @param request
* @param writer
*/
::grpc::Status
Search(::grpc::ServerContext* context, const ::milvus::grpc::SearchParam* request,
::milvus::grpc::TopKQueryResultList* response) override;
/**
* @brief Internal use query interface
*
* This method is used to query vector in specified files.
*
* @param context, add context for every RPC
* @param request:
* file_id_array, specified files id array, queried.
* query_record_array, all vector are going to be queried.
* query_range_array, optional ranges for conditional search. If not specified, search whole table
* topk, how many similarity vectors will be searched.
*
* @param writer, write query result array.
*
* @return status
*
* @param context
* @param request
* @param writer
*/
::grpc::Status
SearchInFiles(::grpc::ServerContext* context, const ::milvus::grpc::SearchInFilesParam* request,
::milvus::grpc::TopKQueryResultList* response) override;
/**
* @brief Get table schema
*
* This method is used to get table schema.
*
* @param context, add context for every RPC
* @param request, target table name.
* @param response, table schema
*
* @return status
*
* @param context
* @param request
* @param response
*/
// *
// @brief This method is used to get table schema.
//
// @param TableName, target table name.
//
// @return TableSchema
::grpc::Status
DescribeTable(::grpc::ServerContext* context, const ::milvus::grpc::TableName* request,
::milvus::grpc::TableSchema* response) override;
/**
* @brief Get table row count
*
* This method is used to get table row count.
*
* @param context, add context for every RPC
* @param request, target table name.
* @param response, table row count
*
* @return table row count
*
* @param request
* @param response
* @param context
*/
// *
// @brief This method is used to get table schema.
//
// @param TableName, target table name.
//
// @return TableRowCount
::grpc::Status
CountTable(::grpc::ServerContext* context, const ::milvus::grpc::TableName* request,
::milvus::grpc::TableRowCount* response) override;
/**
* @brief List all tables in database
*
* This method is used to list all tables.
*
* @param context, add context for every RPC
* @param request, show table command, usually not use
* @param writer, write tables to client
*
* @return status
*
* @param context
* @param request
* @param writer
*/
// *
// @brief This method is used to list all tables.
//
// @param Command, dummy parameter.
//
// @return TableNameList
::grpc::Status
ShowTables(::grpc::ServerContext* context, const ::milvus::grpc::Command* request,
::milvus::grpc::TableNameList* response) override;
/**
* @brief Give the server status
*
*
* This method is used to give the server status.
* @param context, add context for every RPC
* @param request, give server command
* @param response, server status
*
* @return status
*
* @param context
* @param request
* @param response
*/
// *
// @brief This method is used to delete table.
//
// @param TableName, table name is going to be deleted.
//
// @return TableNameList
::grpc::Status
Cmd(::grpc::ServerContext* context, const ::milvus::grpc::Command* request,
::milvus::grpc::StringReply* response) override;
/**
* @brief delete table by range
*
* This method is used to delete table by range.
* @param context, add context for every RPC
* @param request, table name and range
* @param response, status
*
* @return status
*
* @param context
* @param request
* @param response
*/
DropTable(::grpc::ServerContext* context, const ::milvus::grpc::TableName* request,
::milvus::grpc::Status* response) override;
// *
// @brief This method is used to build index by table in sync mode.
//
// @param IndexParam, index paramters.
//
// @return Status
::grpc::Status
DeleteByRange(::grpc::ServerContext* context, const ::milvus::grpc::DeleteByRangeParam* request,
::milvus::grpc::Status* response) override;
/**
* @brief preload table
*
* This method is used to preload table.
* @param context, add context for every RPC
* @param request, table name
* @param response, status
*
* @return status
*
* @param context
* @param request
* @param response
*/
::grpc::Status
PreloadTable(::grpc::ServerContext* context, const ::milvus::grpc::TableName* request,
::milvus::grpc::Status* response) override;
/**
* @brief Describe index
*
* This method is used to describe index.
* @param context, add context for every RPC
* @param request, table name
* @param response, index informations
*
* @return status
*
* @param context
* @param request
* @param response
*/
CreateIndex(::grpc::ServerContext* context, const ::milvus::grpc::IndexParam* request,
::milvus::grpc::Status* response) override;
// *
// @brief This method is used to describe index
//
// @param TableName, target table name.
//
// @return IndexParam
::grpc::Status
DescribeIndex(::grpc::ServerContext* context, const ::milvus::grpc::TableName* request,
::milvus::grpc::IndexParam* response) override;
/**
* @brief Drop index
*
* This method is used to drop index.
* @param context, add context for every RPC
* @param request, table name
* @param response, status
*
* @return status
*
* @param context
* @param request
* @param response
*/
// *
// @brief This method is used to drop index
//
// @param TableName, target table name.
//
// @return Status
::grpc::Status
DropIndex(::grpc::ServerContext* context, const ::milvus::grpc::TableName* request,
::milvus::grpc::Status* response) override;
// *
// @brief This method is used to create partition
//
// @param PartitionParam, partition parameters.
//
// @return Status
::grpc::Status
CreatePartition(::grpc::ServerContext* context, const ::milvus::grpc::PartitionParam* request,
::milvus::grpc::Status* response) override;
// *
// @brief This method is used to show partition information
//
// @param TableName, target table name.
//
// @return PartitionList
::grpc::Status
ShowPartitions(::grpc::ServerContext* context, const ::milvus::grpc::TableName* request,
::milvus::grpc::PartitionList* response) override;
// *
// @brief This method is used to drop partition
//
// @param PartitionName, target partition name.
//
// @return Status
::grpc::Status
DropPartition(::grpc::ServerContext* context, const ::milvus::grpc::PartitionParam* request,
::milvus::grpc::Status* response) override;
// *
// @brief This method is used to add vector array to table.
//
// @param InsertParam, insert parameters.
//
// @return VectorIds
::grpc::Status
Insert(::grpc::ServerContext* context, const ::milvus::grpc::InsertParam* request,
::milvus::grpc::VectorIds* response) override;
// *
// @brief This method is used to query vector in table.
//
// @param SearchParam, search parameters.
//
// @return TopKQueryResultList
::grpc::Status
Search(::grpc::ServerContext* context, const ::milvus::grpc::SearchParam* request,
::milvus::grpc::TopKQueryResultList* response) override;
// *
// @brief This method is used to query vector in specified files.
//
// @param SearchInFilesParam, search in files paremeters.
//
// @return TopKQueryResultList
::grpc::Status
SearchInFiles(::grpc::ServerContext* context, const ::milvus::grpc::SearchInFilesParam* request,
::milvus::grpc::TopKQueryResultList* response) override;
// *
// @brief This method is used to give the server status.
//
// @param Command, command string
//
// @return StringReply
::grpc::Status
Cmd(::grpc::ServerContext* context, const ::milvus::grpc::Command* request,
::milvus::grpc::StringReply* response) override;
// *
// @brief This method is used to delete vector by date range
//
// @param DeleteByDateParam, delete parameters.
//
// @return status
::grpc::Status
DeleteByDate(::grpc::ServerContext* context, const ::milvus::grpc::DeleteByDateParam* request,
::milvus::grpc::Status* response) override;
// *
// @brief This method is used to preload table
//
// @param TableName, target table name.
//
// @return Status
::grpc::Status
PreloadTable(::grpc::ServerContext* context, const ::milvus::grpc::TableName* request,
::milvus::grpc::Status* response) override;
};
} // namespace grpc

166
core/src/server/grpc_impl/GrpcRequestTask.cpp
View File

@ -366,7 +366,7 @@ DropTableTask::OnExecute() {
// step 3: Drop table
std::vector<DB_DATE> dates;
status = DBWrapper::DB()->DeleteTable(table_name_, dates);
status = DBWrapper::DB()->DropTable(table_name_, dates);
if (!status.ok()) {
return status;
}
@ -505,7 +505,8 @@ InsertTask::OnExecute() {
memcpy(target_data, src_data, static_cast<size_t>(sizeof(int64_t) * insert_param_->row_id_array_size()));
}
status = DBWrapper::DB()->InsertVectors(insert_param_->table_name(), vec_count, vec_f.data(), vec_ids);
status = DBWrapper::DB()->InsertVectors(insert_param_->table_name(), insert_param_->partition_tag(), vec_count,
vec_f.data(), vec_ids);
rc.ElapseFromBegin("add vectors to engine");
if (!status.ok()) {
return status;
@ -637,7 +638,8 @@ SearchTask::OnExecute() {
rc.RecordSection("prepare vector data");
// step 6: search vectors
engine::QueryResults results;
engine::ResultIds result_ids;
engine::ResultDistances result_distances;
auto record_count = (uint64_t)search_param_->query_record_array().size();
#ifdef MILVUS_ENABLE_PROFILING
@ -647,11 +649,21 @@ SearchTask::OnExecute() {
#endif
if (file_id_array_.empty()) {
status =
DBWrapper::DB()->Query(table_name_, (size_t)top_k, record_count, nprobe, vec_f.data(), dates, results);
std::vector<std::string> partition_tags;
for (size_t i = 0; i < search_param_->partition_tag_array_size(); i++) {
partition_tags.emplace_back(search_param_->partition_tag_array(i));
}
status = ValidationUtil::ValidatePartitionTags(partition_tags);
if (!status.ok()) {
return status;
}
status = DBWrapper::DB()->Query(table_name_, partition_tags, (size_t)top_k, record_count, nprobe,
vec_f.data(), dates, result_ids, result_distances);
} else {
status = DBWrapper::DB()->Query(table_name_, file_id_array_, (size_t)top_k, record_count, nprobe,
vec_f.data(), dates, results);
status = DBWrapper::DB()->QueryByFileID(table_name_, file_id_array_, (size_t)top_k, record_count, nprobe,
vec_f.data(), dates, result_ids, result_distances);
}
#ifdef MILVUS_ENABLE_PROFILING
@ -663,23 +675,20 @@ SearchTask::OnExecute() {
return status;
}
if (results.empty()) {
if (result_ids.empty()) {
return Status::OK(); // empty table
}
if (results.size() != record_count) {
std::string msg = "Search " + std::to_string(record_count) + " vectors but only return " +
std::to_string(results.size()) + " results";
return Status(SERVER_ILLEGAL_SEARCH_RESULT, msg);
}
size_t result_k = result_ids.size() / record_count;
// step 7: construct result array
for (auto& result : results) {
for (size_t i = 0; i < record_count; i++) {
::milvus::grpc::TopKQueryResult* topk_query_result = topk_result_list->add_topk_query_result();
for (auto& pair : result) {
for (size_t j = 0; j < result_k; j++) {
::milvus::grpc::QueryResult* grpc_result = topk_query_result->add_query_result_arrays();
grpc_result->set_id(pair.first);
grpc_result->set_distance(pair.second);
size_t idx = i * result_k + j;
grpc_result->set_id(result_ids[idx]);
grpc_result->set_distance(result_distances[idx]);
}
}
@ -759,22 +768,22 @@ CmdTask::OnExecute() {
}
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
DeleteByRangeTask::DeleteByRangeTask(const ::milvus::grpc::DeleteByRangeParam* delete_by_range_param)
DeleteByDateTask::DeleteByDateTask(const ::milvus::grpc::DeleteByDateParam* delete_by_range_param)
: GrpcBaseTask(DDL_DML_TASK_GROUP), delete_by_range_param_(delete_by_range_param) {
}
BaseTaskPtr
DeleteByRangeTask::Create(const ::milvus::grpc::DeleteByRangeParam* delete_by_range_param) {
DeleteByDateTask::Create(const ::milvus::grpc::DeleteByDateParam* delete_by_range_param) {
if (delete_by_range_param == nullptr) {
SERVER_LOG_ERROR << "grpc input is null!";
return nullptr;
}
return std::shared_ptr<GrpcBaseTask>(new DeleteByRangeTask(delete_by_range_param));
return std::shared_ptr<GrpcBaseTask>(new DeleteByDateTask(delete_by_range_param));
}
Status
DeleteByRangeTask::OnExecute() {
DeleteByDateTask::OnExecute() {
try {
TimeRecorder rc("DeleteByRangeTask");
@ -815,7 +824,7 @@ DeleteByRangeTask::OnExecute() {
std::string fname = "/tmp/search_nq_" + this->delete_by_range_param_->table_name() + ".profiling";
ProfilerStart(fname.c_str());
#endif
status = DBWrapper::DB()->DeleteTable(table_name, dates);
status = DBWrapper::DB()->DropTable(table_name, dates);
if (!status.ok()) {
return status;
}
@ -946,6 +955,119 @@ DropIndexTask::OnExecute() {
return Status::OK();
}
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
CreatePartitionTask::CreatePartitionTask(const ::milvus::grpc::PartitionParam* partition_param)
: GrpcBaseTask(DDL_DML_TASK_GROUP), partition_param_(partition_param) {
}
BaseTaskPtr
CreatePartitionTask::Create(const ::milvus::grpc::PartitionParam* partition_param) {
if (partition_param == nullptr) {
SERVER_LOG_ERROR << "grpc input is null!";
return nullptr;
}
return std::shared_ptr<GrpcBaseTask>(new CreatePartitionTask(partition_param));
}
Status
CreatePartitionTask::OnExecute() {
TimeRecorder rc("CreatePartitionTask");
try {
// step 1: check arguments
auto status = ValidationUtil::ValidateTableName(partition_param_->table_name());
if (!status.ok()) {
return status;
}
status = ValidationUtil::ValidateTableName(partition_param_->partition_name());
if (!status.ok()) {
return status;
}
status = ValidationUtil::ValidatePartitionTags({partition_param_->tag()});
if (!status.ok()) {
return status;
}
// step 2: create partition
status = DBWrapper::DB()->CreatePartition(partition_param_->table_name(), partition_param_->partition_name(),
partition_param_->tag());
if (!status.ok()) {
// partition could exist
if (status.code() == DB_ALREADY_EXIST) {
return Status(SERVER_INVALID_TABLE_NAME, status.message());
}
return status;
}
} catch (std::exception& ex) {
return Status(SERVER_UNEXPECTED_ERROR, ex.what());
}
rc.ElapseFromBegin("totally cost");
return Status::OK();
}
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
ShowPartitionsTask::ShowPartitionsTask(const std::string& table_name, ::milvus::grpc::PartitionList* partition_list)
: GrpcBaseTask(INFO_TASK_GROUP), table_name_(table_name), partition_list_(partition_list) {
}
BaseTaskPtr
ShowPartitionsTask::Create(const std::string& table_name, ::milvus::grpc::PartitionList* partition_list) {
return std::shared_ptr<GrpcBaseTask>(new ShowPartitionsTask(table_name, partition_list));
}
Status
ShowPartitionsTask::OnExecute() {
std::vector<engine::meta::TableSchema> schema_array;
auto statuts = DBWrapper::DB()->ShowPartitions(table_name_, schema_array);
if (!statuts.ok()) {
return statuts;
}
for (auto& schema : schema_array) {
::milvus::grpc::PartitionParam* param = partition_list_->add_partition_array();
param->set_table_name(schema.owner_table_);
param->set_partition_name(schema.table_id_);
param->set_tag(schema.partition_tag_);
}
return Status::OK();
}
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
DropPartitionTask::DropPartitionTask(const ::milvus::grpc::PartitionParam* partition_param)
: GrpcBaseTask(DDL_DML_TASK_GROUP), partition_param_(partition_param) {
}
BaseTaskPtr
DropPartitionTask::Create(const ::milvus::grpc::PartitionParam* partition_param) {
return std::shared_ptr<GrpcBaseTask>(new DropPartitionTask(partition_param));
}
Status
DropPartitionTask::OnExecute() {
if (!partition_param_->partition_name().empty()) {
auto status = ValidationUtil::ValidateTableName(partition_param_->partition_name());
if (!status.ok()) {
return status;
}
return DBWrapper::DB()->DropPartition(partition_param_->partition_name());
} else {
auto status = ValidationUtil::ValidateTableName(partition_param_->table_name());
if (!status.ok()) {
return status;
}
status = ValidationUtil::ValidatePartitionTags({partition_param_->tag()});
if (!status.ok()) {
return status;
}
return DBWrapper::DB()->DropPartitionByTag(partition_param_->table_name(), partition_param_->tag());
}
}
} // namespace grpc
} // namespace server
} // namespace milvus

57
core/src/server/grpc_impl/GrpcRequestTask.h
View File

@ -203,19 +203,19 @@ class CmdTask : public GrpcBaseTask {
};
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
class DeleteByRangeTask : public GrpcBaseTask {
class DeleteByDateTask : public GrpcBaseTask {
public:
static BaseTaskPtr
Create(const ::milvus::grpc::DeleteByRangeParam* delete_by_range_param);
Create(const ::milvus::grpc::DeleteByDateParam* delete_by_range_param);
protected:
explicit DeleteByRangeTask(const ::milvus::grpc::DeleteByRangeParam* delete_by_range_param);
explicit DeleteByDateTask(const ::milvus::grpc::DeleteByDateParam* delete_by_range_param);
Status
OnExecute() override;
private:
const ::milvus::grpc::DeleteByRangeParam* delete_by_range_param_;
const ::milvus::grpc::DeleteByDateParam* delete_by_range_param_;
};
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
@ -267,6 +267,55 @@ class DropIndexTask : public GrpcBaseTask {
std::string table_name_;
};
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
class CreatePartitionTask : public GrpcBaseTask {
public:
static BaseTaskPtr
Create(const ::milvus::grpc::PartitionParam* partition_param);
protected:
explicit CreatePartitionTask(const ::milvus::grpc::PartitionParam* partition_param);
Status
OnExecute() override;
private:
const ::milvus::grpc::PartitionParam* partition_param_;
};
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
class ShowPartitionsTask : public GrpcBaseTask {
public:
static BaseTaskPtr
Create(const std::string& table_name, ::milvus::grpc::PartitionList* partition_list);
protected:
ShowPartitionsTask(const std::string& table_name, ::milvus::grpc::PartitionList* partition_list);
Status
OnExecute() override;
private:
std::string table_name_;
::milvus::grpc::PartitionList* partition_list_;
};
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
class DropPartitionTask : public GrpcBaseTask {
public:
static BaseTaskPtr
Create(const ::milvus::grpc::PartitionParam* partition_param);
protected:
explicit DropPartitionTask(const ::milvus::grpc::PartitionParam* partition_param);
Status
OnExecute() override;
private:
const ::milvus::grpc::PartitionParam* partition_param_;
};
} // namespace grpc
} // namespace server
} // namespace milvus

2
core/src/utils/Json.h
View File

@ -16,7 +16,7 @@
// under the License.
#pragma once
#include "external/nlohmann/json.hpp"
#include "nlohmann/json.hpp"
namespace milvus {
using json = nlohmann::json;

2
core/src/utils/Log.h
View File

@ -17,7 +17,7 @@
#pragma once
#include "external/easyloggingpp/easylogging++.h"
#include "easyloggingpp/easylogging++.h"
namespace milvus {

2
core/src/utils/LogUtil.h
View File

@ -17,7 +17,7 @@
#pragma once
#include "external/easyloggingpp/easylogging++.h"
#include "easyloggingpp/easylogging++.h"
#include "utils/Status.h"
#include <sstream>

62
core/src/utils/StringHelpFunctions.cpp
View File

@ -17,6 +17,7 @@
#include "utils/StringHelpFunctions.h"
#include <regex>
#include <string>
namespace milvus {
@ -39,39 +40,53 @@ StringHelpFunctions::TrimStringQuote(std::string& string, const std::string& qou
}
}
Status
void
StringHelpFunctions::SplitStringByDelimeter(const std::string& str, const std::string& delimeter,
std::vector<std::string>& result) {
if (str.empty()) {
return Status::OK();
return;
}
size_t last = 0;
size_t index = str.find_first_of(delimeter, last);
while (index != std::string::npos) {
result.emplace_back(str.substr(last, index - last));
last = index + 1;
index = str.find_first_of(delimeter, last);
size_t prev = 0, pos = 0;
while (true) {
pos = str.find_first_of(delimeter, prev);
if (pos == std::string::npos) {
result.emplace_back(str.substr(prev));
break;
} else {
result.emplace_back(str.substr(prev, pos - prev));
prev = pos + 1;
}
}
if (index - last > 0) {
std::string temp = str.substr(last);
result.emplace_back(temp);
}
void
StringHelpFunctions::MergeStringWithDelimeter(const std::vector<std::string>& strs, const std::string& delimeter,
std::string& result) {
if (strs.empty()) {
result = "";
return;
}
return Status::OK();
result = strs[0];
for (size_t i = 1; i < strs.size(); i++) {
result = result + delimeter + strs[i];
}
}
Status
StringHelpFunctions::SplitStringByQuote(const std::string& str, const std::string& delimeter, const std::string& quote,
std::vector<std::string>& result) {
if (quote.empty()) {
return SplitStringByDelimeter(str, delimeter, result);
SplitStringByDelimeter(str, delimeter, result);
return Status::OK();
}
size_t last = 0;
size_t index = str.find_first_of(quote, last);
if (index == std::string::npos) {
return SplitStringByDelimeter(str, delimeter, result);
SplitStringByDelimeter(str, delimeter, result);
return Status::OK();
}
std::string process_str = str;
@ -116,11 +131,28 @@ StringHelpFunctions::SplitStringByQuote(const std::string& str, const std::strin
}
if (!process_str.empty()) {
return SplitStringByDelimeter(process_str, delimeter, result);
SplitStringByDelimeter(process_str, delimeter, result);
}
return Status::OK();
}
bool
StringHelpFunctions::IsRegexMatch(const std::string& target_str, const std::string& pattern_str) {
// if target_str equals pattern_str, return true
if (target_str == pattern_str) {
return true;
}
// regex match
std::regex pattern(pattern_str);
std::smatch results;
if (std::regex_search(target_str, results, pattern)) {
return true;
} else {
return false;
}
}
} // namespace server
} // namespace milvus

10
core/src/utils/StringHelpFunctions.h
View File

@ -43,9 +43,12 @@ class StringHelpFunctions {
// ,b, | b |
// ,, | |
// a a
static Status
static void
SplitStringByDelimeter(const std::string& str, const std::string& delimeter, std::vector<std::string>& result);
static void
MergeStringWithDelimeter(const std::vector<std::string>& strs, const std::string& delimeter, std::string& result);
// assume the table has two columns, quote='\"', delimeter=','
// a,b a | b
// "aa,gg,yy",b aa,gg,yy | b
@ -56,6 +59,11 @@ class StringHelpFunctions {
static Status
SplitStringByQuote(const std::string& str, const std::string& delimeter, const std::string& quote,
std::vector<std::string>& result);
// std regex match function
// regex grammar reference: http://www.cplusplus.com/reference/regex/ECMAScript/
static bool
IsRegexMatch(const std::string& target_str, const std::string& pattern);
};
} // namespace server

13
core/src/utils/ValidationUtil.cpp
View File

@ -168,6 +168,19 @@ ValidationUtil::ValidateSearchNprobe(int64_t nprobe, const engine::meta::TableSc
return Status::OK();
}
Status
ValidationUtil::ValidatePartitionTags(const std::vector<std::string>& partition_tags) {
for (auto& tag : partition_tags) {
if (tag.empty()) {
std::string msg = "Invalid partition tag: " + tag + ". " + "Partition tag should not be empty.";
SERVER_LOG_ERROR << msg;
return Status(SERVER_INVALID_NPROBE, msg);
}
}
return Status::OK();
}
Status
ValidationUtil::ValidateGpuIndex(uint32_t gpu_index) {
#ifdef MILVUS_GPU_VERSION

4
core/src/utils/ValidationUtil.h
View File

@ -21,6 +21,7 @@
#include "utils/Status.h"
#include <string>
#include <vector>
namespace milvus {
namespace server {
@ -54,6 +55,9 @@ class ValidationUtil {
static Status
ValidateSearchNprobe(int64_t nprobe, const engine::meta::TableSchema& table_schema);
static Status
ValidatePartitionTags(const std::vector<std::string>& partition_tags);
static Status
ValidateGpuIndex(uint32_t gpu_index);

8
core/src/wrapper/VecIndex.cpp
View File

@ -145,6 +145,10 @@ GetVecIndexFactory(const IndexType& type, const Config& cfg) {
index = std::make_shared<knowhere::GPUIVFPQ>(gpu_device);
break;
}
case IndexType::FAISS_IVFPQ_MIX: {
index = std::make_shared<knowhere::GPUIVFPQ>(gpu_device);
return std::make_shared<IVFMixIndex>(index, IndexType::FAISS_IVFPQ_MIX);
}
case IndexType::FAISS_IVFSQ8_MIX: {
index = std::make_shared<knowhere::GPUIVFSQ>(gpu_device);
return std::make_shared<IVFMixIndex>(index, IndexType::FAISS_IVFSQ8_MIX);
@ -168,10 +172,6 @@ GetVecIndexFactory(const IndexType& type, const Config& cfg) {
index = std::make_shared<knowhere::NSG>(gpu_device);
break;
}
case IndexType::FAISS_IVFPQ_MIX: {
index = std::make_shared<knowhere::GPUIVFPQ>(gpu_device);
return std::make_shared<IVFMixIndex>(index, IndexType::FAISS_IVFPQ_MIX);
}
default: { return nullptr; }
}
return std::make_shared<VecIndexImpl>(index, type);

0
core/src/external/easyloggingpp/easylogging++.cc → core/thirdparty/easyloggingpp/easylogging++.cc vendored
View File

0
core/src/external/easyloggingpp/easylogging++.h → core/thirdparty/easyloggingpp/easylogging++.h vendored
View File

0
core/src/external/nlohmann/json.hpp → core/thirdparty/nlohmann/json.hpp vendored
View File

1
core/thirdparty/versions.txt vendored
View File

@ -9,6 +9,5 @@ LIBUNWIND_VERSION=1.3.1
GPERFTOOLS_VERSION=2.7
GRPC_VERSION=master
ZLIB_VERSION=v1.2.11
MKL_VERSION=2019.5.281
# vim: set filetype=sh:

16
core/unittest/CMakeLists.txt
View File

@ -23,6 +23,7 @@ endforeach ()
include_directories(${MILVUS_SOURCE_DIR})
include_directories(${MILVUS_ENGINE_SRC})
include_directories(${MILVUS_THIRDPARTY_SRC})
include_directories(${CMAKE_CURRENT_SOURCE_DIR})
aux_source_directory(${MILVUS_ENGINE_SRC}/cache cache_files)
@ -57,11 +58,11 @@ set(scheduler_files
${scheduler_optimizer_files}
)
aux_source_directory(${MILVUS_ENGINE_SRC}/external/easyloggingpp external_easyloggingpp_files)
aux_source_directory(${MILVUS_ENGINE_SRC}/external/nlohmann external_nlohmann_files)
set(external_files
${external_easyloggingpp_files}
${external_nlohmann_files}
aux_source_directory(${MILVUS_THIRDPARTY_SRC}/easyloggingpp thirdparty_easyloggingpp_files)
aux_source_directory(${MILVUS_THIRDPARTY_SRC}/nlohmann thirdparty_nlohmann_files)
set(thirdparty_files
${thirdparty_easyloggingpp_files}
${thirdparty_nlohmann_files}
)
aux_source_directory(${MILVUS_ENGINE_SRC}/server server_files)
@ -77,8 +78,9 @@ set(helper_files
${MILVUS_ENGINE_SRC}/utils/CommonUtil.cpp
${MILVUS_ENGINE_SRC}/utils/TimeRecorder.cpp
${MILVUS_ENGINE_SRC}/utils/Status.cpp
${MILVUS_ENGINE_SRC}/utils/StringHelpFunctions.cpp
${MILVUS_ENGINE_SRC}/utils/ValidationUtil.cpp
${MILVUS_ENGINE_SRC}/external/easyloggingpp/easylogging++.cc
${MILVUS_THIRDPARTY_SRC}/easyloggingpp/easylogging++.cc
)
set(common_files
@ -89,7 +91,7 @@ set(common_files
${db_insert_files}
${db_meta_files}
${metrics_files}
${external_files}
${thirdparty_files}
${scheduler_files}
${wrapper_files}
${helper_files}

Some files were not shown because too many files have changed in this diff Show More