diff --git a/internal/core/src/index/CMakeLists.txt b/internal/core/src/index/CMakeLists.txt index 084ea422d2..0180bae81f 100644 --- a/internal/core/src/index/CMakeLists.txt +++ b/internal/core/src/index/CMakeLists.txt @@ -94,8 +94,8 @@ endif () set(INDEX_INCLUDE_DIRS ${INDEX_INCLUDE_DIRS} PARENT_SCOPE) if (KNOWHERE_BUILD_TESTS) - #set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DELPP_DISABLE_LOGS") - #add_subdirectory(unittest) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DELPP_DISABLE_LOGS") + add_subdirectory(unittest) endif () config_summary() diff --git a/internal/core/src/index/unittest/CMakeLists.txt b/internal/core/src/index/unittest/CMakeLists.txt index 2df08a55c1..57112f4202 100644 --- a/internal/core/src/index/unittest/CMakeLists.txt +++ b/internal/core/src/index/unittest/CMakeLists.txt @@ -82,105 +82,18 @@ set(faiss_srcs ${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_offset_index/IndexIVF_NM.cpp ) if (MILVUS_GPU_VERSION) -set(faiss_srcs ${faiss_srcs} - ${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_index/ConfAdapter.cpp - ${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_index/helpers/Cloner.cpp - ${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_index/gpu/IndexGPUIDMAP.cpp - ${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_index/gpu/IndexGPUIVF.cpp - ${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_index/gpu/IndexGPUIVFSQ.cpp - ${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_index/gpu/IndexGPUIVFPQ.cpp - ${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_index/gpu/IndexIVFSQHybrid.cpp - ${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_offset_index/gpu/IndexGPUIVF_NM.cpp - ) + set(faiss_srcs ${faiss_srcs} + ${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_index/ConfAdapter.cpp + ${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_index/helpers/Cloner.cpp + ${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_index/gpu/IndexGPUIDMAP.cpp + ${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_index/gpu/IndexGPUIVF.cpp + ${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_index/gpu/IndexGPUIVFSQ.cpp + ${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_index/gpu/IndexGPUIVFPQ.cpp + ${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_index/gpu/IndexIVFSQHybrid.cpp + ${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_offset_index/gpu/IndexGPUIVF_NM.cpp + ) endif () -################################################################################ -# -if (NOT TARGET test_instructionset) - add_executable(test_instructionset test_instructionset.cpp) -endif () -target_link_libraries(test_instructionset ${depend_libs} ${unittest_libs}) -install(TARGETS test_instructionset DESTINATION unittest) - -################################################################################ -# -if (NOT TARGET test_knowhere_common) - add_executable(test_knowhere_common test_common.cpp ${util_srcs}) -endif () -target_link_libraries(test_knowhere_common ${depend_libs} ${unittest_libs} ${basic_libs}) -install(TARGETS test_knowhere_common DESTINATION unittest) - -if (MILVUS_GPU_VERSION) -################################################################################ -# -add_executable(test_gpuresource test_gpuresource.cpp ${util_srcs} ${faiss_srcs}) -target_link_libraries(test_gpuresource ${depend_libs} ${unittest_libs} ${basic_libs}) -install(TARGETS test_gpuresource DESTINATION unittest) - -################################################################################ -# -add_executable(test_customized_index test_customized_index.cpp ${util_srcs} ${faiss_srcs}) -target_link_libraries(test_customized_index ${depend_libs} ${unittest_libs} ${basic_libs}) -install(TARGETS test_customized_index DESTINATION unittest) -endif () - -################################################################################ -# -if (NOT TARGET test_idmap) - add_executable(test_idmap test_idmap.cpp ${faiss_srcs} ${util_srcs}) -endif () -target_link_libraries(test_idmap ${depend_libs} ${unittest_libs} ${basic_libs}) -install(TARGETS test_idmap DESTINATION unittest) - -################################################################################ -# -if (NOT TARGET test_ivf) - add_executable(test_ivf test_ivf.cpp ${faiss_srcs} ${util_srcs}) -endif () -target_link_libraries(test_ivf ${depend_libs} ${unittest_libs} ${basic_libs}) -install(TARGETS test_ivf DESTINATION unittest) - -################################################################################ -# -if (NOT TARGET test_ivf_hnsw) - add_executable(test_ivf_hnsw test_ivf_hnsw.cpp ${faiss_srcs} ${util_srcs}) -endif () -target_link_libraries(test_ivf_hnsw ${depend_libs} ${unittest_libs} ${basic_libs}) -install(TARGETS test_ivf_hnsw DESTINATION unittest) - -################################################################################ -# -if (NOT TARGET test_ivf_cpu_nm) - add_executable(test_ivf_cpu_nm test_ivf_cpu_nm.cpp ${faiss_srcs} ${util_srcs}) -endif () -target_link_libraries(test_ivf_cpu_nm ${depend_libs} ${unittest_libs} ${basic_libs}) -install(TARGETS test_ivf_cpu_nm DESTINATION unittest) - -################################################################################ -# -if (NOT TARGET test_ivf_gpu_nm) - add_executable(test_ivf_gpu_nm test_ivf_gpu_nm.cpp ${faiss_srcs} ${util_srcs}) -endif () -target_link_libraries(test_ivf_gpu_nm ${depend_libs} ${unittest_libs} ${basic_libs}) -install(TARGETS test_ivf_gpu_nm DESTINATION unittest) - -################################################################################ -# -if (NOT TARGET test_binaryidmap) - add_executable(test_binaryidmap test_binaryidmap.cpp ${faiss_srcs} ${util_srcs}) -endif () -target_link_libraries(test_binaryidmap ${depend_libs} ${unittest_libs} ${basic_libs}) -install(TARGETS test_binaryidmap DESTINATION unittest) - -################################################################################ -# -if (NOT TARGET test_binaryivf) - add_executable(test_binaryivf test_binaryivf.cpp ${faiss_srcs} ${util_srcs}) -endif () -target_link_libraries(test_binaryivf ${depend_libs} ${unittest_libs} ${basic_libs}) -install(TARGETS test_binaryivf DESTINATION unittest) - - ################################################################################ # add_definitions(-std=c++11 -O3 -march=native -Werror -DINFO) @@ -193,140 +106,42 @@ else () message(FATAL_ERROR "no OpenMP supprot") endif () -include_directories(${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_index/impl/nsg) -aux_source_directory(${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_index/impl/nsg nsg_src) -set(interface_src - ${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_offset_index/IndexNSG_NM.cpp +set(all_indexing_srcs + ${util_srcs} + test_instructionset.cpp + test_common.cpp + test_customized_index.cpp + test_idmap.cpp + test_ivf.cpp + test_ivf_hnsw.cpp + test_ivf_cpu_nm.cpp + test_binaryidmap.cpp + test_binaryivf.cpp + test_nsg.cpp + test_hnsw.cpp + test_rhnsw_flat.cpp + test_rhnsw_pq.cpp + test_rhnsw_sq8.cpp + test_structured_index_sort.cpp + test_structured_index_flat.cpp + test_ngtpanng.cpp + test_ngtonng.cpp ) -if (NOT TARGET test_nsg) - add_executable(test_nsg test_nsg.cpp ${interface_src} ${nsg_src} ${util_srcs} ${faiss_srcs}) -endif () -target_link_libraries(test_nsg ${depend_libs} ${unittest_libs} ${basic_libs}) -install(TARGETS test_nsg DESTINATION unittest) - -################################################################################ -# -set(hnsw_srcs - ${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_index/IndexHNSW.cpp - ) -if (NOT TARGET test_hnsw) - add_executable(test_hnsw test_hnsw.cpp ${hnsw_srcs} ${util_srcs}) -endif () -target_link_libraries(test_hnsw ${depend_libs} ${unittest_libs} ${basic_libs}) -install(TARGETS test_hnsw DESTINATION unittest) - -################################################################################ -# -set(rhnsw_flat_srcs - ${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_index/IndexRHNSW.cpp - ${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_index/IndexRHNSWFlat.cpp - ) -if (NOT TARGET test_rhnsw_flat) - add_executable(test_rhnsw_flat test_rhnsw_flat.cpp ${rhnsw_flat_srcs} ${util_srcs} ${faiss_srcs}) -endif () -target_link_libraries(test_rhnsw_flat ${depend_libs} ${unittest_libs} ${basic_libs}) -install(TARGETS test_rhnsw_flat DESTINATION unittest) - -################################################################################ -# -set(rhnsw_pq_srcs - ${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_index/IndexRHNSW.cpp - ${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_index/IndexRHNSWPQ.cpp - ) -if (NOT TARGET test_rhnsw_pq) - add_executable(test_rhnsw_pq test_rhnsw_pq.cpp ${rhnsw_pq_srcs} ${util_srcs} ${faiss_srcs}) -endif () -target_link_libraries(test_rhnsw_pq ${depend_libs} ${unittest_libs} ${basic_libs}) -install(TARGETS test_rhnsw_pq DESTINATION unittest) - -################################################################################ -# -set(rhnsw_sq8_srcs - ${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_index/IndexRHNSW.cpp - ${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_index/IndexRHNSWSQ.cpp - ) -if (NOT TARGET test_rhnsw_sq8) - add_executable(test_rhnsw_sq8 test_rhnsw_sq8.cpp ${rhnsw_sq8_srcs} ${util_srcs} ${faiss_srcs}) -endif () -target_link_libraries(test_rhnsw_sq8 ${depend_libs} ${unittest_libs} ${basic_libs}) -install(TARGETS test_rhnsw_sq8 DESTINATION unittest) - -################################################################################ -# -if (MILVUS_SUPPORT_SPTAG) - set(sptag_srcs - ${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_index/adapter/SptagAdapter.cpp - ${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_index/helpers/SPTAGParameterMgr.cpp - ${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_index/IndexSPTAG.cpp +if (MILVUS_GPU_VERSION) + set(all_indexing_srcs + ${all_indexing_srcs} + test_ivf_gpu_nm.cpp + test_gpuresource.cpp ) - if (NOT TARGET test_sptag) - add_executable(test_sptag test_sptag.cpp ${sptag_srcs} ${util_srcs}) - endif () - target_link_libraries(test_sptag - SPTAGLibStatic - ${depend_libs} ${unittest_libs} ${basic_libs}) - install(TARGETS test_sptag DESTINATION unittest) endif () -################################################################################ -# -set(annoy_srcs - ${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_index/IndexAnnoy.cpp - ) -if (NOT TARGET test_annoy) - add_executable(test_annoy test_annoy.cpp ${annoy_srcs} ${util_srcs}) -endif () -target_link_libraries(test_annoy ${depend_libs} ${unittest_libs} ${basic_libs}) -install(TARGETS test_annoy DESTINATION unittest) +if (MILVUS_SUPPORT_SPTAG) + set(all_indexing_srcs + test_sptag.cpp + ) +endif() -################################################################################ -# -set(structured_index_sort_srcs - ${INDEX_SOURCE_DIR}/knowhere/knowhere/index/structured_index/StructuredIndexSort-inl.h - ) -if (NOT TARGET test_structured_index_sort) - add_executable(test_structured_index_sort test_structured_index_sort.cpp ${structured_index_sort_srcs} ${util_srcs}) -endif () -target_link_libraries(test_structured_index_sort ${depend_libs} ${unittest_libs} ${basic_libs}) -install(TARGETS test_structured_index_sort DESTINATION unittest) +add_executable(test_knowhere ${all_indexing_srcs}) +target_link_libraries(test_knowhere knowhere ${depend_libs}) -#add_subdirectory(faiss_benchmark) -#add_subdirectory(metric_alg_benchmark) -################################################################################ - -# -set(structured_index_flat_srcs - ${INDEX_SOURCE_DIR}/knowhere/knowhere/index/structured_index/StructuredIndexFlat-inl.h - ) -if (NOT TARGET test_structured_index_flat) - add_executable(test_structured_index_flat test_structured_index_flat.cpp ${structured_index_flat_srcs} ${util_srcs}) -endif () -target_link_libraries(test_structured_index_flat ${depend_libs} ${unittest_libs} ${basic_libs}) -install(TARGETS test_structured_index_flat DESTINATION unittest) - -################################################################################ - -# -set(ngtpanng_srcs - ${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_index/IndexNGTPANNG.cpp - ${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_index/IndexNGT.cpp - ) -if (NOT TARGET test_ngtpanng) - add_executable(test_ngtpanng test_ngtpanng.cpp ${ngtpanng_srcs} ${util_srcs}) -endif () -target_link_libraries(test_ngtpanng ${depend_libs} ${unittest_libs} ${basic_libs}) -install(TARGETS test_ngtpanng DESTINATION unittest) - -################################################################################ -# -set(ngtonng_srcs - ${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_index/IndexNGTONNG.cpp - ${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_index/IndexNGT.cpp - ) -if (NOT TARGET test_ngtonng) - add_executable(test_ngtonng test_ngtonng.cpp ${ngtonng_srcs} ${util_srcs}) -endif () -target_link_libraries(test_ngtonng ${depend_libs} ${unittest_libs} ${basic_libs}) -install(TARGETS test_ngtonng DESTINATION unittest) - -################################################################################ +install(TARGETS test_knowhere DESTINATION unittest) diff --git a/internal/core/src/index/unittest/Helper.h b/internal/core/src/index/unittest/Helper.h index d2a8e6fc4a..88b9153a9b 100644 --- a/internal/core/src/index/unittest/Helper.h +++ b/internal/core/src/index/unittest/Helper.h @@ -8,6 +8,7 @@ // Unless required by applicable law or agreed to in writing, software distributed under the License // is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express // or implied. See the License for the specific language governing permissions and limitations under the License. +#pragma once #include #include @@ -28,7 +29,7 @@ #include "knowhere/index/vector_offset_index/gpu/IndexGPUIVF_NM.h" #endif -int DEVICEID = 0; +constexpr int DEVICEID = 0; constexpr int64_t DIM = 128; constexpr int64_t NB = 10000; constexpr int64_t NQ = 10; @@ -37,7 +38,7 @@ constexpr int64_t PINMEM = 1024 * 1024 * 200; constexpr int64_t TEMPMEM = 1024 * 1024 * 300; constexpr int64_t RESNUM = 2; -milvus::knowhere::IVFPtr +inline milvus::knowhere::IVFPtr IndexFactory(const milvus::knowhere::IndexType& type, const milvus::knowhere::IndexMode mode) { if (mode == milvus::knowhere::IndexMode::MODE_CPU) { if (type == milvus::knowhere::IndexEnum::INDEX_FAISS_IVFFLAT) { @@ -71,7 +72,7 @@ IndexFactory(const milvus::knowhere::IndexType& type, const milvus::knowhere::In return nullptr; } -milvus::knowhere::IVFNMPtr +inline milvus::knowhere::IVFNMPtr IndexFactoryNM(const milvus::knowhere::IndexType& type, const milvus::knowhere::IndexMode mode) { if (mode == milvus::knowhere::IndexMode::MODE_CPU) { if (type == milvus::knowhere::IndexEnum::INDEX_FAISS_IVFFLAT) { diff --git a/internal/core/src/index/unittest/test_structured_index_flat.cpp b/internal/core/src/index/unittest/test_structured_index_flat.cpp index e0a2a134f9..bcb7219987 100644 --- a/internal/core/src/index/unittest/test_structured_index_flat.cpp +++ b/internal/core/src/index/unittest/test_structured_index_flat.cpp @@ -20,7 +20,7 @@ #include "unittest/utils.h" -void +static void gen_rand_data(int range, int n, int*& p) { srand((unsigned int)time(nullptr)); p = (int*)malloc(n * sizeof(int)); @@ -30,7 +30,7 @@ gen_rand_data(int range, int n, int*& p) { } } -void +static void gen_rand_int64_data(int64_t range, int64_t n, int64_t*& p) { srand((int64_t)time(nullptr)); p = (int64_t*)malloc(n * sizeof(int64_t)); @@ -40,7 +40,7 @@ gen_rand_int64_data(int64_t range, int64_t n, int64_t*& p) { } } -void +static void gen_rand_double_data(double range, int64_t n, double*& p) { std::uniform_real_distribution unif(0, range); std::default_random_engine re;