diff --git a/.github/workflows/code-checker.yaml b/.github/workflows/code-checker.yaml index 076c367e1d..c5a49e19f6 100644 --- a/.github/workflows/code-checker.yaml +++ b/.github/workflows/code-checker.yaml @@ -109,7 +109,7 @@ jobs: if [[ ! -d "/var/tmp/ccache" ]];then mkdir -p /var/tmp/ccache fi - brew install boost libomp ninja tbb ccache + brew install boost libomp ninja tbb openblas ccache make check-proto-product && make verifiers centos: name: Code Checker CentOS 7 diff --git a/internal/core/CMakeLists.txt b/internal/core/CMakeLists.txt index 225d1c7557..36c0bed2ee 100644 --- a/internal/core/CMakeLists.txt +++ b/internal/core/CMakeLists.txt @@ -31,26 +31,7 @@ project(core) include(CheckCXXCompilerFlag) if ( APPLE ) message(STATUS "==============Darwin Environment==============") - check_cxx_compiler_flag(-std=c++11 HAS_STD_CPP11_FLAG) - if(HAS_STD_CPP11_FLAG) - add_compile_options(-std=c++11) - endif() - if(CMAKE_C_COMPILER_ID MATCHES "Clang") - set(OpenMP_C "${CMAKE_C_COMPILER}" CACHE STRING "" FORCE) - set(OpenMP_C_FLAGS "-fopenmp=libomp -Wno-unused-command-line-argument" CACHE STRING "" FORCE) - set(OpenMP_C_LIB_NAMES "libomp" "libgomp" "libiomp5" CACHE STRING "" FORCE) - set(OpenMP_libomp_LIBRARY ${OpenMP_C_LIB_NAMES} CACHE STRING "" FORCE) - set(OpenMP_libgomp_LIBRARY ${OpenMP_C_LIB_NAMES} CACHE STRING "" FORCE) - set(OpenMP_libiomp5_LIBRARY ${OpenMP_C_LIB_NAMES} CACHE STRING "" FORCE) - endif() - if(CMAKE_CXX_COMPILER_ID MATCHES "Clang") - set(OpenMP_CXX "${CMAKE_CXX_COMPILER}" CACHE STRING "" FORCE) - set(OpenMP_CXX_FLAGS "-fopenmp=libomp -Wno-unused-command-line-argument" CACHE STRING "" FORCE) - set(OpenMP_CXX_LIB_NAMES "libomp" "libgomp" "libiomp5" CACHE STRING "" FORCE) - set(OpenMP_libomp_LIBRARY ${OpenMP_CXX_LIB_NAMES} CACHE STRING "" FORCE) - set(OpenMP_libgomp_LIBRARY ${OpenMP_CXX_LIB_NAMES} CACHE STRING "" FORCE) - set(OpenMP_libiomp5_LIBRARY ${OpenMP_CXX_LIB_NAMES} CACHE STRING "" FORCE) - endif() + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -I/usr/local/opt/llvm/include -I/usr/local/include -I/usr/local/opt/libomp/include -L/usr/local/opt/libomp/lib") elseif (${CMAKE_SYSTEM_NAME} MATCHES "Linux") message(STATUS "==============Linux Environment===============") set(LINUX TRUE) diff --git a/internal/core/src/index/IndexFactory-inl.h b/internal/core/src/index/IndexFactory-inl.h index 15d4dfc7b5..d4a9758d2a 100644 --- a/internal/core/src/index/IndexFactory-inl.h +++ b/internal/core/src/index/IndexFactory-inl.h @@ -34,8 +34,9 @@ inline ScalarIndexPtr IndexFactory::CreateIndex(const std::string& index_type) { #ifdef __linux__ return CreateStringIndexMarisa(); -#endif +#else throw std::runtime_error("unsupported platform"); +#endif } } // namespace milvus::scalar diff --git a/internal/core/src/index/StringIndexMarisa.cpp b/internal/core/src/index/StringIndexMarisa.cpp index aa84e657a8..6d3eedb758 100644 --- a/internal/core/src/index/StringIndexMarisa.cpp +++ b/internal/core/src/index/StringIndexMarisa.cpp @@ -80,7 +80,7 @@ StringIndexMarisa::Serialize(const Config& config) { res_set.Append(MARISA_TRIE_INDEX, index_data, size); res_set.Append(MARISA_STR_IDS, str_ids, str_ids_len); - knowhere::Disassemble(4 * 1024 * 1024, res_set); + knowhere::Disassemble(res_set, config); return res_set; } diff --git a/internal/core/src/indexbuilder/VecIndexCreator.cpp b/internal/core/src/indexbuilder/VecIndexCreator.cpp index e6d616ee66..708e086bf0 100644 --- a/internal/core/src/indexbuilder/VecIndexCreator.cpp +++ b/internal/core/src/indexbuilder/VecIndexCreator.cpp @@ -238,10 +238,9 @@ VecIndexCreator::Serialize() { std::shared_ptr raw_data(new uint8_t[raw_data_.size()], std::default_delete()); memcpy(raw_data.get(), raw_data_.data(), raw_data_.size()); ret.Append(RAW_DATA, raw_data, raw_data_.size()); - auto slice_size = get_index_file_slice_size(); // https://github.com/milvus-io/milvus/issues/6421 // Disassemble will only divide the raw vectors, other keys were already divided - knowhere::Disassemble(slice_size * 1024 * 1024, ret); + knowhere::Disassemble(ret, config_); } return ret; } @@ -300,7 +299,7 @@ VecIndexCreator::get_index_file_slice_size() { if (config_.contains(knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE)) { return config_[knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE].get(); } - return 4; // by default + return knowhere::index_file_slice_size; // by default } std::unique_ptr diff --git a/internal/core/src/indexbuilder/utils.cpp b/internal/core/src/indexbuilder/utils.cpp index 28648609d2..d77efdb71b 100644 --- a/internal/core/src/indexbuilder/utils.cpp +++ b/internal/core/src/indexbuilder/utils.cpp @@ -25,7 +25,9 @@ std::vector NM_List() { static std::vector ret{ knowhere::IndexEnum::INDEX_FAISS_IVFFLAT, +#ifdef MILVUS_SUPPORT_NSG knowhere::IndexEnum::INDEX_NSG, +#endif knowhere::IndexEnum::INDEX_RHNSWFlat, }; return ret; @@ -53,7 +55,9 @@ Need_ID_List() { std::vector Need_BuildAll_list() { static std::vector ret{ +#ifdef MILVUS_SUPPORT_NSG knowhere::IndexEnum::INDEX_NSG, +#endif }; return ret; } diff --git a/internal/core/src/query/SearchBruteForce.cpp b/internal/core/src/query/SearchBruteForce.cpp index 92b45113b6..050833fcfc 100644 --- a/internal/core/src/query/SearchBruteForce.cpp +++ b/internal/core/src/query/SearchBruteForce.cpp @@ -12,15 +12,8 @@ #include #include -#ifdef __APPLE__ -#include "knowhere/index/vector_index/impl/bruteforce/BruteForce.h" -#include "knowhere/common/Heap.h" -#elif defined(__linux__) || defined(__MINGW64__) #include #include -#else -#error "Unsupported OS environment."; -#endif #include "SearchBruteForce.h" #include "SubSearchResult.h" @@ -42,7 +35,6 @@ raw_search(MetricType metric_type, float* D, idx_t* labels, const BitsetView bitset) { -#if defined(__linux__) || defined(__MINGW64__) using namespace faiss; // NOLINT if (metric_type == METRIC_Jaccard || metric_type == METRIC_Tanimoto) { float_maxheap_array_t res = {size_t(n), size_t(k), labels, D}; @@ -70,9 +62,6 @@ raw_search(MetricType metric_type, std::string("binary search not support metric type: ") + segcore::MetricTypeToString(metric_type); PanicInfo(msg); } -#else - PanicInfo("Unsupported brute force for binary search on current OS environment!"); -#endif } SubSearchResult @@ -111,17 +100,6 @@ FloatSearchBruteForce(const dataset::SearchDataset& dataset, SubSearchResult sub_qr(num_queries, topk, metric_type, round_decimal); auto query_data = reinterpret_cast(dataset.query_data); auto chunk_data = reinterpret_cast(chunk_data_raw); -#ifdef __APPLE__ - if (metric_type == MetricType::METRIC_L2) { - knowhere::float_maxheap_array_t buf{(size_t)num_queries, (size_t)topk, sub_qr.get_ids(), - sub_qr.get_distances()}; - knowhere::knn_L2sqr_sse(query_data, chunk_data, dim, num_queries, size_per_chunk, &buf, bitset); - } else { - knowhere::float_minheap_array_t buf{(size_t)num_queries, (size_t)topk, sub_qr.get_ids(), - sub_qr.get_distances()}; - knowhere::knn_inner_product_sse(query_data, chunk_data, dim, num_queries, size_per_chunk, &buf, bitset); - } -#elif defined(__linux__) || defined(__MINGW64__) if (metric_type == MetricType::METRIC_L2) { faiss::float_maxheap_array_t buf{(size_t)num_queries, (size_t)topk, sub_qr.get_ids(), sub_qr.get_distances()}; faiss::knn_L2sqr(query_data, chunk_data, dim, num_queries, size_per_chunk, &buf, nullptr, bitset); @@ -129,9 +107,6 @@ FloatSearchBruteForce(const dataset::SearchDataset& dataset, faiss::float_minheap_array_t buf{(size_t)num_queries, (size_t)topk, sub_qr.get_ids(), sub_qr.get_distances()}; faiss::knn_inner_product(query_data, chunk_data, dim, num_queries, size_per_chunk, &buf, bitset); } -#else -#error "Unsupported OS environment!"; -#endif sub_qr.round_values(); return sub_qr; } diff --git a/internal/core/src/query/visitors/VerifyPlanNodeVisitor.cpp b/internal/core/src/query/visitors/VerifyPlanNodeVisitor.cpp index c0a47c07ef..d913ed2d53 100644 --- a/internal/core/src/query/visitors/VerifyPlanNodeVisitor.cpp +++ b/internal/core/src/query/visitors/VerifyPlanNodeVisitor.cpp @@ -41,11 +41,15 @@ InferIndexType(const Json& search_params) { namespace ip = knowhere::IndexParams; namespace ie = knowhere::IndexEnum; list.emplace(ip::nprobe, ie::INDEX_FAISS_IVFFLAT); +#ifdef MILVUS_SUPPORT_NSG list.emplace(ip::search_length, ie::INDEX_NSG); +#endif list.emplace(ip::ef, ie::INDEX_HNSW); list.emplace(ip::search_k, ie::INDEX_ANNOY); +#ifdef MILVUS_SUPPORT_NGT list.emplace(ip::max_search_edges, ie::INDEX_NGTONNG); list.emplace(ip::epsilon, ie::INDEX_NGTONNG); +#endif return list; }(); auto dbg_str = search_params.dump(); diff --git a/internal/core/src/segcore/FieldIndexing.cpp b/internal/core/src/segcore/FieldIndexing.cpp index d6cec8aa11..5a7159716e 100644 --- a/internal/core/src/segcore/FieldIndexing.cpp +++ b/internal/core/src/segcore/FieldIndexing.cpp @@ -14,9 +14,7 @@ #include #include "common/SystemProperty.h" -#if defined(__linux__) || defined(__MINGW64__) #include "knowhere/index/vector_index/IndexIVF.h" -#endif #include "knowhere/index/vector_index/adapter/VectorAdapter.h" #include "segcore/FieldIndexing.h" @@ -24,7 +22,6 @@ namespace milvus::segcore { void VectorFieldIndexing::BuildIndexRange(int64_t ack_beg, int64_t ack_end, const VectorBase* vec_base) { -#if defined(__linux__) || defined(__MINGW64__) AssertInfo(field_meta_.get_data_type() == DataType::VECTOR_FLOAT, "Data type of vector field is not VECTOR_FLOAT"); auto dim = field_meta_.get_dim(); @@ -43,9 +40,6 @@ VectorFieldIndexing::BuildIndexRange(int64_t ack_beg, int64_t ack_end, const Vec indexing->AddWithoutIds(dataset, conf); data_[chunk_id] = std::move(indexing); } -#else - throw std::runtime_error("Unsupported BuildIndexRange on current platform!"); -#endif } knowhere::Config diff --git a/internal/core/thirdparty/knowhere/CMakeLists.txt b/internal/core/thirdparty/knowhere/CMakeLists.txt index 2927c9e480..e039c0b955 100644 --- a/internal/core/thirdparty/knowhere/CMakeLists.txt +++ b/internal/core/thirdparty/knowhere/CMakeLists.txt @@ -11,8 +11,8 @@ # or implied. See the License for the specific language governing permissions and limitations under the License. #------------------------------------------------------------------------------- -set( KNOWHERE_VERSION v1.1.1 ) -set( KNOWHERE_SOURCE_MD5 "3f6dff91dc947a9b003434b8901e9130" ) +set( KNOWHERE_VERSION v1.1.2 ) +set( KNOWHERE_SOURCE_MD5 "7c6043bfaa073c6580a1c5b1e904c995" ) if ( DEFINED ENV{MILVUS_KNOWHERE_URL} ) set( KNOWHERE_SOURCE_URL "$ENV{MILVUS_KNOWHERE_URL}" ) diff --git a/internal/core/unittest/CMakeLists.txt b/internal/core/unittest/CMakeLists.txt index da6accce37..325f012d62 100644 --- a/internal/core/unittest/CMakeLists.txt +++ b/internal/core/unittest/CMakeLists.txt @@ -14,37 +14,45 @@ include_directories(${CMAKE_HOME_DIRECTORY}/src/thirdparty) add_definitions(-DMILVUS_TEST_SEGCORE_YAML_PATH="${CMAKE_SOURCE_DIR}/unittest/test_utils/test_segcore.yaml") +# TODO: better to use ls/find pattern +set(MILVUS_TEST_FILES + init_gtest.cpp + test_binary.cpp + test_bitmap.cpp + test_bool_index.cpp + test_common.cpp + test_concurrent_vector.cpp + test_conf_adapter_mgr.cpp + test_c_api.cpp + test_expr.cpp + test_indexing.cpp + test_index_c_api.cpp + test_index_wrapper.cpp + test_init.cpp + test_plan_proto.cpp + test_query.cpp + test_reduce.cpp + test_reduce_c.cpp + test_retrieve.cpp + test_scalar_index.cpp + test_sealed.cpp + test_segcore.cpp + test_similarity_corelation.cpp + test_span.cpp + test_timestamp_index.cpp + test_utils.cpp + ) + +# These 2 cases cannot run in mac because of tbb build error if (LINUX) - # TODO: better to use ls/find pattern set(MILVUS_TEST_FILES - init_gtest.cpp - test_binary.cpp - test_bitmap.cpp - test_common.cpp - test_concurrent_vector.cpp - test_c_api.cpp - test_expr.cpp - test_retrieve.cpp - test_indexing.cpp - test_index_wrapper.cpp - test_init.cpp - test_plan_proto.cpp - test_query.cpp - test_reduce.cpp - test_sealed.cpp - test_segcore.cpp - test_span.cpp - test_timestamp_index.cpp - test_reduce_c.cpp - test_conf_adapter_mgr.cpp - test_similarity_corelation.cpp - test_utils.cpp + ${MILVUS_TEST_FILES} test_scalar_index_creator.cpp - test_index_c_api.cpp - test_scalar_index.cpp test_string_index.cpp - test_bool_index.cpp ) +endif() + +if (LINUX) # check if memory leak exists in index builder set(INDEX_BUILDER_TEST_FILES test_index_wrapper.cpp @@ -68,22 +76,6 @@ if (LINUX) install(TARGETS index_builder_test DESTINATION unittest) endif() -if ( APPLE ) - set(MILVUS_TEST_FILES - init_gtest.cpp - test_common.cpp - test_concurrent_vector.cpp - test_conf_adapter_mgr.cpp - test_plan_proto.cpp - test_reduce.cpp - test_reduce_c.cpp - test_segcore.cpp - test_timestamp_index.cpp - test_similarity_corelation.cpp - test_utils.cpp - ) -endif () - add_executable(all_tests ${MILVUS_TEST_FILES} ) diff --git a/internal/core/unittest/test_index_c_api.cpp b/internal/core/unittest/test_index_c_api.cpp index 032965773a..342f975fc5 100644 --- a/internal/core/unittest/test_index_c_api.cpp +++ b/internal/core/unittest/test_index_c_api.cpp @@ -236,6 +236,8 @@ TEST(CInt64IndexTest, All) { } } +// disable this case since marisa not supported in mac +#ifdef __linux__ TEST(CStringIndexTest, All) { auto strs = GenStrArr(NB); schemapb::StringArray str_arr; @@ -288,3 +290,4 @@ TEST(CStringIndexTest, All) { delete[](char*) str_ds->Get(knowhere::meta::TENSOR); } +#endif diff --git a/internal/core/unittest/test_indexing.cpp b/internal/core/unittest/test_indexing.cpp index d950d02b59..375d11fbe8 100644 --- a/internal/core/unittest/test_indexing.cpp +++ b/internal/core/unittest/test_indexing.cpp @@ -298,82 +298,41 @@ TEST(Indexing, BinaryBruteForce) { auto json = SearchResultToJson(sr); std::cout << json.dump(2); +#ifdef __linux__ auto ref = json::parse(R"( [ [ - [ - "1024->0.000000", - "48942->0.642000", - "18494->0.644000", - "68225->0.644000", - "93557->0.644000" - ], - [ - "1025->0.000000", - "73557->0.641000", - "53086->0.643000", - "9737->0.643000", - "62855->0.644000" - ], - [ - "1026->0.000000", - "62904->0.644000", - "46758->0.644000", - "57969->0.645000", - "98113->0.646000" - ], - [ - "1027->0.000000", - "92446->0.638000", - "96034->0.640000", - "92129->0.644000", - "45887->0.644000" - ], - [ - "1028->0.000000", - "22992->0.643000", - "73903->0.644000", - "19969->0.645000", - "65178->0.645000" - ], - [ - "1029->0.000000", - "19776->0.641000", - "15166->0.642000", - "85470->0.642000", - "16730->0.643000" - ], - [ - "1030->0.000000", - "55939->0.640000", - "84253->0.643000", - "31958->0.644000", - "11667->0.646000" - ], - [ - "1031->0.000000", - "89536->0.637000", - "61622->0.638000", - "9275->0.639000", - "91403->0.640000" - ], - [ - "1032->0.000000", - "69504->0.642000", - "23414->0.644000", - "48770->0.645000", - "23231->0.645000" - ], - [ - "1033->0.000000", - "33540->0.636000", - "25310->0.640000", - "18576->0.640000", - "73729->0.642000" - ] + [ "1024->0.000000", "48942->0.642000", "18494->0.644000", "68225->0.644000", "93557->0.644000" ], + [ "1025->0.000000", "73557->0.641000", "53086->0.643000", "9737->0.643000", "62855->0.644000" ], + [ "1026->0.000000", "62904->0.644000", "46758->0.644000", "57969->0.645000", "98113->0.646000" ], + [ "1027->0.000000", "92446->0.638000", "96034->0.640000", "92129->0.644000", "45887->0.644000" ], + [ "1028->0.000000", "22992->0.643000", "73903->0.644000", "19969->0.645000", "65178->0.645000" ], + [ "1029->0.000000", "19776->0.641000", "15166->0.642000", "85470->0.642000", "16730->0.643000" ], + [ "1030->0.000000", "55939->0.640000", "84253->0.643000", "31958->0.644000", "11667->0.646000" ], + [ "1031->0.000000", "89536->0.637000", "61622->0.638000", "9275->0.639000", "91403->0.640000" ], + [ "1032->0.000000", "69504->0.642000", "23414->0.644000", "48770->0.645000", "23231->0.645000" ], + [ "1033->0.000000", "33540->0.636000", "25310->0.640000", "18576->0.640000", "73729->0.642000" ] ] ] )"); +#else // for mac +auto ref = json::parse(R"( +[ + [ + [ "1024->0.000000", "59169->0.645000", "98548->0.646000", "3356->0.646000", "90373->0.647000" ], + [ "1025->0.000000", "61245->0.638000", "95271->0.639000", "31087->0.639000", "31549->0.640000" ], + [ "1026->0.000000", "65225->0.648000", "35750->0.648000", "14971->0.649000", "75385->0.649000" ], + [ "1027->0.000000", "70158->0.640000", "27076->0.640000", "3407->0.641000", "59527->0.641000" ], + [ "1028->0.000000", "45757->0.645000", "3356->0.645000", "77230->0.646000", "28690->0.647000" ], + [ "1029->0.000000", "13291->0.642000", "24960->0.643000", "83770->0.643000", "88244->0.643000" ], + [ "1030->0.000000", "96807->0.641000", "39920->0.643000", "62943->0.644000", "12603->0.644000" ], + [ "1031->0.000000", "65769->0.648000", "60493->0.648000", "48738->0.648000", "4353->0.648000" ], + [ "1032->0.000000", "57827->0.637000", "8213->0.638000", "22221->0.639000", "23328->0.640000" ], + [ "1033->0.000000", "676->0.645000", "91430->0.646000", "85353->0.646000", "6014->0.646000" ] + ] +] +)"); +#endif auto json_str = json.dump(2); auto ref_str = ref.dump(2); ASSERT_EQ(json_str, ref_str); diff --git a/internal/core/unittest/test_query.cpp b/internal/core/unittest/test_query.cpp index e3206fe31f..d455992a07 100644 --- a/internal/core/unittest/test_query.cpp +++ b/internal/core/unittest/test_query.cpp @@ -186,6 +186,7 @@ TEST(Query, ExecWithPredicateLoader) { int topk = 5; Json json = SearchResultToJson(*sr); +#ifdef __linux__ auto ref = json::parse(R"( [ [ @@ -196,6 +197,18 @@ TEST(Query, ExecWithPredicateLoader) { ["66353->5.696000", "41087->5.917000", "97780->6.811000", "99239->7.562000", "86527->7.751000"] ] ])"); +#else // for mac + auto ref = json::parse(R"( +[ + [ + ["982->0.000000", "31864->4.270000", "18916->4.651000", "71547->5.125000", "13227->6.010000"], + ["96984->4.192000", "65514->6.011000", "89328->6.138000", "80284->6.526000", "68218->6.563000"], + ["30119->2.464000", "52595->4.323000", "32673->4.851000", "74834->5.009000", "50806->5.446000"], + ["99625->6.129000", "86582->6.900000", "10069->7.388000", "89982->7.672000", "85934->7.792000"], + ["37759->3.581000", "97019->5.557000", "53543->5.844000", "63535->6.439000", "94009->6.572000"] + ] +])"); +#endif std::cout << json.dump(2); ASSERT_EQ(json.dump(2), ref.dump(2)); } @@ -301,6 +314,7 @@ TEST(Query, ExecWithPredicate) { int topk = 5; Json json = SearchResultToJson(*sr); +#ifdef __linux__ auto ref = json::parse(R"( [ [ @@ -311,6 +325,18 @@ TEST(Query, ExecWithPredicate) { ["66353->5.696000", "41087->5.917000", "97780->6.811000", "99239->7.562000", "86527->7.751000"] ] ])"); +#else // for mac +auto ref = json::parse(R"( +[ + [ + ["982->0.000000", "31864->4.270000", "18916->4.651000", "71547->5.125000", "13227->6.010000"], + ["96984->4.192000", "65514->6.011000", "89328->6.138000", "80284->6.526000", "68218->6.563000"], + ["30119->2.464000", "52595->4.323000", "32673->4.851000", "74834->5.009000", "50806->5.446000"], + ["99625->6.129000", "86582->6.900000", "10069->7.388000", "89982->7.672000", "85934->7.792000"], + ["37759->3.581000", "97019->5.557000", "53543->5.844000", "63535->6.439000", "94009->6.572000"] + ] +])"); +#endif std::cout << json.dump(2); ASSERT_EQ(json.dump(2), ref.dump(2)); } @@ -498,6 +524,7 @@ TEST(Query, ExecWithoutPredicate) { std::vector> results; int topk = 5; auto json = SearchResultToJson(*sr); +#ifdef __linux__ auto ref = json::parse(R"( [ [ @@ -508,6 +535,18 @@ TEST(Query, ExecWithoutPredicate) { ["66353->5.696000", "41087->5.917000", "24554->6.195000", "68019->6.654000", "97780->6.811000"] ] ])"); +#else // for mac +auto ref = json::parse(R"( +[ + [ + ["982->0.000000", "31864->4.270000", "18916->4.651000", "78227->4.808000", "71547->5.125000"], + ["96984->4.192000", "45733->4.912000", "32891->5.016000", "65514->6.011000", "89328->6.138000"], + ["30119->2.464000", "52595->4.323000", "32673->4.851000", "74834->5.009000", "76784->5.195000"], + ["99625->6.129000", "86582->6.900000", "60608->7.285000", "10069->7.388000", "89982->7.672000"], + ["37759->3.581000", "45814->4.872000", "97019->5.557000", "23626->5.839000", "53543->5.844000"] + ] +])"); +#endif std::cout << json.dump(2); ASSERT_EQ(json.dump(2), ref.dump(2)); } diff --git a/internal/core/unittest/test_sealed.cpp b/internal/core/unittest/test_sealed.cpp index dbda40b16e..f1e57f438c 100644 --- a/internal/core/unittest/test_sealed.cpp +++ b/internal/core/unittest/test_sealed.cpp @@ -298,6 +298,7 @@ TEST(Sealed, LoadFieldData) { ASSERT_EQ(json.dump(-2), json2.dump(-2)); segment->DropFieldData(double_id); ASSERT_ANY_THROW(segment->Search(plan.get(), *ph_group, time)); +#ifdef __linux__ auto std_json = Json::parse(R"( [ [ @@ -308,6 +309,18 @@ TEST(Sealed, LoadFieldData) { ["66353->5.696000", "30664->5.881000", "41087->5.917000", "10393->6.633000", "90215->7.202000"] ] ])"); +#else // for mac +auto std_json = Json::parse(R"( +[ + [ + ["982->0.000000", "31864->4.270000", "18916->4.651000", "71547->5.125000", "86706->5.991000"], + ["96984->4.192000", "65514->6.011000", "89328->6.138000", "80284->6.526000", "68218->6.563000"], + ["30119->2.464000", "82365->4.725000", "74834->5.009000", "79995->5.725000", "33359->5.816000"], + ["99625->6.129000", "86582->6.900000", "85934->7.792000", "60450->8.087000", "19257->8.530000"], + ["37759->3.581000", "31292->5.780000", "98124->6.216000", "63535->6.439000", "11707->6.553000"] + ] +])"); +#endif ASSERT_EQ(std_json.dump(-2), json.dump(-2)); } diff --git a/internal/core/unittest/test_utils/indexbuilder_test_utils.h b/internal/core/unittest/test_utils/indexbuilder_test_utils.h index 28e031bd68..8b66e4f45f 100644 --- a/internal/core/unittest/test_utils/indexbuilder_test_utils.h +++ b/internal/core/unittest/test_utils/indexbuilder_test_utils.h @@ -56,7 +56,7 @@ generate_conf(const knowhere::IndexType& index_type, const knowhere::MetricType& {knowhere::meta::DIM, DIM}, {knowhere::meta::TOPK, K}, {knowhere::Metric::TYPE, metric_type}, - {knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE, 4}, + {knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE, knowhere::index_file_slice_size}, }; } else if (index_type == knowhere::IndexEnum::INDEX_FAISS_IVFPQ) { return knowhere::Config{ @@ -67,7 +67,7 @@ generate_conf(const knowhere::IndexType& index_type, const knowhere::MetricType& {knowhere::IndexParams::m, 4}, {knowhere::IndexParams::nbits, 8}, {knowhere::Metric::TYPE, metric_type}, - {knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE, 4}, + {knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE, knowhere::index_file_slice_size}, }; } else if (index_type == knowhere::IndexEnum::INDEX_FAISS_IVFFLAT) { return knowhere::Config{ @@ -76,7 +76,7 @@ generate_conf(const knowhere::IndexType& index_type, const knowhere::MetricType& {knowhere::IndexParams::nlist, 16}, {knowhere::IndexParams::nprobe, 4}, {knowhere::Metric::TYPE, metric_type}, - {knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE, 4}, + {knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE, knowhere::index_file_slice_size}, #ifdef MILVUS_GPU_VERSION {knowhere::meta::DEVICEID, DEVICEID}, #endif @@ -89,7 +89,7 @@ generate_conf(const knowhere::IndexType& index_type, const knowhere::MetricType& {knowhere::IndexParams::nprobe, 4}, {knowhere::IndexParams::nbits, 8}, {knowhere::Metric::TYPE, metric_type}, - {knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE, 4}, + {knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE, knowhere::index_file_slice_size}, #ifdef MILVUS_GPU_VERSION {knowhere::meta::DEVICEID, DEVICEID}, #endif @@ -103,7 +103,7 @@ generate_conf(const knowhere::IndexType& index_type, const knowhere::MetricType& {knowhere::IndexParams::m, 4}, {knowhere::IndexParams::nbits, 8}, {knowhere::Metric::TYPE, metric_type}, - {knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE, 4}, + {knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE, knowhere::index_file_slice_size}, }; } else if (index_type == knowhere::IndexEnum::INDEX_FAISS_BIN_IDMAP) { return knowhere::Config{ @@ -111,6 +111,7 @@ generate_conf(const knowhere::IndexType& index_type, const knowhere::MetricType& {knowhere::meta::TOPK, K}, {knowhere::Metric::TYPE, metric_type}, }; +#ifdef MILVUS_SUPPORT_NSG } else if (index_type == knowhere::IndexEnum::INDEX_NSG) { return knowhere::Config{ {knowhere::meta::DIM, DIM}, @@ -123,20 +124,21 @@ generate_conf(const knowhere::IndexType& index_type, const knowhere::MetricType& {knowhere::IndexParams::candidate, 100}, {knowhere::Metric::TYPE, metric_type}, }; +#endif #ifdef MILVUS_SUPPORT_SPTAG } else if (index_type == knowhere::IndexEnum::INDEX_SPTAG_KDT_RNT) { return knowhere::Config{ {knowhere::meta::DIM, DIM}, // {knowhere::meta::TOPK, 10}, {knowhere::Metric::TYPE, metric_type}, - {knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE, 4}, + {knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE, knowhere::index_file_slice_size}, }; } else if (index_type == knowhere::IndexEnum::INDEX_SPTAG_BKT_RNT) { return knowhere::Config{ {knowhere::meta::DIM, DIM}, // {knowhere::meta::TOPK, 10}, {knowhere::Metric::TYPE, metric_type}, - {knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE, 4}, + {knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE, knowhere::index_file_slice_size}, }; #endif } else if (index_type == knowhere::IndexEnum::INDEX_HNSW) { @@ -152,7 +154,7 @@ generate_conf(const knowhere::IndexType& index_type, const knowhere::MetricType& {knowhere::IndexParams::n_trees, 4}, {knowhere::IndexParams::search_k, 100}, {knowhere::Metric::TYPE, metric_type}, - {knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE, 4}, + {knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE, knowhere::index_file_slice_size}, }; } else if (index_type == knowhere::IndexEnum::INDEX_RHNSWFlat) { return knowhere::Config{ @@ -162,7 +164,7 @@ generate_conf(const knowhere::IndexType& index_type, const knowhere::MetricType& {knowhere::IndexParams::efConstruction, 200}, {knowhere::IndexParams::ef, 200}, {knowhere::Metric::TYPE, metric_type}, - {knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE, 4}, + {knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE, knowhere::index_file_slice_size}, }; } else if (index_type == knowhere::IndexEnum::INDEX_RHNSWPQ) { return knowhere::Config{ @@ -172,7 +174,7 @@ generate_conf(const knowhere::IndexType& index_type, const knowhere::MetricType& {knowhere::IndexParams::efConstruction, 200}, {knowhere::IndexParams::ef, 200}, {knowhere::Metric::TYPE, metric_type}, - {knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE, 4}, + {knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE, knowhere::index_file_slice_size}, {knowhere::IndexParams::PQM, 8}, }; } else if (index_type == knowhere::IndexEnum::INDEX_RHNSWSQ) { @@ -183,8 +185,9 @@ generate_conf(const knowhere::IndexType& index_type, const knowhere::MetricType& {knowhere::IndexParams::efConstruction, 200}, {knowhere::IndexParams::ef, 200}, {knowhere::Metric::TYPE, metric_type}, - {knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE, 4}, + {knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE, knowhere::index_file_slice_size}, }; +#ifdef MILVUS_SUPPORT_NGT } else if (index_type == knowhere::IndexEnum::INDEX_NGTPANNG) { return knowhere::Config{ {knowhere::meta::DIM, DIM}, @@ -195,7 +198,7 @@ generate_conf(const knowhere::IndexType& index_type, const knowhere::MetricType& {knowhere::IndexParams::max_search_edges, 50}, {knowhere::IndexParams::forcedly_pruned_edge_size, 60}, {knowhere::IndexParams::selectively_pruned_edge_size, 30}, - {knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE, 4}, + {knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE, knowhere::index_file_slice_size}, }; } else if (index_type == knowhere::IndexEnum::INDEX_NGTONNG) { return knowhere::Config{ @@ -207,8 +210,9 @@ generate_conf(const knowhere::IndexType& index_type, const knowhere::MetricType& {knowhere::IndexParams::max_search_edges, 50}, {knowhere::IndexParams::outgoing_edge_size, 5}, {knowhere::IndexParams::incoming_edge_size, 40}, - {knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE, 4}, + {knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE, knowhere::index_file_slice_size}, }; +#endif } return knowhere::Config(); }