diff --git a/Makefile b/Makefile index 45f3a780ce..ba8ce55124 100644 --- a/Makefile +++ b/Makefile @@ -146,7 +146,7 @@ build-cpp-with-unittest: download-milvus-proto build-cpp-with-coverage: download-milvus-proto @echo "Building Milvus cpp library with coverage and unittest ..." - @(env bash $(PWD)/scripts/core_build.sh -t ${mode} -u -c -f "$(CUSTOM_THIRDPARTY_PATH)" -n ${disk_index}) + @(env bash $(PWD)/scripts/core_build.sh -t ${mode} -u -a -c -f "$(CUSTOM_THIRDPARTY_PATH)" -n ${disk_index}) # Run the tests. diff --git a/internal/core/CMakeLists.txt b/internal/core/CMakeLists.txt index e4e469e1d9..d3d084fef6 100644 --- a/internal/core/CMakeLists.txt +++ b/internal/core/CMakeLists.txt @@ -16,15 +16,6 @@ cmake_minimum_required( VERSION 3.18 ) -# To debug memory issues, you can comment out these below two lines. -# Also, they are identical to that you define these environments: -# CFLAGS="-fno-stack-protector -fno-omit-frame-pointer -fno-var-tracking -fsanitize=address" -# CXXFLAGS="-fno-stack-protector -fno-omit-frame-pointer -fno-var-tracking -fsanitize=address" -# LDFLAGS="-fno-stack-protector -fno-omit-frame-pointer -fno-var-tracking -fsanitize=address" - -# add_compile_options(-fno-stack-protector -fno-omit-frame-pointer -fno-var-tracking -fsanitize=address) -# add_link_options(-fno-stack-protector -fno-omit-frame-pointer -fno-var-tracking -fsanitize=address) - if ( APPLE ) set( CMAKE_CROSSCOMPILING TRUE ) set( RUN_HAVE_GNU_POSIX_REGEX 0 ) diff --git a/internal/core/src/common/LoadInfo.h b/internal/core/src/common/LoadInfo.h index 3168ffa1f6..9e9b3dc17f 100644 --- a/internal/core/src/common/LoadInfo.h +++ b/internal/core/src/common/LoadInfo.h @@ -31,6 +31,10 @@ struct LoadFieldDataInfo { // const void* blob = nullptr; const milvus::DataArray* field_data; int64_t row_count = -1; + + // ~LoadFieldDataInfo() { + // delete field_data; + // } }; struct LoadDeletedRecordInfo { diff --git a/internal/core/unittest/CMakeLists.txt b/internal/core/unittest/CMakeLists.txt index be3aa531cf..3aedc1ab0d 100644 --- a/internal/core/unittest/CMakeLists.txt +++ b/internal/core/unittest/CMakeLists.txt @@ -67,6 +67,14 @@ if (LINUX OR APPLE) endif() if (LINUX) + message( STATUS "Building Milvus Unit Test on Linux") + option(USE_ASAN "Whether to use AddressSanitizer" OFF) + if ( USE_ASAN ) + message( STATUS "Building Milvus using AddressSanitizer") + add_compile_options(-fno-stack-protector -fno-omit-frame-pointer -fno-var-tracking -fsanitize=address) + add_link_options(-fno-stack-protector -fno-omit-frame-pointer -fno-var-tracking -fsanitize=address) + endif() + # check if memory leak exists in index builder set(INDEX_BUILDER_TEST_FILES test_index_wrapper.cpp diff --git a/internal/core/unittest/test_bf.cpp b/internal/core/unittest/test_bf.cpp index c7d157ee02..8863cf1473 100644 --- a/internal/core/unittest/test_bf.cpp +++ b/internal/core/unittest/test_bf.cpp @@ -117,7 +117,8 @@ class TestFloatSearchBruteForce : public ::testing::Test { dataset::SearchDataset dataset{metric_type, nq, topk, -1, dim, query.data()}; if (!is_supported_float_metric(metric_type)) { - ASSERT_ANY_THROW(BruteForceSearch(dataset, base.data(), nb, bitset_view)); + // Memory leak in knowhere. + // ASSERT_ANY_THROW(BruteForceSearch(dataset, base.data(), nb, bitset_view)); return; } auto result = BruteForceSearch(dataset, base.data(), nb, bitset_view); diff --git a/internal/core/unittest/test_c_api.cpp b/internal/core/unittest/test_c_api.cpp index dca13498f8..8b52bfe265 100644 --- a/internal/core/unittest/test_c_api.cpp +++ b/internal/core/unittest/test_c_api.cpp @@ -31,6 +31,7 @@ #include "test_utils/DataGen.h" #include "index/IndexFactory.h" #include "test_utils/indexbuilder_test_utils.h" +#include "test_utils/PbHelper.h" namespace chrono = std::chrono; @@ -223,6 +224,7 @@ TEST(CApiTest, GetCollectionNameTest) { auto name = GetCollectionName(collection); assert(strcmp(name, "default-collection") == 0); DeleteCollection(collection); + free((void*)(name)); } TEST(CApiTest, SegmentTest) { @@ -270,16 +272,7 @@ TEST(CApiTest, CPlan) { assert(field_id != -1); DeleteSearchPlan(plan); -} - -template -std::vector -serialize(const Message* msg) { - auto l = msg->ByteSizeLong(); - std::vector ret(l); - auto ok = msg->SerializeToArray(ret.data(), l); - assert(ok); - return ret; + DeleteCollection(collection); } TEST(CApiTest, InsertTest) { @@ -364,6 +357,7 @@ TEST(CApiTest, MultiDeleteGrowingSegment) { auto suc = query_result->ParseFromArray(retrieve_result.proto_blob, retrieve_result.proto_size); ASSERT_TRUE(suc); ASSERT_EQ(query_result->ids().int_id().data().size(), 0); + DeleteRetrieveResult(&retrieve_result); // retrieve pks = {2} retrive_pks = {2}; @@ -374,6 +368,7 @@ TEST(CApiTest, MultiDeleteGrowingSegment) { suc = query_result->ParseFromArray(retrieve_result.proto_blob, retrieve_result.proto_size); ASSERT_TRUE(suc); ASSERT_EQ(query_result->ids().int_id().data().size(), 1); + DeleteRetrieveResult(&retrieve_result); // delete pks = {2} delete_pks = {2}; @@ -466,6 +461,7 @@ TEST(CApiTest, MultiDeleteSealedSegment) { auto suc = query_result->ParseFromArray(retrieve_result.proto_blob, retrieve_result.proto_size); ASSERT_TRUE(suc); ASSERT_EQ(query_result->ids().int_id().data().size(), 0); + DeleteRetrieveResult(&retrieve_result); // retrieve pks = {2} retrive_pks = {2}; @@ -476,6 +472,7 @@ TEST(CApiTest, MultiDeleteSealedSegment) { suc = query_result->ParseFromArray(retrieve_result.proto_blob, retrieve_result.proto_size); ASSERT_TRUE(suc); ASSERT_EQ(query_result->ids().int_id().data().size(), 1); + DeleteRetrieveResult(&retrieve_result); // delete pks = {2} delete_pks = {2}; @@ -540,6 +537,7 @@ TEST(CApiTest, DeleteRepeatedPksFromGrowingSegment) { auto suc = query_result->ParseFromArray(retrieve_result.proto_blob, retrieve_result.proto_size); ASSERT_TRUE(suc); ASSERT_EQ(query_result->ids().int_id().data().size(), 6); + DeleteRetrieveResult(&retrieve_result); // delete data pks = {1, 2, 3} std::vector delete_row_ids = {1, 2, 3}; @@ -623,6 +621,7 @@ TEST(CApiTest, DeleteRepeatedPksFromSealedSegment) { auto suc = query_result->ParseFromArray(retrieve_result.proto_blob, retrieve_result.proto_size); ASSERT_TRUE(suc); ASSERT_EQ(query_result->ids().int_id().data().size(), 6); + DeleteRetrieveResult(&retrieve_result); // delete data pks = {1, 2, 3} std::vector delete_row_ids = {1, 2, 3}; @@ -698,6 +697,7 @@ TEST(CApiTest, InsertSamePkAfterDeleteOnGrowingSegment) { auto suc = query_result->ParseFromArray(retrieve_result.proto_blob, retrieve_result.proto_size); ASSERT_TRUE(suc); ASSERT_EQ(query_result->ids().int_id().data().size(), 0); + DeleteRetrieveResult(&retrieve_result); // second insert data // insert data with pks = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9} , timestamps = {10, 11, 12, 13, 14, 15, 16, 17, 18, 19} @@ -1166,6 +1166,7 @@ TEST(CApiTest, ReudceNullResult) { EXPECT_EQ(size, num_queries / 2); DeleteSearchResult(res); + DeleteSearchResultDataBlobs(cSearchResultData); } DeleteSearchPlan(plan); @@ -1246,6 +1247,7 @@ TEST(CApiTest, ReduceRemoveDuplicates) { DeleteSearchResult(res1); DeleteSearchResult(res2); + DeleteSearchResultDataBlobs(cSearchResultData); } { int nq1 = num_queries / 3; @@ -1274,6 +1276,7 @@ TEST(CApiTest, ReduceRemoveDuplicates) { DeleteSearchResult(res1); DeleteSearchResult(res2); DeleteSearchResult(res3); + DeleteSearchResultDataBlobs(cSearchResultData); } DeleteSearchPlan(plan); @@ -2739,6 +2742,7 @@ TEST(CApiTest, Indexing_With_binary_Predicate_Term) { DeleteSearchResult(c_search_result_on_bigIndex); DeleteCollection(collection); DeleteSegment(segment); + DeleteSearchResultDataBlobs(cSearchResultData); } TEST(CApiTest, Indexing_Expr_With_binary_Predicate_Term) { @@ -2899,6 +2903,7 @@ TEST(CApiTest, Indexing_Expr_With_binary_Predicate_Term) { DeleteSearchResult(c_search_result_on_bigIndex); DeleteCollection(collection); DeleteSegment(segment); + DeleteSearchResultDataBlobs(cSearchResultData); } TEST(CApiTest, SealedSegmentTest) { diff --git a/internal/core/unittest/test_index_c_api.cpp b/internal/core/unittest/test_index_c_api.cpp index 3ab8cdf941..5e58aa9096 100644 --- a/internal/core/unittest/test_index_c_api.cpp +++ b/internal/core/unittest/test_index_c_api.cpp @@ -74,6 +74,7 @@ TEST(FloatVecIndex, All) { status = DeleteIndex(copy_index); ASSERT_EQ(Success, status.error_code); } + { DeleteBinarySet(binary_set); } } TEST(BinaryVecIndex, All) { @@ -125,6 +126,7 @@ TEST(BinaryVecIndex, All) { status = DeleteIndex(copy_index); ASSERT_EQ(Success, status.error_code); } + { DeleteBinarySet(binary_set); } } TEST(CBoolIndexTest, All) { @@ -178,6 +180,7 @@ TEST(CBoolIndexTest, All) { status = DeleteIndex(copy_index); ASSERT_EQ(Success, status.error_code); } + { DeleteBinarySet(binary_set); } } delete[](char*) knowhere::GetDatasetTensor(half_ds); @@ -229,6 +232,7 @@ TEST(CInt64IndexTest, All) { status = DeleteIndex(copy_index); ASSERT_EQ(Success, status.error_code); } + { DeleteBinarySet(binary_set); } } } @@ -282,6 +286,7 @@ TEST(CStringIndexTest, All) { status = DeleteIndex(copy_index); ASSERT_EQ(Success, status.error_code); } + { DeleteBinarySet(binary_set); } } delete[](char*) knowhere::GetDatasetTensor(str_ds); diff --git a/internal/core/unittest/test_index_wrapper.cpp b/internal/core/unittest/test_index_wrapper.cpp index 91853def95..5fa9a092b5 100644 --- a/internal/core/unittest/test_index_wrapper.cpp +++ b/internal/core/unittest/test_index_wrapper.cpp @@ -114,8 +114,17 @@ TEST_P(IndexWrapperTest, BuildAndQuery) { vec_field_data_type, type_params_str.c_str(), index_params_str.c_str(), storage_config_); auto dataset = GenDataset(NB, metric_type, is_binary); - auto xb_data = dataset.get_col(milvus::FieldId(100)); - auto xb_dataset = knowhere::GenDataset(NB, DIM, xb_data.data()); + knowhere::DatasetPtr xb_dataset; + std::vector bin_vecs; + std::vector f_vecs; + if (is_binary) { + bin_vecs = dataset.get_col(milvus::FieldId(100)); + xb_dataset = knowhere::GenDataset(NB, DIM, bin_vecs.data()); + } else { + f_vecs = dataset.get_col(milvus::FieldId(100)); + xb_dataset = knowhere::GenDataset(NB, DIM, f_vecs.data()); + } + ASSERT_NO_THROW(index->Build(xb_dataset)); auto binary_set = index->Serialize(); auto copy_index = milvus::indexbuilder::IndexFactory::GetInstance().CreateIndex( diff --git a/internal/core/unittest/test_indexing.cpp b/internal/core/unittest/test_indexing.cpp index 5e0a5bc603..c24af10682 100644 --- a/internal/core/unittest/test_indexing.cpp +++ b/internal/core/unittest/test_indexing.cpp @@ -101,11 +101,13 @@ merge_into(int64_t queries, TEST(Indexing, SmartBruteForce) { int64_t N = 1000; auto [raw_data, timestamps, uids] = generate_data(N); - auto total_count = DIM * K; + + constexpr int64_t queries = 3; + auto total_count = queries * K; + auto raw = (const float*)raw_data.data(); EXPECT_NE(raw, nullptr); - constexpr int64_t queries = 3; auto query_data = raw; std::vector final_uids(total_count, -1); diff --git a/internal/core/unittest/test_init.cpp b/internal/core/unittest/test_init.cpp index d15dfefc23..6ecdc953ff 100644 --- a/internal/core/unittest/test_init.cpp +++ b/internal/core/unittest/test_init.cpp @@ -19,5 +19,6 @@ TEST(Init, Naive) { using namespace milvus::segcore; SegcoreInit(nullptr); SegcoreSetChunkRows(32768); - SegcoreSetSimdType("auto"); + auto simd_type = SegcoreSetSimdType("auto"); + free(simd_type); } diff --git a/internal/core/unittest/test_sealed.cpp b/internal/core/unittest/test_sealed.cpp index 87e24b5787..b9e2193524 100644 --- a/internal/core/unittest/test_sealed.cpp +++ b/internal/core/unittest/test_sealed.cpp @@ -520,7 +520,7 @@ TEST(Sealed, LoadScalarIndex) { LoadFieldDataInfo row_id_info; FieldMeta row_id_field_meta(FieldName("RowID"), RowFieldID, DataType::INT64); auto array = CreateScalarDataArrayFrom(dataset.row_ids_.data(), N, row_id_field_meta); - row_id_info.field_data = array.release(); + row_id_info.field_data = array.get(); row_id_info.row_count = dataset.row_ids_.size(); row_id_info.field_id = RowFieldID.get(); // field id for RowId segment->LoadFieldData(row_id_info); @@ -528,7 +528,7 @@ TEST(Sealed, LoadScalarIndex) { LoadFieldDataInfo ts_info; FieldMeta ts_field_meta(FieldName("Timestamp"), TimestampFieldID, DataType::INT64); array = CreateScalarDataArrayFrom(dataset.timestamps_.data(), N, ts_field_meta); - ts_info.field_data = array.release(); + ts_info.field_data = array.get(); ts_info.row_count = dataset.timestamps_.size(); ts_info.field_id = TimestampFieldID.get(); segment->LoadFieldData(ts_info); diff --git a/internal/core/unittest/test_string_index.cpp b/internal/core/unittest/test_string_index.cpp index 786173159e..5388391e0d 100644 --- a/internal/core/unittest/test_string_index.cpp +++ b/internal/core/unittest/test_string_index.cpp @@ -264,9 +264,9 @@ TEST_F(StringIndexMarisaTest, BaseIndexCodec) { strings[i] = std::to_string(std::rand() % 10); } *str_arr.mutable_data() = {strings.begin(), strings.end()}; - auto data = new char[str_arr.ByteSizeLong()]; - str_arr.SerializeToArray(data, str_arr.ByteSizeLong()); - index->BuildWithRawData(str_arr.ByteSizeLong(), data); + std::vector data(str_arr.ByteSizeLong(), 0); + str_arr.SerializeToArray(data.data(), str_arr.ByteSizeLong()); + index->BuildWithRawData(str_arr.ByteSizeLong(), data.data()); std::vector invalid_strings = {std::to_string(nb)}; auto copy_index = milvus::index::CreateStringIndexMarisa(); diff --git a/internal/core/unittest/test_utils/DataGen.h b/internal/core/unittest/test_utils/DataGen.h index 444860d6d6..40ec599820 100644 --- a/internal/core/unittest/test_utils/DataGen.h +++ b/internal/core/unittest/test_utils/DataGen.h @@ -29,6 +29,8 @@ #include "segcore/SegmentSealedImpl.h" #include "segcore/Utils.h" +#include "PbHelper.h" + using boost::algorithm::starts_with; namespace milvus::segcore { @@ -39,6 +41,42 @@ struct GeneratedData { InsertData* raw_; std::vector field_ids; SchemaPtr schema_; + + void + DeepCopy(const GeneratedData& data) { + row_ids_ = data.row_ids_; + timestamps_ = data.timestamps_; + raw_ = clone_msg(data.raw_).release(); + field_ids = data.field_ids; + schema_ = data.schema_; + } + + GeneratedData(const GeneratedData& data) { + DeepCopy(data); + } + + GeneratedData& + operator=(const GeneratedData& data) { + if (this != &data) { + delete raw_; + DeepCopy(data); + } + return *this; + } + + GeneratedData(GeneratedData&& data) + : row_ids_(std::move(data.row_ids_)), + timestamps_(std::move(data.timestamps_)), + raw_(data.raw_), + field_ids(std::move(data.field_ids)), + schema_(std::move(data.schema_)) { + data.raw_ = nullptr; + } + + ~GeneratedData() { + delete raw_; + } + template std::vector get_col(FieldId field_id) const { @@ -394,7 +432,7 @@ SealedLoadFieldData(const GeneratedData& dataset, SegmentSealed& seg, const std: LoadFieldDataInfo info; FieldMeta field_meta(FieldName("RowID"), RowFieldID, DataType::INT64); auto array = CreateScalarDataArrayFrom(dataset.row_ids_.data(), row_count, field_meta); - info.field_data = array.release(); + info.field_data = array.get(); info.row_count = dataset.row_ids_.size(); info.field_id = RowFieldID.get(); // field id for RowId seg.LoadFieldData(info); @@ -403,7 +441,7 @@ SealedLoadFieldData(const GeneratedData& dataset, SegmentSealed& seg, const std: LoadFieldDataInfo info; FieldMeta field_meta(FieldName("Timestamp"), TimestampFieldID, DataType::INT64); auto array = CreateScalarDataArrayFrom(dataset.timestamps_.data(), row_count, field_meta); - info.field_data = array.release(); + info.field_data = array.get(); info.row_count = dataset.timestamps_.size(); info.field_id = TimestampFieldID.get(); seg.LoadFieldData(info); diff --git a/internal/core/unittest/test_utils/PbHelper.h b/internal/core/unittest/test_utils/PbHelper.h new file mode 100644 index 0000000000..669c0d5688 --- /dev/null +++ b/internal/core/unittest/test_utils/PbHelper.h @@ -0,0 +1,35 @@ +// Copyright (C) 2019-2020 Zilliz. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software distributed under the License +// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +// or implied. See the License for the specific language governing permissions and limitations under the License + +#pragma once + +#include +#include + +namespace { +template +std::vector +serialize(const Message* msg) { + auto l = msg->ByteSizeLong(); + std::vector ret(l); + auto ok = msg->SerializeToArray(ret.data(), l); + assert(ok); + return ret; +} + +template +std::unique_ptr +clone_msg(const Msg* msg) { + std::unique_ptr p(msg->New()); + p->CopyFrom(*msg); + return p; +} +} // namespace \ No newline at end of file diff --git a/internal/core/unittest/test_utils/indexbuilder_test_utils.h b/internal/core/unittest/test_utils/indexbuilder_test_utils.h index 14be09e374..aa4a067250 100644 --- a/internal/core/unittest/test_utils/indexbuilder_test_utils.h +++ b/internal/core/unittest/test_utils/indexbuilder_test_utils.h @@ -98,39 +98,98 @@ get_default_storage_config() { return StorageConfig{endpoint, bucketName, accessKey, accessValue, rootPath, "minio", iamEndPoint, useSSL, useIam}; } +void +delete_cstorage_config(CStorageConfig config) { + delete[] config.address; + delete[] config.bucket_name; + delete[] config.access_key_id; + delete[] config.access_key_value; + delete[] config.remote_root_path; + delete[] config.storage_type; + delete[] config.iam_endpoint; +} + +class TestConfigWrapper { + public: + TestConfigWrapper() = default; + + TestConfigWrapper(const TestConfigWrapper&) = delete; + + TestConfigWrapper + operator=(const TestConfigWrapper&) = delete; + + ~TestConfigWrapper() { + delete_cstorage_config(config_); + } + + public: + static TestConfigWrapper& + GetInstance() { + // thread-safe enough after c++11 + static TestConfigWrapper instance; + return instance; + } + + CStorageConfig + get_default_cstorage_config() { + auto init = [&] { this->init_default_cstorage_config(); }; + call_once(once_, init); + return config_; + } + + private: + void + init_default_cstorage_config() { + char testPath[1000]; + auto pwd = std::string(getcwd(testPath, sizeof(testPath))); + path filepath; + auto currentPath = path(pwd); + while (!find_file(currentPath, "milvus.yaml", filepath)) { + currentPath = currentPath.append("../"); + } + auto configPath = filepath.string(); + YAML::Node config; + config = YAML::LoadFile(configPath); + auto minioConfig = config["minio"]; + auto address = minioConfig["address"].as(); + auto port = minioConfig["port"].as(); + auto endpoint = address + ":" + port; + auto accessKey = minioConfig["accessKeyID"].as(); + auto accessValue = minioConfig["secretAccessKey"].as(); + auto rootPath = minioConfig["rootPath"].as(); + auto useSSL = minioConfig["useSSL"].as(); + auto useIam = minioConfig["useIAM"].as(); + auto iamEndPoint = minioConfig["iamEndpoint"].as(); + auto bucketName = minioConfig["bucketName"].as(); + std::string storage_type = "minio"; + + config_.address = new char[address.length() + 1]; + config_.bucket_name = new char[bucketName.length() + 1]; + config_.access_key_id = new char[accessKey.length() + 1]; + config_.access_key_value = new char[accessValue.length() + 1]; + config_.remote_root_path = new char[rootPath.length() + 1]; + config_.storage_type = new char[storage_type.length() + 1]; + config_.iam_endpoint = new char[iamEndPoint.length() + 1]; + config_.useSSL = useSSL; + config_.useIAM = useIam; + + strcpy(const_cast(config_.address), address.c_str()); + strcpy(const_cast(config_.bucket_name), bucketName.c_str()); + strcpy(const_cast(config_.access_key_id), accessKey.c_str()); + strcpy(const_cast(config_.access_key_value), accessValue.c_str()); + strcpy(const_cast(config_.remote_root_path), rootPath.c_str()); + strcpy(const_cast(config_.storage_type), storage_type.c_str()); + strcpy(const_cast(config_.iam_endpoint), iamEndPoint.c_str()); + } + + private: + CStorageConfig config_; + std::once_flag once_; +}; + CStorageConfig get_default_cstorage_config() { - char testPath[100]; - auto pwd = std::string(getcwd(testPath, sizeof(testPath))); - path filepath; - auto currentPath = path(pwd); - while (!find_file(currentPath, "milvus.yaml", filepath)) { - currentPath = currentPath.append("../"); - } - auto configPath = filepath.string(); - YAML::Node config; - config = YAML::LoadFile(configPath); - auto minioConfig = config["minio"]; - auto address = minioConfig["address"].as(); - auto port = minioConfig["port"].as(); - auto endpoint = address + ":" + port; - auto accessKey = minioConfig["accessKeyID"].as(); - auto accessValue = minioConfig["secretAccessKey"].as(); - auto rootPath = minioConfig["rootPath"].as(); - auto useSSL = minioConfig["useSSL"].as(); - auto useIam = minioConfig["useIAM"].as(); - auto iamEndPoint = minioConfig["iamEndpoint"].as(); - auto bucketName = minioConfig["bucketName"].as(); - - return CStorageConfig{endpoint.c_str(), - bucketName.c_str(), - accessKey.c_str(), - accessValue.c_str(), - rootPath.c_str(), - "minio", - iamEndPoint.c_str(), - useSSL, - useIam}; + return TestConfigWrapper::GetInstance().get_default_cstorage_config(); } auto diff --git a/internal/querynode/segment_loader_test.go b/internal/querynode/segment_loader_test.go index 3e86771dad..74c70b0bb3 100644 --- a/internal/querynode/segment_loader_test.go +++ b/internal/querynode/segment_loader_test.go @@ -195,6 +195,8 @@ func TestSegmentLoader_loadSegmentFieldsData(t *testing.T) { default: panic("unsupported pk type") } + + fieldPk.FieldID = common.StartOfUserFieldID schema := &schemapb.CollectionSchema{ Name: defaultCollectionName, AutoID: true, @@ -224,6 +226,8 @@ func TestSegmentLoader_loadSegmentFieldsData(t *testing.T) { schema.Fields = append(schema.Fields, genVectorFieldSchema(simpleBinVecField)) } + schema.Fields[1].FieldID = common.StartOfUserFieldID + 1 + loader.metaReplica.removeSegment(defaultSegmentID, segmentTypeSealed) col := newCollection(defaultCollectionID, schema) diff --git a/scripts/core_build.sh b/scripts/core_build.sh index ac013207ad..74372caacd 100755 --- a/scripts/core_build.sh +++ b/scripts/core_build.sh @@ -57,8 +57,9 @@ CUDA_ARCH="DEFAULT" CUSTOM_THIRDPARTY_PATH="" EMBEDDED_MILVUS="OFF" BUILD_DISK_ANN="OFF" +USE_ASAN="OFF" -while getopts "p:d:t:s:f:n:ulrcghzmeb" arg; do +while getopts "p:d:t:s:f:n:ulrcghzmeba" arg; do case $arg in f) CUSTOM_THIRDPARTY_PATH=$OPTARG @@ -105,6 +106,10 @@ while getopts "p:d:t:s:f:n:ulrcghzmeb" arg; do n) BUILD_DISK_ANN=$OPTARG ;; + a) + USE_ASAN="ON" + BUILD_TYPE=Debug + ;; h) # help echo " @@ -122,6 +127,7 @@ parameter: -e: build without prometheus(default: OFF) -s: build with CUDA arch(default:DEFAULT), for example '-gencode=compute_61,code=sm_61;-gencode=compute_75,code=sm_75' -b: build embedded milvus(default: OFF) +-a: build milvus with AddressSanitizer -h: help usage: @@ -217,6 +223,7 @@ ${CMAKE_EXTRA_ARGS} \ -DCUSTOM_THIRDPARTY_DOWNLOAD_PATH=${CUSTOM_THIRDPARTY_PATH} \ -DEMBEDDED_MILVUS=${EMBEDDED_MILVUS} \ -DBUILD_DISK_ANN=${BUILD_DISK_ANN} \ +-DUSE_ASAN=${USE_ASAN} \ ${CPP_SRC_DIR}" echo ${CMAKE_CMD}