diff --git a/internal/core/cmake/Utils.cmake b/internal/core/cmake/Utils.cmake index 8facf7cccd..1c6d208b47 100644 --- a/internal/core/cmake/Utils.cmake +++ b/internal/core/cmake/Utils.cmake @@ -103,10 +103,14 @@ endmacro() macro(add_source_at_current_directory_recursively) file(GLOB_RECURSE SOURCE_FILES RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} "*.cc" "*.cpp" "*.c" "*.cxx") + # Exclude test files from production code + list(FILTER SOURCE_FILES EXCLUDE REGEX ".*Test\\.cpp$|.*_test\\.cpp$|.*_test\\.cc$|.*Test\\.cc$") message(STATUS "${CMAKE_CURRENT_SOURCE_DIR} add new source files at current directory recursively: ${SOURCE_FILES}") endmacro() macro(add_source_at_current_directory) file(GLOB SOURCE_FILES RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} "*.cc" "*.cpp" "*.c" "*.cxx") + # Exclude test files from production code + list(FILTER SOURCE_FILES EXCLUDE REGEX ".*Test\\.cpp$|.*_test\\.cpp$|.*_test\\.cc$|.*Test\\.cc$") message(STATUS "${CMAKE_CURRENT_SOURCE_DIR} add new source files at current directory: ${SOURCE_FILES}") endmacro() diff --git a/internal/core/unittest/test_bitmap.cpp b/internal/core/src/common/BitmapTest.cpp similarity index 100% rename from internal/core/unittest/test_bitmap.cpp rename to internal/core/src/common/BitmapTest.cpp diff --git a/internal/core/src/common/ProtobufUtilsTest.cpp b/internal/core/src/common/ProtobufUtilsTest.cpp new file mode 100644 index 0000000000..4b9ae71659 --- /dev/null +++ b/internal/core/src/common/ProtobufUtilsTest.cpp @@ -0,0 +1,21 @@ +// Copyright (C) 2019-2020 Zilliz. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software distributed under the License +// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +// or implied. See the License for the specific language governing permissions and limitations under the License + +#include + +#include "common/protobuf_utils.h" +#include "pb/cgo_msg.pb.h" + +TEST(Util_Common, ProtoLayout) { + milvus::ProtoLayout layout; + milvus::proto::cgo::IndexStats result; + EXPECT_TRUE(layout.SerializeAndHoldProto(result)); +} diff --git a/internal/core/unittest/test_span.cpp b/internal/core/src/common/SpanTest.cpp similarity index 99% rename from internal/core/unittest/test_span.cpp rename to internal/core/src/common/SpanTest.cpp index 6450769bf7..5216118889 100644 --- a/internal/core/unittest/test_span.cpp +++ b/internal/core/src/common/SpanTest.cpp @@ -80,4 +80,4 @@ TEST(Span, Naive) { nullable_valid_data_ptr[i + begin]); } } -} \ No newline at end of file +} diff --git a/internal/core/src/common/UtilsTest.cpp b/internal/core/src/common/UtilsTest.cpp new file mode 100644 index 0000000000..cfc20ca21b --- /dev/null +++ b/internal/core/src/common/UtilsTest.cpp @@ -0,0 +1,33 @@ +// Copyright (C) 2019-2020 Zilliz. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software distributed under the License +// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +// or implied. See the License for the specific language governing permissions and limitations under the License + +#include + +#include + +#include "common/Utils.h" + +TEST(Util_Common, GetCommonPrefix) { + std::string str1 = ""; + std::string str2 = "milvus"; + auto common_prefix = milvus::GetCommonPrefix(str1, str2); + EXPECT_STREQ(common_prefix.c_str(), ""); + + str1 = "milvus"; + str2 = "milvus is great"; + common_prefix = milvus::GetCommonPrefix(str1, str2); + EXPECT_STREQ(common_prefix.c_str(), "milvus"); + + str1 = "milvus"; + str2 = ""; + common_prefix = milvus::GetCommonPrefix(str1, str2); + EXPECT_STREQ(common_prefix.c_str(), ""); +} diff --git a/internal/core/unittest/test_bitmap_index.cpp b/internal/core/src/index/BitmapIndexTest.cpp similarity index 100% rename from internal/core/unittest/test_bitmap_index.cpp rename to internal/core/src/index/BitmapIndexTest.cpp diff --git a/internal/core/src/index/UtilsTest.cpp b/internal/core/src/index/UtilsTest.cpp new file mode 100644 index 0000000000..5fa68c36ef --- /dev/null +++ b/internal/core/src/index/UtilsTest.cpp @@ -0,0 +1,80 @@ +// Copyright (C) 2019-2020 Zilliz. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software distributed under the License +// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +// or implied. See the License for the specific language governing permissions and limitations under the License + +#include + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "index/Utils.h" + +// A simple wrapper that removes a temporary file. +struct TmpFileWrapperIndexUtilsTest { + int fd = -1; + std::string filename; + + explicit TmpFileWrapperIndexUtilsTest(const std::string& _filename) + : filename{_filename} { + fd = open(filename.c_str(), + O_RDWR | O_CREAT | O_EXCL, + S_IRUSR | S_IWUSR | S_IXUSR); + } + TmpFileWrapperIndexUtilsTest(const TmpFileWrapperIndexUtilsTest&) = delete; + TmpFileWrapperIndexUtilsTest(TmpFileWrapperIndexUtilsTest&&) = delete; + TmpFileWrapperIndexUtilsTest& + operator=(const TmpFileWrapperIndexUtilsTest&) = delete; + TmpFileWrapperIndexUtilsTest& + operator=(TmpFileWrapperIndexUtilsTest&&) = delete; + ~TmpFileWrapperIndexUtilsTest() { + if (fd != -1) { + close(fd); + remove(filename.c_str()); + } + } +}; + +TEST(Util_Index, ReadFromFD) { + auto uuid = boost::uuids::random_generator()(); + auto uuid_string = boost::uuids::to_string(uuid); + auto file = std::string("/tmp/") + uuid_string; + + auto tmp_file = TmpFileWrapperIndexUtilsTest(file); + ASSERT_NE(tmp_file.fd, -1); + + size_t data_size = 100 * 1024 * 1024; // 100M + auto index_data = std::shared_ptr(new uint8_t[data_size]); + auto max_loop = size_t(INT_MAX) / data_size + 1; // insert data > 2G + for (int i = 0; i < static_cast(max_loop); ++i) { + auto size_write = write(tmp_file.fd, index_data.get(), data_size); + ASSERT_GE(size_write, 0); + } + + auto read_buf = + std::shared_ptr(new uint8_t[data_size * max_loop]); + EXPECT_NO_THROW(milvus::index::ReadDataFromFD( + tmp_file.fd, read_buf.get(), data_size * max_loop)); + + // On Linux, read() (and similar system calls) will transfer at most 0x7ffff000 (2,147,479,552) bytes once + EXPECT_THROW( + milvus::index::ReadDataFromFD( + tmp_file.fd, read_buf.get(), data_size * max_loop, INT_MAX), + milvus::SegcoreError); +} diff --git a/internal/core/unittest/test_reduce.cpp b/internal/core/src/query/SubSearchResultTest.cpp similarity index 100% rename from internal/core/unittest/test_reduce.cpp rename to internal/core/src/query/SubSearchResultTest.cpp diff --git a/internal/core/src/query/UtilsTest.cpp b/internal/core/src/query/UtilsTest.cpp new file mode 100644 index 0000000000..02ab7298bf --- /dev/null +++ b/internal/core/src/query/UtilsTest.cpp @@ -0,0 +1,86 @@ +// Copyright (C) 2019-2020 Zilliz. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software distributed under the License +// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +// or implied. See the License for the specific language governing permissions and limitations under the License + +#include + +#include +#include + +#include "common/Utils.h" +#include "query/Utils.h" + +TEST(Util_Query, StringMatch) { + using namespace milvus; + using namespace milvus::query; + + ASSERT_ANY_THROW(Match(1, 2, OpType::PrefixMatch)); + ASSERT_ANY_THROW(Match(std::string("not_match_operation"), + std::string("not_match"), + OpType::LessEqual)); + + ASSERT_TRUE(PrefixMatch("prefix1", "prefix")); + ASSERT_TRUE(PostfixMatch("1postfix", "postfix")); + ASSERT_TRUE(InnerMatch("xxinner1xx", "inner")); + ASSERT_TRUE(Match( + std::string("prefix1"), std::string("prefix"), OpType::PrefixMatch)); + ASSERT_TRUE(Match( + std::string("1postfix"), std::string("postfix"), OpType::PostfixMatch)); + ASSERT_TRUE(Match(std::string("xxpostfixxx"), + std::string("postfix"), + OpType::InnerMatch)); + + ASSERT_FALSE(PrefixMatch("", "longer")); + ASSERT_FALSE(PostfixMatch("", "longer")); + ASSERT_FALSE(InnerMatch("", "longer")); + + ASSERT_FALSE(PrefixMatch("dontmatch", "prefix")); + ASSERT_FALSE(InnerMatch("dontmatch", "postfix")); + + ASSERT_TRUE(Match(std::string_view("prefix1"), + std::string("prefix"), + OpType::PrefixMatch)); + ASSERT_TRUE(Match(std::string_view("1postfix"), + std::string("postfix"), + OpType::PostfixMatch)); + ASSERT_TRUE(Match(std::string_view("xxpostfixxx"), + std::string("postfix"), + OpType::InnerMatch)); + ASSERT_TRUE( + Match(std::string_view("x"), std::string("x"), OpType::PrefixMatch)); + ASSERT_FALSE( + Match(std::string_view(""), std::string("x"), OpType::InnerMatch)); + ASSERT_TRUE( + Match(std::string_view("x"), std::string(""), OpType::InnerMatch)); +} + +TEST(Util_Query, OutOfRange) { + using milvus::query::out_of_range; + + ASSERT_FALSE(out_of_range( + static_cast(std::numeric_limits::max()) - 1)); + ASSERT_FALSE(out_of_range( + static_cast(std::numeric_limits::min()) + 1)); + + ASSERT_TRUE(out_of_range( + static_cast(std::numeric_limits::max()) + 1)); + ASSERT_TRUE(out_of_range( + static_cast(std::numeric_limits::min()) - 1)); +} + +TEST(Util_Query, DisCloser) { + EXPECT_TRUE(milvus::query::dis_closer(0.1, 0.2, "L2")); + EXPECT_FALSE(milvus::query::dis_closer(0.2, 0.1, "L2")); + EXPECT_FALSE(milvus::query::dis_closer(0.1, 0.1, "L2")); + + EXPECT_TRUE(milvus::query::dis_closer(0.2, 0.1, "IP")); + EXPECT_FALSE(milvus::query::dis_closer(0.1, 0.2, "IP")); + EXPECT_FALSE(milvus::query::dis_closer(0.1, 0.1, "IP")); +} diff --git a/internal/core/unittest/test_reduce_c.cpp b/internal/core/src/segcore/ReduceStructureTest.cpp similarity index 100% rename from internal/core/unittest/test_reduce_c.cpp rename to internal/core/src/segcore/ReduceStructureTest.cpp diff --git a/internal/core/src/segcore/UtilsTest.cpp b/internal/core/src/segcore/UtilsTest.cpp new file mode 100644 index 0000000000..9d64c3b645 --- /dev/null +++ b/internal/core/src/segcore/UtilsTest.cpp @@ -0,0 +1,96 @@ +// Copyright (C) 2019-2020 Zilliz. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software distributed under the License +// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +// or implied. See the License for the specific language governing permissions and limitations under the License + +#include + +#include +#include +#include +#include +#include + +#include "common/Schema.h" +#include "common/Types.h" +#include "segcore/ConcurrentVector.h" +#include "segcore/DeletedRecord.h" +#include "segcore/InsertRecord.h" +#include "segcore/Record.h" +#include "segcore/Utils.h" + +TEST(Util_Segcore, UpperBound) { + using milvus::Timestamp; + using milvus::segcore::ConcurrentVector; + using milvus::segcore::upper_bound; + + std::vector data{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}; + ConcurrentVector timestamps(1); + timestamps.set_data_raw(0, data.data(), data.size()); + + ASSERT_EQ(1, upper_bound(timestamps, 0, data.size(), 0)); + ASSERT_EQ(5, upper_bound(timestamps, 0, data.size(), 4)); + ASSERT_EQ(10, upper_bound(timestamps, 0, data.size(), 10)); +} + +TEST(Util_Segcore, GetDeleteBitmap) { + using namespace milvus; + using namespace milvus::segcore; + + auto schema = std::make_shared(); + auto vec_fid = schema->AddDebugField( + "fakevec", DataType::VECTOR_FLOAT, 16, knowhere::metric::L2); + (void)vec_fid; + auto i64_fid = schema->AddDebugField("age", DataType::INT64); + schema->set_primary_field_id(i64_fid); + auto N = 10; + InsertRecord insert_record(*schema, N); + DeletedRecord delete_record( + &insert_record, + [&insert_record]( + const std::vector& pks, + const Timestamp* timestamps, + std::function + cb) { + for (size_t i = 0; i < pks.size(); ++i) { + auto timestamp = timestamps[i]; + auto offsets = insert_record.search_pk(pks[i], timestamp); + for (auto offset : offsets) { + cb(offset, timestamp); + } + } + }, + 0); + + // fill insert record, all insert records has same pk = 1, timestamps= {1 ... N} + std::vector age_data(N); + std::vector tss(N); + for (int i = 0; i < N; ++i) { + age_data[i] = 1; + tss[i] = i + 1; + insert_record.insert_pk(1, i); + } + auto insert_offset = insert_record.reserved.fetch_add(N); + insert_record.timestamps_.set_data_raw(insert_offset, tss.data(), N); + auto field_data = insert_record.get_data_base(i64_fid); + field_data->set_data_raw(insert_offset, age_data.data(), N); + insert_record.ack_responder_.AddSegment(insert_offset, insert_offset + N); + + // test case delete pk1(ts = 0) -> insert repeated pk1 (ts = {1 ... N}) -> query (ts = N) + std::vector delete_ts = {0}; + std::vector delete_pk = {1}; + delete_record.StreamPush(delete_pk, delete_ts.data()); + + auto query_timestamp = tss[N - 1]; + auto insert_barrier = get_barrier(insert_record, query_timestamp); + BitsetType res_bitmap(insert_barrier); + BitsetTypeView res_view(res_bitmap); + delete_record.Query(res_view, insert_barrier, query_timestamp); + ASSERT_EQ(res_view.count(), 0); +} diff --git a/internal/core/unittest/test_thread_pool.cpp b/internal/core/src/storage/ThreadPoolsTest.cpp similarity index 98% rename from internal/core/unittest/test_thread_pool.cpp rename to internal/core/src/storage/ThreadPoolsTest.cpp index c6ff28a941..05b002dde6 100644 --- a/internal/core/unittest/test_thread_pool.cpp +++ b/internal/core/src/storage/ThreadPoolsTest.cpp @@ -11,7 +11,6 @@ #include #include "storage/ThreadPools.h" -#include "common/Common.h" TEST(ThreadPool, ThreadNum) { auto& threadPool = diff --git a/internal/core/unittest/CMakeLists.txt b/internal/core/unittest/CMakeLists.txt index 2001a3f387..56a46ae350 100644 --- a/internal/core/unittest/CMakeLists.txt +++ b/internal/core/unittest/CMakeLists.txt @@ -11,6 +11,7 @@ include_directories(${CMAKE_HOME_DIRECTORY}/src) include_directories(${CMAKE_HOME_DIRECTORY}/src/thirdparty) +include_directories(${CMAKE_CURRENT_SOURCE_DIR}) include_directories( ${KNOWHERE_INCLUDE_DIR} ${SIMDJSON_INCLUDE_DIR} @@ -20,10 +21,17 @@ include_directories( ${MILVUS_STORAGE_INCLUDE_DIR} ) -add_definitions(-DMILVUS_TEST_SEGCORE_YAML_PATH="${CMAKE_SOURCE_DIR}/unittest/test_utils/test_segcore.yaml") +add_definitions(-DMILVUS_TEST_SEGCORE_YAML_PATH="${CMAKE_CURRENT_SOURCE_DIR}/test_utils/test_segcore.yaml") + +# Collect test files from source directories using glob pattern +file(GLOB_RECURSE SOURCE_TEST_FILES + "${CMAKE_HOME_DIRECTORY}/src/**/*Test.cpp" + "${CMAKE_HOME_DIRECTORY}/src/**/*_test.cpp" +) # TODO: better to use ls/find pattern set(MILVUS_TEST_FILES + ${SOURCE_TEST_FILES} init_gtest.cpp test_packed_c.cpp test_arrow_fs_c.cpp @@ -35,7 +43,6 @@ set(MILVUS_TEST_FILES test_bf_sparse.cpp test_binary.cpp test_binlog_index.cpp - test_bitmap_index.cpp test_bool_index.cpp test_c_api.cpp test_chunk.cpp @@ -74,8 +81,6 @@ set(MILVUS_TEST_FILES test_plan_proto.cpp test_query.cpp test_range_search_sort.cpp - test_reduce_c.cpp - test_reduce.cpp test_regex_query.cpp test_regex_query_util.cpp test_relational.cpp @@ -84,13 +89,11 @@ set(MILVUS_TEST_FILES test_sealed.cpp test_segcore.cpp test_similarity_corelation.cpp - test_span.cpp test_storage.cpp test_string_expr.cpp test_text_match.cpp test_timestamp_index.cpp test_tracer.cpp - test_utils.cpp test_chunked_segment.cpp test_chunked_column.cpp test_rust_result.cpp @@ -105,7 +108,6 @@ set(MILVUS_TEST_FILES test_group_chunk_translator.cpp test_chunked_segment_storage_v2.cpp test_expr_cache.cpp - test_thread_pool.cpp test_json_flat_index.cpp test_vector_array.cpp test_ngram_query.cpp diff --git a/internal/core/unittest/test_utils.cpp b/internal/core/unittest/test_utils.cpp deleted file mode 100644 index bed7e7291e..0000000000 --- a/internal/core/unittest/test_utils.cpp +++ /dev/null @@ -1,246 +0,0 @@ -// Copyright (C) 2019-2020 Zilliz. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software distributed under the License -// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -// or implied. See the License for the specific language governing permissions and limitations under the License - -#include -#include -#include - -#include -#include -#include -#include -#include - -#include "common/EasyAssert.h" -#include "common/Types.h" -#include "common/Utils.h" -#include "segcore/Record.h" -#include "common/Exception.h" -#include "knowhere/sparse_utils.h" -#include "pb/schema.pb.h" -#include "query/Utils.h" -#include "test_utils/DataGen.h" - -TEST(Util, StringMatch) { - using namespace milvus; - using namespace milvus::query; - - ASSERT_ANY_THROW(Match(1, 2, OpType::PrefixMatch)); - ASSERT_ANY_THROW(Match(std::string("not_match_operation"), - std::string("not_match"), - OpType::LessEqual)); - - ASSERT_TRUE(PrefixMatch("prefix1", "prefix")); - ASSERT_TRUE(PostfixMatch("1postfix", "postfix")); - ASSERT_TRUE(InnerMatch("xxinner1xx", "inner")); - ASSERT_TRUE(Match( - std::string("prefix1"), std::string("prefix"), OpType::PrefixMatch)); - ASSERT_TRUE(Match( - std::string("1postfix"), std::string("postfix"), OpType::PostfixMatch)); - ASSERT_TRUE(Match(std::string("xxpostfixxx"), - std::string("postfix"), - OpType::InnerMatch)); - - ASSERT_FALSE(PrefixMatch("", "longer")); - ASSERT_FALSE(PostfixMatch("", "longer")); - ASSERT_FALSE(InnerMatch("", "longer")); - - ASSERT_FALSE(PrefixMatch("dontmatch", "prefix")); - ASSERT_FALSE(InnerMatch("dontmatch", "postfix")); - - ASSERT_TRUE(Match(std::string_view("prefix1"), - std::string("prefix"), - OpType::PrefixMatch)); - - ASSERT_TRUE(Match(std::string_view("1postfix"), - std::string("postfix"), - OpType::PostfixMatch)); - - ASSERT_TRUE(Match(std::string_view("xxpostfixxx"), - std::string("postfix"), - OpType::InnerMatch)); - ASSERT_TRUE( - Match(std::string_view("x"), std::string("x"), OpType::PrefixMatch)); - ASSERT_FALSE( - Match(std::string_view(""), std::string("x"), OpType::InnerMatch)); - ASSERT_TRUE( - Match(std::string_view("x"), std::string(""), OpType::InnerMatch)); -} - -TEST(Util, GetDeleteBitmap) { - using namespace milvus; - using namespace milvus::query; - using namespace milvus::segcore; - - auto schema = std::make_shared(); - auto vec_fid = schema->AddDebugField( - "fakevec", DataType::VECTOR_FLOAT, 16, knowhere::metric::L2); - auto i64_fid = schema->AddDebugField("age", DataType::INT64); - schema->set_primary_field_id(i64_fid); - auto N = 10; - uint64_t seg_id = 101; - InsertRecord insert_record(*schema, N); - DeletedRecord delete_record( - &insert_record, - [&insert_record]( - const std::vector& pks, - const Timestamp* timestamps, - std::function - cb) { - for (size_t i = 0; i < pks.size(); ++i) { - auto timestamp = timestamps[i]; - auto offsets = insert_record.search_pk(pks[i], timestamp); - for (auto offset : offsets) { - cb(offset, timestamp); - } - } - }, - 0); - - // fill insert record, all insert records has same pk = 1, timestamps= {1 ... N} - std::vector age_data(N); - std::vector tss(N); - for (int i = 0; i < N; ++i) { - age_data[i] = 1; - tss[i] = i + 1; - insert_record.insert_pk(1, i); - } - auto insert_offset = insert_record.reserved.fetch_add(N); - insert_record.timestamps_.set_data_raw(insert_offset, tss.data(), N); - auto field_data = insert_record.get_data_base(i64_fid); - field_data->set_data_raw(insert_offset, age_data.data(), N); - insert_record.ack_responder_.AddSegment(insert_offset, insert_offset + N); - - // test case delete pk1(ts = 0) -> insert repeated pk1 (ts = {1 ... N}) -> query (ts = N) - std::vector delete_ts = {0}; - std::vector delete_pk = {1}; - delete_record.StreamPush(delete_pk, delete_ts.data()); - - auto query_timestamp = tss[N - 1]; - auto insert_barrier = get_barrier(insert_record, query_timestamp); - BitsetType res_bitmap(insert_barrier); - BitsetTypeView res_view(res_bitmap); - delete_record.Query(res_view, insert_barrier, query_timestamp); - ASSERT_EQ(res_view.count(), 0); -} - -TEST(Util, OutOfRange) { - using milvus::query::out_of_range; - - ASSERT_FALSE(out_of_range( - static_cast(std::numeric_limits::max()) - 1)); - ASSERT_FALSE(out_of_range( - static_cast(std::numeric_limits::min()) + 1)); - - ASSERT_TRUE(out_of_range( - static_cast(std::numeric_limits::max()) + 1)); - ASSERT_TRUE(out_of_range( - static_cast(std::numeric_limits::min()) - 1)); -} - -TEST(Util, upper_bound) { - using milvus::Timestamp; - using milvus::segcore::ConcurrentVector; - using milvus::segcore::upper_bound; - - std::vector data{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}; - ConcurrentVector timestamps(1); - timestamps.set_data_raw(0, data.data(), data.size()); - - ASSERT_EQ(1, upper_bound(timestamps, 0, data.size(), 0)); - ASSERT_EQ(5, upper_bound(timestamps, 0, data.size(), 4)); - ASSERT_EQ(10, upper_bound(timestamps, 0, data.size(), 10)); -} - -// A simple wrapper that removes a temporary file. -struct TmpFileWrapper { - int fd = -1; - std::string filename; - - TmpFileWrapper(const std::string& _filename) : filename{_filename} { - fd = open(filename.c_str(), - O_RDWR | O_CREAT | O_EXCL, - S_IRUSR | S_IWUSR | S_IXUSR); - } - TmpFileWrapper(const TmpFileWrapper&) = delete; - TmpFileWrapper(TmpFileWrapper&&) = delete; - TmpFileWrapper& - operator=(const TmpFileWrapper&) = delete; - TmpFileWrapper& - operator=(TmpFileWrapper&&) = delete; - ~TmpFileWrapper() { - if (fd != -1) { - close(fd); - remove(filename.c_str()); - } - } -}; - -TEST(Util, read_from_fd) { - auto uuid = boost::uuids::random_generator()(); - auto uuid_string = boost::uuids::to_string(uuid); - auto file = std::string("/tmp/") + uuid_string; - - auto tmp_file = TmpFileWrapper(file); - ASSERT_NE(tmp_file.fd, -1); - - size_t data_size = 100 * 1024 * 1024; // 100M - auto index_data = std::shared_ptr(new uint8_t[data_size]); - auto max_loop = size_t(INT_MAX) / data_size + 1; // insert data > 2G - for (int i = 0; i < max_loop; ++i) { - auto size_write = write(tmp_file.fd, index_data.get(), data_size); - ASSERT_GE(size_write, 0); - } - - auto read_buf = - std::shared_ptr(new uint8_t[data_size * max_loop]); - EXPECT_NO_THROW(milvus::index::ReadDataFromFD( - tmp_file.fd, read_buf.get(), data_size * max_loop)); - - // On Linux, read() (and similar system calls) will transfer at most 0x7ffff000 (2,147,479,552) bytes once - EXPECT_THROW( - milvus::index::ReadDataFromFD( - tmp_file.fd, read_buf.get(), data_size * max_loop, INT_MAX), - milvus::SegcoreError); -} - -TEST(Util, get_common_prefix) { - std::string str1 = ""; - std::string str2 = "milvus"; - auto common_prefix = milvus::GetCommonPrefix(str1, str2); - EXPECT_STREQ(common_prefix.c_str(), ""); - - str1 = "milvus"; - str2 = "milvus is great"; - common_prefix = milvus::GetCommonPrefix(str1, str2); - EXPECT_STREQ(common_prefix.c_str(), "milvus"); - - str1 = "milvus"; - str2 = ""; - common_prefix = milvus::GetCommonPrefix(str1, str2); - EXPECT_STREQ(common_prefix.c_str(), ""); -} - -TEST(Util, dis_closer) { - EXPECT_TRUE(milvus::query::dis_closer(0.1, 0.2, "L2")); - EXPECT_FALSE(milvus::query::dis_closer(0.2, 0.1, "L2")); - EXPECT_FALSE(milvus::query::dis_closer(0.1, 0.1, "L2")); - - EXPECT_TRUE(milvus::query::dis_closer(0.2, 0.1, "IP")); - EXPECT_FALSE(milvus::query::dis_closer(0.1, 0.2, "IP")); - EXPECT_FALSE(milvus::query::dis_closer(0.1, 0.1, "IP")); -} - -TEST(Util, ProtoLayout) { - milvus::ProtoLayout layout; - milvus::proto::cgo::IndexStats result; - EXPECT_TRUE(layout.SerializeAndHoldProto(result)); -} \ No newline at end of file