mirror of
https://gitee.com/milvus-io/milvus.git
synced 2025-12-06 17:18:35 +08:00
enhance: Move c API unittest aside to src files (#44458)
Related to #43931 Signed-off-by: Congqi Xia <congqi.xia@zilliz.com>
This commit is contained in:
congqixia
GitHub
parent
7b83314bf3
commit
b532a3e026
@ -24,8 +24,12 @@
|
||||
#include <sys/types.h>
|
||||
#include <unistd.h>
|
||||
|
||||
#include "common/Common.h"
|
||||
#include "index/Utils.h"
|
||||
|
||||
using namespace milvus;
|
||||
using namespace milvus::index;
|
||||
|
||||
// A simple wrapper that removes a temporary file.
|
||||
struct TmpFileWrapperIndexUtilsTest {
|
||||
int fd = -1;
|
||||
@ -51,7 +55,7 @@ struct TmpFileWrapperIndexUtilsTest {
|
||||
}
|
||||
};
|
||||
|
||||
TEST(Util_Index, ReadFromFD) {
|
||||
TEST(UtilIndex, ReadFromFD) {
|
||||
auto uuid = boost::uuids::random_generator()();
|
||||
auto uuid_string = boost::uuids::to_string(uuid);
|
||||
auto file = std::string("/tmp/") + uuid_string;
|
||||
@ -78,3 +82,54 @@ TEST(Util_Index, ReadFromFD) {
|
||||
tmp_file.fd, read_buf.get(), data_size * max_loop, INT_MAX),
|
||||
milvus::SegcoreError);
|
||||
}
|
||||
|
||||
TEST(UtilIndex, TestGetValueFromConfig) {
|
||||
nlohmann::json cfg = nlohmann::json::parse(
|
||||
R"({"a" : 100, "b" : true, "c" : "true", "d" : 1.234, "e" : null})");
|
||||
auto a_value = GetValueFromConfig<int64_t>(cfg, "a");
|
||||
ASSERT_EQ(a_value.value(), 100);
|
||||
|
||||
auto b_value = GetValueFromConfig<bool>(cfg, "b");
|
||||
ASSERT_TRUE(b_value.value());
|
||||
|
||||
auto c_value = GetValueFromConfig<bool>(cfg, "c");
|
||||
ASSERT_TRUE(c_value.value());
|
||||
|
||||
auto d_value = GetValueFromConfig<double>(cfg, "d");
|
||||
ASSERT_NEAR(d_value.value(), 1.234, 0.001);
|
||||
|
||||
try {
|
||||
GetValueFromConfig<std::string>(cfg, "d");
|
||||
} catch (const std::exception& e) {
|
||||
std::cout << e.what() << std::endl;
|
||||
ASSERT_EQ(std::string(e.what()).find("config type error for key") !=
|
||||
std::string::npos,
|
||||
true);
|
||||
}
|
||||
|
||||
auto e_value = GetValueFromConfig<std::string>(cfg, "e");
|
||||
ASSERT_FALSE(e_value.has_value());
|
||||
}
|
||||
|
||||
TEST(UtilIndex, TestGetValueFromConfigWithoutTypeCheck) {
|
||||
nlohmann::json cfg = nlohmann::json::parse(
|
||||
R"({"a" : 100, "b" : true, "c" : "true", "d" : 1.234, "e" : "1.234", "f" : null})");
|
||||
SetDefaultConfigParamTypeCheck(false);
|
||||
auto a_value = GetValueFromConfig<int64_t>(cfg, "a");
|
||||
ASSERT_EQ(a_value.value(), 100);
|
||||
std::cout << "a_value: " << a_value.value() << std::endl;
|
||||
|
||||
auto b_value = GetValueFromConfig<bool>(cfg, "b");
|
||||
ASSERT_TRUE(b_value.value());
|
||||
std::cout << "b_value: " << b_value.value() << std::endl;
|
||||
auto c_value = GetValueFromConfig<bool>(cfg, "c");
|
||||
ASSERT_TRUE(c_value.value());
|
||||
std::cout << "c_value: " << c_value.value() << std::endl;
|
||||
auto d_value = GetValueFromConfig<double>(cfg, "d");
|
||||
ASSERT_NEAR(d_value.value(), 1.234, 0.001);
|
||||
std::cout << "d_value: " << d_value.value() << std::endl;
|
||||
auto e_value = GetValueFromConfig<double>(cfg, "e");
|
||||
ASSERT_FALSE(e_value.has_value());
|
||||
auto f_value = GetValueFromConfig<bool>(cfg, "f");
|
||||
ASSERT_FALSE(f_value.has_value());
|
||||
}
|
||||
109
internal/core/src/segcore/collection_c_test.cpp
Normal file
109
internal/core/src/segcore/collection_c_test.cpp
Normal file
@ -0,0 +1,109 @@
|
||||
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software distributed under the License
|
||||
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
|
||||
// or implied. See the License for the specific language governing permissions and limitations under the License
|
||||
|
||||
#include <gtest/gtest.h>
|
||||
|
||||
#include "knowhere/comp/index_param.h"
|
||||
#include "segcore/collection_c.h"
|
||||
#include "test_utils/c_api_test_utils.h"
|
||||
|
||||
using namespace milvus;
|
||||
using namespace milvus::segcore;
|
||||
|
||||
TEST(CApiTest, CollectionTest) {
|
||||
auto collection = NewCollection(get_default_schema_config().c_str());
|
||||
DeleteCollection(collection);
|
||||
}
|
||||
|
||||
TEST(CApiTest, UpdateSchemaTest) {
|
||||
std::string schema_string = generate_collection_schema<milvus::FloatVector>(
|
||||
knowhere::metric::L2, DIM);
|
||||
// CCollection
|
||||
auto collection = NewCollection(schema_string.c_str());
|
||||
|
||||
// create updated schema with extra field
|
||||
namespace schema = milvus::proto::schema;
|
||||
schema::CollectionSchema collection_schema;
|
||||
collection_schema.set_name("collection_test");
|
||||
|
||||
auto vec_field_schema = collection_schema.add_fields();
|
||||
vec_field_schema->set_name("fakevec");
|
||||
vec_field_schema->set_fieldid(100);
|
||||
vec_field_schema->set_data_type(schema::DataType::FloatVector);
|
||||
auto metric_type_param = vec_field_schema->add_index_params();
|
||||
metric_type_param->set_key("metric_type");
|
||||
metric_type_param->set_value(knowhere::metric::L2);
|
||||
auto dim_param = vec_field_schema->add_type_params();
|
||||
dim_param->set_key("dim");
|
||||
dim_param->set_value(std::to_string(DIM));
|
||||
|
||||
auto other_field_schema = collection_schema.add_fields();
|
||||
other_field_schema->set_name("counter");
|
||||
other_field_schema->set_fieldid(101);
|
||||
other_field_schema->set_data_type(schema::DataType::Int64);
|
||||
other_field_schema->set_is_primary_key(true);
|
||||
|
||||
auto other_field_schema2 = collection_schema.add_fields();
|
||||
other_field_schema2->set_name("doubleField");
|
||||
other_field_schema2->set_fieldid(102);
|
||||
other_field_schema2->set_data_type(schema::DataType::Double);
|
||||
|
||||
auto added_field = collection_schema.add_fields();
|
||||
added_field->set_name("added_field");
|
||||
added_field->set_fieldid(103);
|
||||
added_field->set_data_type(schema::DataType::Int64);
|
||||
added_field->set_nullable(true);
|
||||
|
||||
std::string updated_schema_string;
|
||||
auto marshal = collection_schema.SerializeToString(&updated_schema_string);
|
||||
ASSERT_TRUE(marshal);
|
||||
|
||||
// Call UpdateSchema CApi here
|
||||
// UpdateSchema(CCollection, const void*, const int64_t)
|
||||
auto status = UpdateSchema(collection,
|
||||
updated_schema_string.c_str(),
|
||||
updated_schema_string.length(),
|
||||
100);
|
||||
ASSERT_EQ(status.error_code, Success);
|
||||
|
||||
auto col = static_cast<milvus::segcore::Collection*>(collection);
|
||||
auto updated_schema = col->get_schema();
|
||||
auto add_field = updated_schema->operator[](FieldId(103));
|
||||
|
||||
ASSERT_EQ(add_field.get_name().get(), "added_field");
|
||||
|
||||
// Test failure case, no panicking with failure code
|
||||
status = UpdateSchema(collection, nullptr, 0, 200);
|
||||
ASSERT_NE(status.error_code, Success);
|
||||
|
||||
DeleteCollection(collection);
|
||||
// free error msg, which shall be responsible for go side to call C.free()
|
||||
free(const_cast<char*>(status.error_msg));
|
||||
}
|
||||
|
||||
TEST(CApiTest, SetIndexMetaTest) {
|
||||
auto collection = NewCollection(get_default_schema_config().c_str());
|
||||
|
||||
milvus::proto::segcore::CollectionIndexMeta indexMeta;
|
||||
indexMeta.ParseFromString(get_default_index_meta());
|
||||
char buffer[indexMeta.ByteSizeLong()];
|
||||
indexMeta.SerializeToArray(buffer, indexMeta.ByteSizeLong());
|
||||
SetIndexMeta(collection, buffer, indexMeta.ByteSizeLong());
|
||||
DeleteCollection(collection);
|
||||
}
|
||||
|
||||
TEST(CApiTest, GetCollectionNameTest) {
|
||||
auto collection = NewCollection(get_default_schema_config().c_str());
|
||||
auto name = GetCollectionName(collection);
|
||||
ASSERT_EQ(strcmp(name, "default-collection"), 0);
|
||||
DeleteCollection(collection);
|
||||
free((void*)(name));
|
||||
}
|
||||
29
internal/core/src/segcore/load_field_data_c_test.cpp
Normal file
29
internal/core/src/segcore/load_field_data_c_test.cpp
Normal file
@ -0,0 +1,29 @@
|
||||
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software distributed under the License
|
||||
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
|
||||
// or implied. See the License for the specific language governing permissions and limitations under the License
|
||||
|
||||
#include <gtest/gtest.h>
|
||||
|
||||
#include "segcore/collection_c.h"
|
||||
#include "segcore/segment_c.h"
|
||||
#include "segcore/reduce_c.h"
|
||||
|
||||
#include "test_utils/c_api_test_utils.h"
|
||||
#include "test_utils/storage_test_utils.h"
|
||||
#include "test_utils/GenExprProto.h"
|
||||
|
||||
TEST(CApiTest, LoadInfoTest) {
|
||||
auto load_info = std::make_shared<LoadFieldDataInfo>();
|
||||
auto c_load_info = reinterpret_cast<CLoadFieldDataInfo*>(load_info.get());
|
||||
AppendLoadFieldInfo(c_load_info, 100, 100);
|
||||
EnableMmap(c_load_info, 100, true);
|
||||
|
||||
EXPECT_TRUE(load_info->field_infos.at(100).enable_mmap);
|
||||
}
|
||||
1927
internal/core/src/segcore/load_index_c_test.cpp
Normal file
1927
internal/core/src/segcore/load_index_c_test.cpp
Normal file
File diff suppressed because it is too large
Load Diff
68
internal/core/src/segcore/plan_c_test.cpp
Normal file
68
internal/core/src/segcore/plan_c_test.cpp
Normal file
@ -0,0 +1,68 @@
|
||||
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software distributed under the License
|
||||
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
|
||||
// or implied. See the License for the specific language governing permissions and limitations under the License
|
||||
|
||||
#include <gtest/gtest.h>
|
||||
|
||||
#include "knowhere/comp/index_param.h"
|
||||
#include "segcore/plan_c.h"
|
||||
#include "test_utils/c_api_test_utils.h"
|
||||
|
||||
using namespace milvus;
|
||||
using namespace milvus::segcore;
|
||||
|
||||
template <class TraitType>
|
||||
void
|
||||
Test_CPlan(const knowhere::MetricType& metric_type) {
|
||||
std::string schema_string =
|
||||
generate_collection_schema<TraitType>(knowhere::metric::JACCARD, DIM);
|
||||
auto collection = NewCollection(schema_string.c_str());
|
||||
|
||||
milvus::proto::plan::PlanNode plan_node;
|
||||
auto vector_anns = plan_node.mutable_vector_anns();
|
||||
vector_anns->set_vector_type(TraitType::vector_type);
|
||||
vector_anns->set_placeholder_tag("$0");
|
||||
vector_anns->set_field_id(100);
|
||||
auto query_info = vector_anns->mutable_query_info();
|
||||
query_info->set_topk(10);
|
||||
query_info->set_round_decimal(3);
|
||||
query_info->set_metric_type("L2");
|
||||
query_info->set_search_params(R"({"nprobe": 10})");
|
||||
auto plan_str = plan_node.SerializeAsString();
|
||||
|
||||
void* plan = nullptr;
|
||||
auto status = CreateSearchPlanByExpr(
|
||||
collection, plan_str.data(), plan_str.size(), &plan);
|
||||
ASSERT_EQ(status.error_code, Success);
|
||||
|
||||
int64_t field_id = -1;
|
||||
status = GetFieldID(plan, &field_id);
|
||||
ASSERT_EQ(status.error_code, Success);
|
||||
|
||||
auto col = static_cast<Collection*>(collection);
|
||||
for (auto& [target_field_id, field_meta] :
|
||||
col->get_schema()->get_fields()) {
|
||||
if (field_meta.is_vector()) {
|
||||
ASSERT_EQ(field_id, target_field_id.get());
|
||||
}
|
||||
}
|
||||
ASSERT_NE(field_id, -1);
|
||||
|
||||
DeleteSearchPlan(plan);
|
||||
DeleteCollection(collection);
|
||||
}
|
||||
|
||||
TEST(CApiTest, CPlan) {
|
||||
Test_CPlan<milvus::BinaryVector>(knowhere::metric::JACCARD);
|
||||
Test_CPlan<milvus::FloatVector>(knowhere::metric::L2);
|
||||
Test_CPlan<milvus::Float16Vector>(knowhere::metric::L2);
|
||||
Test_CPlan<milvus::BFloat16Vector>(knowhere::metric::L2);
|
||||
Test_CPlan<milvus::Int8Vector>(knowhere::metric::L2);
|
||||
}
|
||||
431
internal/core/src/segcore/reduce_c_test.cpp
Normal file
431
internal/core/src/segcore/reduce_c_test.cpp
Normal file
@ -0,0 +1,431 @@
|
||||
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software distributed under the License
|
||||
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
|
||||
// or implied. See the License for the specific language governing permissions and limitations under the License
|
||||
|
||||
#include <gtest/gtest.h>
|
||||
|
||||
#include "segcore/collection_c.h"
|
||||
#include "segcore/segment_c.h"
|
||||
#include "segcore/reduce_c.h"
|
||||
|
||||
#include "test_utils/c_api_test_utils.h"
|
||||
#include "test_utils/storage_test_utils.h"
|
||||
#include "test_utils/GenExprProto.h"
|
||||
|
||||
using namespace milvus;
|
||||
using namespace milvus::segcore;
|
||||
using namespace milvus::test;
|
||||
|
||||
TEST(CApiTest, ReduceNullResult) {
|
||||
auto collection = NewCollection(get_default_schema_config().c_str());
|
||||
CSegmentInterface segment;
|
||||
auto status = NewSegment(collection, Growing, -1, &segment, false);
|
||||
ASSERT_EQ(status.error_code, Success);
|
||||
auto schema = ((milvus::segcore::Collection*)collection)->get_schema();
|
||||
int N = 10000;
|
||||
auto dataset = DataGen(schema, N);
|
||||
int64_t offset;
|
||||
|
||||
PreInsert(segment, N, &offset);
|
||||
auto insert_data = serialize(dataset.raw_);
|
||||
auto ins_res = Insert(segment,
|
||||
offset,
|
||||
N,
|
||||
dataset.row_ids_.data(),
|
||||
dataset.timestamps_.data(),
|
||||
insert_data.data(),
|
||||
insert_data.size());
|
||||
ASSERT_EQ(ins_res.error_code, Success);
|
||||
|
||||
milvus::proto::plan::PlanNode plan_node;
|
||||
auto vector_anns = plan_node.mutable_vector_anns();
|
||||
vector_anns->set_vector_type(milvus::proto::plan::VectorType::FloatVector);
|
||||
vector_anns->set_placeholder_tag("$0");
|
||||
vector_anns->set_field_id(100);
|
||||
auto query_info = vector_anns->mutable_query_info();
|
||||
query_info->set_topk(10);
|
||||
query_info->set_round_decimal(3);
|
||||
query_info->set_metric_type("L2");
|
||||
query_info->set_search_params(R"({"nprobe": 10})");
|
||||
auto plan_str = plan_node.SerializeAsString();
|
||||
|
||||
int num_queries = 10;
|
||||
|
||||
auto blob = generate_max_float_query_data(num_queries, num_queries / 2);
|
||||
|
||||
void* plan = nullptr;
|
||||
status = CreateSearchPlanByExpr(
|
||||
collection, plan_str.data(), plan_str.size(), &plan);
|
||||
ASSERT_EQ(status.error_code, Success);
|
||||
|
||||
void* placeholderGroup = nullptr;
|
||||
status = ParsePlaceholderGroup(
|
||||
plan, blob.data(), blob.length(), &placeholderGroup);
|
||||
ASSERT_EQ(status.error_code, Success);
|
||||
|
||||
std::vector<CPlaceholderGroup> placeholderGroups;
|
||||
placeholderGroups.push_back(placeholderGroup);
|
||||
dataset.timestamps_.clear();
|
||||
dataset.timestamps_.push_back(1);
|
||||
|
||||
{
|
||||
auto slice_nqs = std::vector<int64_t>{10};
|
||||
auto slice_topKs = std::vector<int64_t>{1};
|
||||
std::vector<CSearchResult> results;
|
||||
CSearchResult res;
|
||||
status = CSearch(segment, plan, placeholderGroup, 1L << 63, &res);
|
||||
ASSERT_EQ(status.error_code, Success);
|
||||
results.push_back(res);
|
||||
CSearchResultDataBlobs cSearchResultData;
|
||||
status = ReduceSearchResultsAndFillData({},
|
||||
&cSearchResultData,
|
||||
plan,
|
||||
results.data(),
|
||||
results.size(),
|
||||
slice_nqs.data(),
|
||||
slice_topKs.data(),
|
||||
slice_nqs.size());
|
||||
ASSERT_EQ(status.error_code, Success);
|
||||
|
||||
auto search_result = (SearchResult*)results[0];
|
||||
auto size = search_result->result_offsets_.size();
|
||||
EXPECT_EQ(size, num_queries / 2);
|
||||
|
||||
DeleteSearchResult(res);
|
||||
DeleteSearchResultDataBlobs(cSearchResultData);
|
||||
}
|
||||
|
||||
DeleteSearchPlan(plan);
|
||||
DeletePlaceholderGroup(placeholderGroup);
|
||||
DeleteCollection(collection);
|
||||
DeleteSegment(segment);
|
||||
}
|
||||
|
||||
TEST(CApiTest, ReduceRemoveDuplicates) {
|
||||
auto collection = NewCollection(get_default_schema_config().c_str());
|
||||
CSegmentInterface segment;
|
||||
auto status = NewSegment(collection, Growing, -1, &segment, false);
|
||||
ASSERT_EQ(status.error_code, Success);
|
||||
|
||||
auto schema = ((milvus::segcore::Collection*)collection)->get_schema();
|
||||
int N = 10000;
|
||||
auto dataset = DataGen(schema, N);
|
||||
|
||||
int64_t offset;
|
||||
PreInsert(segment, N, &offset);
|
||||
|
||||
auto insert_data = serialize(dataset.raw_);
|
||||
auto ins_res = Insert(segment,
|
||||
offset,
|
||||
N,
|
||||
dataset.row_ids_.data(),
|
||||
dataset.timestamps_.data(),
|
||||
insert_data.data(),
|
||||
insert_data.size());
|
||||
ASSERT_EQ(ins_res.error_code, Success);
|
||||
|
||||
milvus::proto::plan::PlanNode plan_node;
|
||||
auto vector_anns = plan_node.mutable_vector_anns();
|
||||
vector_anns->set_vector_type(milvus::proto::plan::VectorType::FloatVector);
|
||||
vector_anns->set_placeholder_tag("$0");
|
||||
vector_anns->set_field_id(100);
|
||||
auto query_info = vector_anns->mutable_query_info();
|
||||
query_info->set_topk(10);
|
||||
query_info->set_round_decimal(3);
|
||||
query_info->set_metric_type("L2");
|
||||
query_info->set_search_params(R"({"nprobe": 10})");
|
||||
auto plan_str = plan_node.SerializeAsString();
|
||||
|
||||
int num_queries = 10;
|
||||
int topK = 10;
|
||||
|
||||
auto blob = generate_query_data<milvus::FloatVector>(num_queries);
|
||||
|
||||
void* plan = nullptr;
|
||||
status = CreateSearchPlanByExpr(
|
||||
collection, plan_str.data(), plan_str.size(), &plan);
|
||||
ASSERT_EQ(status.error_code, Success);
|
||||
|
||||
void* placeholderGroup = nullptr;
|
||||
status = ParsePlaceholderGroup(
|
||||
plan, blob.data(), blob.length(), &placeholderGroup);
|
||||
ASSERT_EQ(status.error_code, Success);
|
||||
|
||||
std::vector<CPlaceholderGroup> placeholderGroups;
|
||||
placeholderGroups.push_back(placeholderGroup);
|
||||
dataset.timestamps_.clear();
|
||||
dataset.timestamps_.push_back(1);
|
||||
|
||||
{
|
||||
auto slice_nqs = std::vector<int64_t>{num_queries / 2, num_queries / 2};
|
||||
auto slice_topKs = std::vector<int64_t>{topK / 2, topK};
|
||||
std::vector<CSearchResult> results;
|
||||
CSearchResult res1, res2;
|
||||
status = CSearch(
|
||||
segment, plan, placeholderGroup, dataset.timestamps_[0], &res1);
|
||||
ASSERT_EQ(status.error_code, Success);
|
||||
status = CSearch(
|
||||
segment, plan, placeholderGroup, dataset.timestamps_[0], &res2);
|
||||
ASSERT_EQ(status.error_code, Success);
|
||||
results.push_back(res1);
|
||||
results.push_back(res2);
|
||||
|
||||
CSearchResultDataBlobs cSearchResultData;
|
||||
status = ReduceSearchResultsAndFillData({},
|
||||
&cSearchResultData,
|
||||
plan,
|
||||
results.data(),
|
||||
results.size(),
|
||||
slice_nqs.data(),
|
||||
slice_topKs.data(),
|
||||
slice_nqs.size());
|
||||
ASSERT_EQ(status.error_code, Success);
|
||||
// TODO:: insert no duplicate pks and check reduce results
|
||||
CheckSearchResultDuplicate(results);
|
||||
|
||||
DeleteSearchResult(res1);
|
||||
DeleteSearchResult(res2);
|
||||
DeleteSearchResultDataBlobs(cSearchResultData);
|
||||
}
|
||||
{
|
||||
int nq1 = num_queries / 3;
|
||||
int nq2 = num_queries / 3;
|
||||
int nq3 = num_queries - nq1 - nq2;
|
||||
auto slice_nqs = std::vector<int64_t>{nq1, nq2, nq3};
|
||||
auto slice_topKs = std::vector<int64_t>{topK / 2, topK, topK};
|
||||
std::vector<CSearchResult> results;
|
||||
CSearchResult res1, res2, res3;
|
||||
status = CSearch(
|
||||
segment, plan, placeholderGroup, dataset.timestamps_[0], &res1);
|
||||
ASSERT_EQ(status.error_code, Success);
|
||||
status = CSearch(
|
||||
segment, plan, placeholderGroup, dataset.timestamps_[0], &res2);
|
||||
ASSERT_EQ(status.error_code, Success);
|
||||
status = CSearch(
|
||||
segment, plan, placeholderGroup, dataset.timestamps_[0], &res3);
|
||||
ASSERT_EQ(status.error_code, Success);
|
||||
results.push_back(res1);
|
||||
results.push_back(res2);
|
||||
results.push_back(res3);
|
||||
CSearchResultDataBlobs cSearchResultData;
|
||||
status = ReduceSearchResultsAndFillData({},
|
||||
&cSearchResultData,
|
||||
plan,
|
||||
results.data(),
|
||||
results.size(),
|
||||
slice_nqs.data(),
|
||||
slice_topKs.data(),
|
||||
slice_nqs.size());
|
||||
ASSERT_EQ(status.error_code, Success);
|
||||
// TODO:: insert no duplicate pks and check reduce results
|
||||
CheckSearchResultDuplicate(results);
|
||||
|
||||
DeleteSearchResult(res1);
|
||||
DeleteSearchResult(res2);
|
||||
DeleteSearchResult(res3);
|
||||
DeleteSearchResultDataBlobs(cSearchResultData);
|
||||
}
|
||||
|
||||
DeleteSearchPlan(plan);
|
||||
DeletePlaceholderGroup(placeholderGroup);
|
||||
DeleteCollection(collection);
|
||||
DeleteSegment(segment);
|
||||
}
|
||||
|
||||
template <class TraitType>
|
||||
void
|
||||
testReduceSearchWithExpr(int N,
|
||||
int topK,
|
||||
int num_queries,
|
||||
bool filter_all = false) {
|
||||
std::cerr << "testReduceSearchWithExpr(" << N << ", " << topK << ", "
|
||||
<< num_queries << ")" << std::endl;
|
||||
|
||||
auto collection =
|
||||
NewCollection(get_default_schema_config<TraitType>().c_str());
|
||||
CSegmentInterface segment;
|
||||
auto status = NewSegment(collection, Growing, -1, &segment, false);
|
||||
ASSERT_EQ(status.error_code, Success);
|
||||
|
||||
auto schema = ((milvus::segcore::Collection*)collection)->get_schema();
|
||||
auto dataset = DataGen(schema, N);
|
||||
|
||||
int64_t offset;
|
||||
PreInsert(segment, N, &offset);
|
||||
|
||||
auto insert_data = serialize(dataset.raw_);
|
||||
auto ins_res = Insert(segment,
|
||||
offset,
|
||||
N,
|
||||
dataset.row_ids_.data(),
|
||||
dataset.timestamps_.data(),
|
||||
insert_data.data(),
|
||||
insert_data.size());
|
||||
ASSERT_EQ(ins_res.error_code, Success);
|
||||
|
||||
auto fmt = boost::format(R"(vector_anns: <
|
||||
field_id: 100
|
||||
query_info: <
|
||||
topk: %1%
|
||||
metric_type: "L2"
|
||||
search_params: "{\"nprobe\": 10}"
|
||||
>
|
||||
placeholder_tag: "$0">
|
||||
output_field_ids: 100)") %
|
||||
topK;
|
||||
|
||||
// construct the predicate that filter out all data
|
||||
if (filter_all) {
|
||||
fmt = boost::format(R"(vector_anns: <
|
||||
field_id: 100
|
||||
predicates: <
|
||||
unary_range_expr: <
|
||||
column_info: <
|
||||
field_id: 101
|
||||
data_type: Int64
|
||||
>
|
||||
op: GreaterThan
|
||||
value: <
|
||||
int64_val: %2%
|
||||
>
|
||||
>
|
||||
>
|
||||
query_info: <
|
||||
topk: %1%
|
||||
metric_type: "L2"
|
||||
search_params: "{\"nprobe\": 10}"
|
||||
>
|
||||
placeholder_tag: "$0">
|
||||
output_field_ids: 100)") %
|
||||
topK % N;
|
||||
}
|
||||
auto serialized_expr_plan = fmt.str();
|
||||
auto blob = generate_query_data<TraitType>(num_queries);
|
||||
|
||||
void* plan = nullptr;
|
||||
auto binary_plan =
|
||||
translate_text_plan_to_binary_plan(serialized_expr_plan.data());
|
||||
status = CreateSearchPlanByExpr(
|
||||
collection, binary_plan.data(), binary_plan.size(), &plan);
|
||||
ASSERT_EQ(status.error_code, Success);
|
||||
|
||||
void* placeholderGroup = nullptr;
|
||||
status = ParsePlaceholderGroup(
|
||||
plan, blob.data(), blob.length(), &placeholderGroup);
|
||||
ASSERT_EQ(status.error_code, Success);
|
||||
|
||||
std::vector<CPlaceholderGroup> placeholderGroups;
|
||||
placeholderGroups.push_back(placeholderGroup);
|
||||
dataset.timestamps_.clear();
|
||||
dataset.timestamps_.push_back(1);
|
||||
|
||||
std::vector<CSearchResult> results;
|
||||
CSearchResult res1;
|
||||
CSearchResult res2;
|
||||
auto res = CSearch(
|
||||
segment, plan, placeholderGroup, dataset.timestamps_[N - 1], &res1);
|
||||
ASSERT_EQ(res.error_code, Success);
|
||||
res = CSearch(
|
||||
segment, plan, placeholderGroup, dataset.timestamps_[N - 1], &res2);
|
||||
ASSERT_EQ(res.error_code, Success);
|
||||
results.push_back(res1);
|
||||
results.push_back(res2);
|
||||
|
||||
auto slice_nqs = std::vector<int64_t>{num_queries / 2, num_queries / 2};
|
||||
if (num_queries == 1) {
|
||||
slice_nqs = std::vector<int64_t>{num_queries};
|
||||
}
|
||||
auto slice_topKs = std::vector<int64_t>{topK / 2, topK};
|
||||
if (topK == 1) {
|
||||
slice_topKs = std::vector<int64_t>{topK, topK};
|
||||
}
|
||||
|
||||
// 1. reduce
|
||||
CSearchResultDataBlobs cSearchResultData;
|
||||
status = ReduceSearchResultsAndFillData({},
|
||||
&cSearchResultData,
|
||||
plan,
|
||||
results.data(),
|
||||
results.size(),
|
||||
slice_nqs.data(),
|
||||
slice_topKs.data(),
|
||||
slice_nqs.size());
|
||||
ASSERT_EQ(status.error_code, Success);
|
||||
|
||||
auto search_result_data_blobs =
|
||||
reinterpret_cast<milvus::segcore::SearchResultDataBlobs*>(
|
||||
cSearchResultData);
|
||||
|
||||
// check result
|
||||
for (size_t i = 0; i < slice_nqs.size(); i++) {
|
||||
milvus::proto::schema::SearchResultData search_result_data;
|
||||
auto suc = search_result_data.ParseFromArray(
|
||||
search_result_data_blobs->blobs[i].data(),
|
||||
search_result_data_blobs->blobs[i].size());
|
||||
ASSERT_TRUE(suc);
|
||||
ASSERT_EQ(search_result_data.num_queries(), slice_nqs[i]);
|
||||
ASSERT_EQ(search_result_data.top_k(), slice_topKs[i]);
|
||||
ASSERT_EQ(search_result_data.ids().int_id().data_size(),
|
||||
search_result_data.topks().at(0) * slice_nqs[i]);
|
||||
ASSERT_EQ(search_result_data.scores().size(),
|
||||
search_result_data.topks().at(0) * slice_nqs[i]);
|
||||
|
||||
// check real topks
|
||||
ASSERT_EQ(search_result_data.topks().size(), slice_nqs[i]);
|
||||
for (auto real_topk : search_result_data.topks()) {
|
||||
ASSERT_LE(real_topk, slice_topKs[i]);
|
||||
if (filter_all) {
|
||||
ASSERT_EQ(real_topk, 0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
DeleteSearchResultDataBlobs(cSearchResultData);
|
||||
DeleteSearchPlan(plan);
|
||||
DeletePlaceholderGroup(placeholderGroup);
|
||||
DeleteSearchResult(res1);
|
||||
DeleteSearchResult(res2);
|
||||
DeleteCollection(collection);
|
||||
DeleteSegment(segment);
|
||||
}
|
||||
|
||||
TEST(CApiTest, ReduceSearchWithExpr) {
|
||||
// float32
|
||||
testReduceSearchWithExpr<milvus::FloatVector>(2, 1, 1);
|
||||
testReduceSearchWithExpr<milvus::FloatVector>(2, 10, 10);
|
||||
testReduceSearchWithExpr<milvus::FloatVector>(100, 1, 1);
|
||||
testReduceSearchWithExpr<milvus::FloatVector>(100, 10, 10);
|
||||
testReduceSearchWithExpr<milvus::FloatVector>(10000, 1, 1);
|
||||
testReduceSearchWithExpr<milvus::FloatVector>(10000, 10, 10);
|
||||
// float16
|
||||
testReduceSearchWithExpr<milvus::Float16Vector>(2, 10, 10);
|
||||
testReduceSearchWithExpr<milvus::Float16Vector>(100, 10, 10);
|
||||
// bfloat16
|
||||
testReduceSearchWithExpr<milvus::BFloat16Vector>(2, 10, 10);
|
||||
testReduceSearchWithExpr<milvus::BFloat16Vector>(100, 10, 10);
|
||||
// int8
|
||||
testReduceSearchWithExpr<milvus::Int8Vector>(2, 10, 10);
|
||||
testReduceSearchWithExpr<milvus::Int8Vector>(100, 10, 10);
|
||||
}
|
||||
|
||||
TEST(CApiTest, ReduceSearchWithExprFilterAll) {
|
||||
// float32
|
||||
testReduceSearchWithExpr<milvus::FloatVector>(2, 1, 1, true);
|
||||
testReduceSearchWithExpr<milvus::FloatVector>(2, 10, 10, true);
|
||||
// float16
|
||||
testReduceSearchWithExpr<milvus::Float16Vector>(2, 1, 1, true);
|
||||
testReduceSearchWithExpr<milvus::Float16Vector>(2, 10, 10, true);
|
||||
// bfloat16
|
||||
testReduceSearchWithExpr<milvus::BFloat16Vector>(2, 1, 1, true);
|
||||
testReduceSearchWithExpr<milvus::BFloat16Vector>(2, 10, 10, true);
|
||||
// int8
|
||||
testReduceSearchWithExpr<milvus::Int8Vector>(2, 1, 1, true);
|
||||
testReduceSearchWithExpr<milvus::Int8Vector>(2, 10, 10, true);
|
||||
}
|
||||
1990
internal/core/src/segcore/segment_c_test.cpp
Normal file
1990
internal/core/src/segcore/segment_c_test.cpp
Normal file
File diff suppressed because it is too large
Load Diff
29
internal/core/src/segcore/vector_index_c_test.cpp
Normal file
29
internal/core/src/segcore/vector_index_c_test.cpp
Normal file
@ -0,0 +1,29 @@
|
||||
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software distributed under the License
|
||||
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
|
||||
// or implied. See the License for the specific language governing permissions and limitations under the License
|
||||
|
||||
#include <gtest/gtest.h>
|
||||
#include "segcore/vector_index_c.h"
|
||||
|
||||
TEST(CApiTest, GetIndexListSizeAndFeatures) {
|
||||
int size = GetIndexListSize();
|
||||
ASSERT_GT(size, 0);
|
||||
|
||||
std::vector<const char*> index_keys(size);
|
||||
std::vector<uint64_t> index_features(size);
|
||||
|
||||
GetIndexFeatures(index_keys.data(), index_features.data());
|
||||
|
||||
for (int i = 0; i < size; i++) {
|
||||
ASSERT_NE(index_keys[i], nullptr);
|
||||
ASSERT_GT(strlen(index_keys[i]), 0);
|
||||
ASSERT_GT(index_features[i], 0);
|
||||
}
|
||||
}
|
||||
@ -33,7 +33,6 @@ file(GLOB_RECURSE SOURCE_TEST_FILES
|
||||
set(MILVUS_TEST_FILES
|
||||
${SOURCE_TEST_FILES}
|
||||
init_gtest.cpp
|
||||
test_c_api.cpp
|
||||
test_loading.cpp
|
||||
test_exec.cpp
|
||||
test_expr_materialized_view.cpp
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@ -24,6 +24,8 @@
|
||||
#include "common/Types.h"
|
||||
#include "common/type_c.h"
|
||||
#include "common/VectorTrait.h"
|
||||
#include "index/Index.h"
|
||||
#include "index/IndexFactory.h"
|
||||
#include "pb/plan.pb.h"
|
||||
#include "segcore/Collection.h"
|
||||
#include "segcore/reduce/Reduce.h"
|
||||
@ -38,6 +40,7 @@
|
||||
|
||||
using namespace milvus;
|
||||
using namespace milvus::segcore;
|
||||
using namespace milvus::index;
|
||||
|
||||
// Test utility function for AppendFieldInfoForTest
|
||||
inline CStatus
|
||||
@ -242,4 +245,153 @@ CSearch(CSegmentInterface c_segment,
|
||||
return status;
|
||||
}
|
||||
|
||||
CStatus
|
||||
CRetrieve(CSegmentInterface c_segment,
|
||||
CRetrievePlan c_plan,
|
||||
uint64_t timestamp,
|
||||
CRetrieveResult** result) {
|
||||
auto future = AsyncRetrieve(
|
||||
{}, c_segment, c_plan, timestamp, DEFAULT_MAX_OUTPUT_SIZE, false, 0, 0);
|
||||
auto futurePtr = static_cast<milvus::futures::IFuture*>(
|
||||
static_cast<void*>(static_cast<CFuture*>(future)));
|
||||
|
||||
std::mutex mu;
|
||||
mu.lock();
|
||||
futurePtr->registerReadyCallback(
|
||||
[](CLockedGoMutex* mutex) { ((std::mutex*)(mutex))->unlock(); },
|
||||
(CLockedGoMutex*)(&mu));
|
||||
mu.lock();
|
||||
|
||||
auto [retrieveResult, status] = futurePtr->leakyGet();
|
||||
future_destroy(future);
|
||||
|
||||
if (status.error_code != 0) {
|
||||
return status;
|
||||
}
|
||||
*result = static_cast<CRetrieveResult*>(retrieveResult);
|
||||
return status;
|
||||
}
|
||||
|
||||
CStatus
|
||||
CRetrieveByOffsets(CSegmentInterface c_segment,
|
||||
CRetrievePlan c_plan,
|
||||
int64_t* offsets,
|
||||
int64_t len,
|
||||
CRetrieveResult** result) {
|
||||
auto future = AsyncRetrieveByOffsets({}, c_segment, c_plan, offsets, len);
|
||||
auto futurePtr = static_cast<milvus::futures::IFuture*>(
|
||||
static_cast<void*>(static_cast<CFuture*>(future)));
|
||||
|
||||
std::mutex mu;
|
||||
mu.lock();
|
||||
futurePtr->registerReadyCallback(
|
||||
[](CLockedGoMutex* mutex) { ((std::mutex*)(mutex))->unlock(); },
|
||||
(CLockedGoMutex*)(&mu));
|
||||
mu.lock();
|
||||
|
||||
auto [retrieveResult, status] = futurePtr->leakyGet();
|
||||
future_destroy(future);
|
||||
|
||||
if (status.error_code != 0) {
|
||||
return status;
|
||||
}
|
||||
*result = static_cast<CRetrieveResult*>(retrieveResult);
|
||||
return status;
|
||||
}
|
||||
|
||||
template <class TraitType>
|
||||
std::string
|
||||
generate_collection_schema(std::string metric_type, int dim) {
|
||||
namespace schema = milvus::proto::schema;
|
||||
GET_SCHEMA_DATA_TYPE_FOR_VECTOR_TRAIT
|
||||
|
||||
schema::CollectionSchema collection_schema;
|
||||
collection_schema.set_name("collection_test");
|
||||
|
||||
auto vec_field_schema = collection_schema.add_fields();
|
||||
vec_field_schema->set_name("fakevec");
|
||||
vec_field_schema->set_fieldid(100);
|
||||
vec_field_schema->set_data_type(schema_data_type);
|
||||
auto metric_type_param = vec_field_schema->add_index_params();
|
||||
metric_type_param->set_key("metric_type");
|
||||
metric_type_param->set_value(metric_type);
|
||||
auto dim_param = vec_field_schema->add_type_params();
|
||||
dim_param->set_key("dim");
|
||||
dim_param->set_value(std::to_string(dim));
|
||||
|
||||
auto other_field_schema = collection_schema.add_fields();
|
||||
other_field_schema->set_name("counter");
|
||||
other_field_schema->set_fieldid(101);
|
||||
other_field_schema->set_data_type(schema::DataType::Int64);
|
||||
other_field_schema->set_is_primary_key(true);
|
||||
|
||||
auto other_field_schema2 = collection_schema.add_fields();
|
||||
other_field_schema2->set_name("doubleField");
|
||||
other_field_schema2->set_fieldid(102);
|
||||
other_field_schema2->set_data_type(schema::DataType::Double);
|
||||
|
||||
auto other_field_schema3 = collection_schema.add_fields();
|
||||
other_field_schema3->set_name("timestamptzField");
|
||||
other_field_schema3->set_fieldid(103);
|
||||
other_field_schema3->set_data_type(schema::DataType::Timestamptz);
|
||||
|
||||
std::string schema_string;
|
||||
auto marshal = google::protobuf::TextFormat::PrintToString(
|
||||
collection_schema, &schema_string);
|
||||
assert(marshal);
|
||||
return schema_string;
|
||||
}
|
||||
|
||||
const char*
|
||||
get_default_index_meta() {
|
||||
static std::string conf = R"(maxIndexRowCount: 1000
|
||||
index_metas: <
|
||||
fieldID: 100
|
||||
collectionID: 1001
|
||||
index_name: "test-index"
|
||||
type_params: <
|
||||
key: "dim"
|
||||
value: "4"
|
||||
>
|
||||
index_params: <
|
||||
key: "index_type"
|
||||
value: "IVF_FLAT"
|
||||
>
|
||||
index_params: <
|
||||
key: "metric_type"
|
||||
value: "L2"
|
||||
>
|
||||
index_params: <
|
||||
key: "nlist"
|
||||
value: "128"
|
||||
>
|
||||
>)";
|
||||
return conf.c_str();
|
||||
}
|
||||
|
||||
IndexBasePtr
|
||||
generate_index(void* raw_data,
|
||||
DataType field_type,
|
||||
MetricType metric_type,
|
||||
IndexType index_type,
|
||||
int64_t dim,
|
||||
int64_t N) {
|
||||
auto engine_version =
|
||||
knowhere::Version::GetCurrentVersion().VersionNumber();
|
||||
CreateIndexInfo create_index_info{
|
||||
field_type, index_type, metric_type, engine_version};
|
||||
auto indexing = milvus::index::IndexFactory::GetInstance().CreateIndex(
|
||||
create_index_info, milvus::storage::FileManagerContext());
|
||||
|
||||
auto database = knowhere::GenDataSet(N, dim, raw_data);
|
||||
auto build_config = generate_build_conf(index_type, metric_type);
|
||||
indexing->BuildWithDataset(database, build_config);
|
||||
|
||||
auto vec_indexing = dynamic_cast<VectorIndex*>(indexing.get());
|
||||
EXPECT_EQ(vec_indexing->Count(), N);
|
||||
EXPECT_EQ(vec_indexing->GetDim(), dim);
|
||||
|
||||
return indexing;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user