diff --git a/internal/core/src/indexbuilder/IndexWrapper.cpp b/internal/core/src/indexbuilder/IndexWrapper.cpp index 4484d1e119..fcced635e0 100644 --- a/internal/core/src/indexbuilder/IndexWrapper.cpp +++ b/internal/core/src/indexbuilder/IndexWrapper.cpp @@ -54,12 +54,7 @@ IndexWrapper::parse_impl(const std::string& serialized_params_str, knowhere::Con conf[key] = value; } - auto stoi_closure = [](const std::string& s) -> auto { - return std::stoi(s); - }; - auto stof_closure = [](const std::string& s) -> auto { - return std::stof(s); - }; + auto stoi_closure = [](const std::string& s) -> int { return std::stoi(s); }; /***************************** meta *******************************/ check_parameter(conf, milvus::knowhere::meta::DIM, stoi_closure, std::nullopt); @@ -93,7 +88,7 @@ IndexWrapper::parse_impl(const std::string& serialized_params_str, knowhere::Con check_parameter(conf, milvus::knowhere::IndexParams::edge_size, stoi_closure, std::nullopt); /************************** NGT Search Params *****************************/ - check_parameter(conf, milvus::knowhere::IndexParams::epsilon, stof_closure, std::nullopt); + check_parameter(conf, milvus::knowhere::IndexParams::epsilon, stoi_closure, std::nullopt); check_parameter(conf, milvus::knowhere::IndexParams::max_search_edges, stoi_closure, std::nullopt); /************************** NGT_PANNG Params *****************************/ @@ -279,12 +274,6 @@ IndexWrapper::QueryWithParam(const knowhere::DatasetPtr& dataset, const char* se std::unique_ptr IndexWrapper::QueryImpl(const knowhere::DatasetPtr& dataset, const knowhere::Config& conf) { - auto load_raw_data_closure = [&]() { LoadRawData(); }; // hide this pointer - auto index_type = get_index_type(); - if (is_in_nm_list(index_type)) { - std::call_once(raw_data_loaded_, load_raw_data_closure); - } - auto res = index_->Query(dataset, conf, nullptr); auto ids = res->Get(milvus::knowhere::meta::IDS); auto distances = res->Get(milvus::knowhere::meta::DISTANCE); @@ -302,19 +291,5 @@ IndexWrapper::QueryImpl(const knowhere::DatasetPtr& dataset, const knowhere::Con return std::move(query_res); } -void -IndexWrapper::LoadRawData() { - auto index_type = get_index_type(); - if (is_in_nm_list(index_type)) { - auto bs = index_->Serialize(config_); - auto bptr = std::make_shared(); - auto deleter = [&](uint8_t*) {}; // avoid repeated deconstruction - bptr->data = std::shared_ptr(static_cast(raw_data_.data()), deleter); - bptr->size = raw_data_.size(); - bs.Append(RAW_DATA, bptr); - index_->Load(bs); - } -} - } // namespace indexbuilder } // namespace milvus diff --git a/internal/core/src/indexbuilder/IndexWrapper.h b/internal/core/src/indexbuilder/IndexWrapper.h index 16f2721712..65c6f149fe 100644 --- a/internal/core/src/indexbuilder/IndexWrapper.h +++ b/internal/core/src/indexbuilder/IndexWrapper.h @@ -66,9 +66,6 @@ class IndexWrapper { void StoreRawData(const knowhere::DatasetPtr& dataset); - void - LoadRawData(); - template void check_parameter(knowhere::Config& conf, @@ -95,7 +92,6 @@ class IndexWrapper { milvus::json index_config_; knowhere::Config config_; std::vector raw_data_; - std::once_flag raw_data_loaded_; }; } // namespace indexbuilder diff --git a/internal/core/src/query/SearchOnSealed.cpp b/internal/core/src/query/SearchOnSealed.cpp index 5ebeedfa12..9812518398 100644 --- a/internal/core/src/query/SearchOnSealed.cpp +++ b/internal/core/src/query/SearchOnSealed.cpp @@ -49,7 +49,7 @@ void SearchOnSealed(const Schema& schema, const segcore::SealedIndexingRecord& record, const QueryInfo& query_info, - const float* query_data, + const void* query_data, int64_t num_queries, Timestamp timestamp, std::optional bitmaps_opt, @@ -58,7 +58,7 @@ SearchOnSealed(const Schema& schema, auto field_offset = query_info.field_offset_; auto& field = schema[field_offset]; - Assert(field.get_data_type() == DataType::VECTOR_FLOAT); + // Assert(field.get_data_type() == DataType::VECTOR_FLOAT); auto dim = field.get_dim(); Assert(record.test_readiness(field_offset)); diff --git a/internal/core/src/query/SearchOnSealed.h b/internal/core/src/query/SearchOnSealed.h index 4d631910d0..c79127dd44 100644 --- a/internal/core/src/query/SearchOnSealed.h +++ b/internal/core/src/query/SearchOnSealed.h @@ -20,7 +20,7 @@ void SearchOnSealed(const Schema& schema, const segcore::SealedIndexingRecord& record, const QueryInfo& query_info, - const float* query_data, + const void* query_data, int64_t num_queries, Timestamp timestamp, std::optional bitmaps_opt, diff --git a/internal/core/src/query/visitors/ExecPlanNodeVisitor.cpp b/internal/core/src/query/visitors/ExecPlanNodeVisitor.cpp index c7cb08a247..946a6088b1 100644 --- a/internal/core/src/query/visitors/ExecPlanNodeVisitor.cpp +++ b/internal/core/src/query/visitors/ExecPlanNodeVisitor.cpp @@ -104,7 +104,13 @@ ExecPlanNodeVisitor::visit(BinaryVectorANNS& node) { bitset_pack = &bitmap_holder; } - BinarySearch(*segment, node.query_info_, src_data, num_queries, timestamp_, bitset_pack, ret); + auto& sealed_indexing = segment->get_sealed_indexing_record(); + if (sealed_indexing.test_readiness(node.query_info_.field_offset_)) { + SearchOnSealed(segment->get_schema(), sealed_indexing, node.query_info_, src_data, num_queries, timestamp_, + bitset_pack, ret); + } else { + BinarySearch(*segment, node.query_info_, src_data, num_queries, timestamp_, bitset_pack, ret); + } ret_ = ret; } diff --git a/internal/core/unittest/test_index_wrapper.cpp b/internal/core/unittest/test_index_wrapper.cpp index d50f11d18a..a885c837a0 100644 --- a/internal/core/unittest/test_index_wrapper.cpp +++ b/internal/core/unittest/test_index_wrapper.cpp @@ -11,8 +11,6 @@ #include #include -#include -#include #include #include @@ -43,16 +41,16 @@ generate_conf(const milvus::knowhere::IndexType& index_type, const milvus::knowh if (index_type == milvus::knowhere::IndexEnum::INDEX_FAISS_IDMAP) { return milvus::knowhere::Config{ {milvus::knowhere::meta::DIM, DIM}, - {milvus::knowhere::meta::TOPK, K}, + // {milvus::knowhere::meta::TOPK, K}, {milvus::knowhere::Metric::TYPE, metric_type}, {milvus::knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE, 4}, }; } else if (index_type == milvus::knowhere::IndexEnum::INDEX_FAISS_IVFPQ) { return milvus::knowhere::Config{ {milvus::knowhere::meta::DIM, DIM}, - {milvus::knowhere::meta::TOPK, K}, + // {milvus::knowhere::meta::TOPK, K}, {milvus::knowhere::IndexParams::nlist, 100}, - {milvus::knowhere::IndexParams::nprobe, 4}, + // {milvus::knowhere::IndexParams::nprobe, 4}, {milvus::knowhere::IndexParams::m, 4}, {milvus::knowhere::IndexParams::nbits, 8}, {milvus::knowhere::Metric::TYPE, metric_type}, @@ -61,9 +59,9 @@ generate_conf(const milvus::knowhere::IndexType& index_type, const milvus::knowh } else if (index_type == milvus::knowhere::IndexEnum::INDEX_FAISS_IVFFLAT) { return milvus::knowhere::Config{ {milvus::knowhere::meta::DIM, DIM}, - {milvus::knowhere::meta::TOPK, K}, + // {milvus::knowhere::meta::TOPK, K}, {milvus::knowhere::IndexParams::nlist, 100}, - {milvus::knowhere::IndexParams::nprobe, 4}, + // {milvus::knowhere::IndexParams::nprobe, 4}, {milvus::knowhere::Metric::TYPE, metric_type}, {milvus::knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE, 4}, #ifdef MILVUS_GPU_VERSION @@ -73,9 +71,9 @@ generate_conf(const milvus::knowhere::IndexType& index_type, const milvus::knowh } else if (index_type == milvus::knowhere::IndexEnum::INDEX_FAISS_IVFSQ8) { return milvus::knowhere::Config{ {milvus::knowhere::meta::DIM, DIM}, - {milvus::knowhere::meta::TOPK, K}, + // {milvus::knowhere::meta::TOPK, K}, {milvus::knowhere::IndexParams::nlist, 100}, - {milvus::knowhere::IndexParams::nprobe, 4}, + // {milvus::knowhere::IndexParams::nprobe, 4}, {milvus::knowhere::IndexParams::nbits, 8}, {milvus::knowhere::Metric::TYPE, metric_type}, {milvus::knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE, 4}, @@ -86,9 +84,9 @@ generate_conf(const milvus::knowhere::IndexType& index_type, const milvus::knowh } else if (index_type == milvus::knowhere::IndexEnum::INDEX_FAISS_BIN_IVFFLAT) { return milvus::knowhere::Config{ {milvus::knowhere::meta::DIM, DIM}, - {milvus::knowhere::meta::TOPK, K}, + // {milvus::knowhere::meta::TOPK, K}, {milvus::knowhere::IndexParams::nlist, 100}, - {milvus::knowhere::IndexParams::nprobe, 4}, + // {milvus::knowhere::IndexParams::nprobe, 4}, {milvus::knowhere::IndexParams::m, 4}, {milvus::knowhere::IndexParams::nbits, 8}, {milvus::knowhere::Metric::TYPE, metric_type}, @@ -97,14 +95,13 @@ generate_conf(const milvus::knowhere::IndexType& index_type, const milvus::knowh } else if (index_type == milvus::knowhere::IndexEnum::INDEX_FAISS_BIN_IDMAP) { return milvus::knowhere::Config{ {milvus::knowhere::meta::DIM, DIM}, - {milvus::knowhere::meta::TOPK, K}, + // {milvus::knowhere::meta::TOPK, K}, {milvus::knowhere::Metric::TYPE, metric_type}, }; } else if (index_type == milvus::knowhere::IndexEnum::INDEX_NSG) { return milvus::knowhere::Config{ {milvus::knowhere::meta::DIM, DIM}, {milvus::knowhere::IndexParams::nlist, 163}, - {milvus::knowhere::meta::TOPK, K}, {milvus::knowhere::IndexParams::nprobe, 8}, {milvus::knowhere::IndexParams::knng, 20}, {milvus::knowhere::IndexParams::search_length, 40}, @@ -130,14 +127,17 @@ generate_conf(const milvus::knowhere::IndexType& index_type, const milvus::knowh #endif } else if (index_type == milvus::knowhere::IndexEnum::INDEX_HNSW) { return milvus::knowhere::Config{ - {milvus::knowhere::meta::DIM, DIM}, {milvus::knowhere::meta::TOPK, K}, - {milvus::knowhere::IndexParams::M, 16}, {milvus::knowhere::IndexParams::efConstruction, 200}, - {milvus::knowhere::IndexParams::ef, 200}, {milvus::knowhere::Metric::TYPE, metric_type}, + {milvus::knowhere::meta::DIM, DIM}, + // {milvus::knowhere::meta::TOPK, 10}, + {milvus::knowhere::IndexParams::M, 16}, + {milvus::knowhere::IndexParams::efConstruction, 200}, + {milvus::knowhere::IndexParams::ef, 200}, + {milvus::knowhere::Metric::TYPE, metric_type}, }; } else if (index_type == milvus::knowhere::IndexEnum::INDEX_ANNOY) { return milvus::knowhere::Config{ {milvus::knowhere::meta::DIM, DIM}, - {milvus::knowhere::meta::TOPK, K}, + // {milvus::knowhere::meta::TOPK, 10}, {milvus::knowhere::IndexParams::n_trees, 4}, {milvus::knowhere::IndexParams::search_k, 100}, {milvus::knowhere::Metric::TYPE, metric_type}, @@ -146,7 +146,7 @@ generate_conf(const milvus::knowhere::IndexType& index_type, const milvus::knowh } else if (index_type == milvus::knowhere::IndexEnum::INDEX_RHNSWFlat) { return milvus::knowhere::Config{ {milvus::knowhere::meta::DIM, DIM}, - {milvus::knowhere::meta::TOPK, K}, + // {milvus::knowhere::meta::TOPK, 10}, {milvus::knowhere::IndexParams::M, 16}, {milvus::knowhere::IndexParams::efConstruction, 200}, {milvus::knowhere::IndexParams::ef, 200}, @@ -156,7 +156,7 @@ generate_conf(const milvus::knowhere::IndexType& index_type, const milvus::knowh } else if (index_type == milvus::knowhere::IndexEnum::INDEX_RHNSWPQ) { return milvus::knowhere::Config{ {milvus::knowhere::meta::DIM, DIM}, - {milvus::knowhere::meta::TOPK, K}, + // {milvus::knowhere::meta::TOPK, 10}, {milvus::knowhere::IndexParams::M, 16}, {milvus::knowhere::IndexParams::efConstruction, 200}, {milvus::knowhere::IndexParams::ef, 200}, @@ -167,7 +167,7 @@ generate_conf(const milvus::knowhere::IndexType& index_type, const milvus::knowh } else if (index_type == milvus::knowhere::IndexEnum::INDEX_RHNSWSQ) { return milvus::knowhere::Config{ {milvus::knowhere::meta::DIM, DIM}, - {milvus::knowhere::meta::TOPK, K}, + // {milvus::knowhere::meta::TOPK, 10}, {milvus::knowhere::IndexParams::M, 16}, {milvus::knowhere::IndexParams::efConstruction, 200}, {milvus::knowhere::IndexParams::ef, 200}, @@ -177,7 +177,7 @@ generate_conf(const milvus::knowhere::IndexType& index_type, const milvus::knowh } else if (index_type == milvus::knowhere::IndexEnum::INDEX_NGTPANNG) { return milvus::knowhere::Config{ {milvus::knowhere::meta::DIM, DIM}, - {milvus::knowhere::meta::TOPK, K}, + // {milvus::knowhere::meta::TOPK, 10}, {milvus::knowhere::Metric::TYPE, metric_type}, {milvus::knowhere::IndexParams::edge_size, 10}, {milvus::knowhere::IndexParams::epsilon, 0.1}, @@ -189,7 +189,7 @@ generate_conf(const milvus::knowhere::IndexType& index_type, const milvus::knowh } else if (index_type == milvus::knowhere::IndexEnum::INDEX_NGTONNG) { return milvus::knowhere::Config{ {milvus::knowhere::meta::DIM, DIM}, - {milvus::knowhere::meta::TOPK, K}, + // {milvus::knowhere::meta::TOPK, 10}, {milvus::knowhere::Metric::TYPE, metric_type}, {milvus::knowhere::IndexParams::edge_size, 20}, {milvus::knowhere::IndexParams::epsilon, 0.1}, @@ -234,100 +234,6 @@ GenDataset(int64_t N, const milvus::knowhere::MetricType& metric_type, bool is_b return milvus::segcore::DataGen(schema, N); } } - -using QueryResultPtr = std::unique_ptr; -void -PrintQueryResult(const QueryResultPtr& result) { - auto nq = result->nq; - auto k = result->topk; - - std::stringstream ss_id; - std::stringstream ss_dist; - - for (auto i = 0; i < nq; i++) { - for (auto j = 0; j < k; ++j) { - ss_id << result->ids[i * k + j] << " "; - ss_dist << result->distances[i * k + j] << " "; - } - ss_id << std::endl; - ss_dist << std::endl; - } - std::cout << "id\n" << ss_id.str() << std::endl; - std::cout << "dist\n" << ss_dist.str() << std::endl; -} - -float -L2(const float* point_a, const float* point_b, int dim) { - float dis = 0; - for (auto i = 0; i < dim; i++) { - auto c_a = point_a[i]; - auto c_b = point_b[i]; - dis += pow(c_b - c_a, 2); - } - return dis; -} - -int -hamming_weight(uint8_t n) { - int count = 0; - while (n != 0) { - count += n & 1; - n >>= 1; - } - return count; -} -float -Jaccard(const uint8_t* point_a, const uint8_t* point_b, int dim) { - float dis; - int len = dim / 8; - float intersection = 0; - float union_num = 0; - for (int i = 0; i < len; i++) { - intersection += hamming_weight(point_a[i] & point_b[i]); - union_num += hamming_weight(point_a[i] | point_b[i]); - } - dis = 1 - (intersection / union_num); - return dis; -} - -float -CountDistance(const void* point_a, - const void* point_b, - int dim, - const milvus::knowhere::MetricType& metric, - bool is_binary = false) { - if (point_a == nullptr || point_b == nullptr) { - return std::numeric_limits::max(); - } - if (metric == milvus::knowhere::Metric::L2) { - return L2(static_cast(point_a), static_cast(point_b), dim); - } else if (metric == milvus::knowhere::Metric::JACCARD) { - return Jaccard(static_cast(point_a), static_cast(point_b), dim); - } else { - return std::numeric_limits::max(); - } -} - -void -CheckDistances(const QueryResultPtr& result, - const milvus::knowhere::DatasetPtr& base_dataset, - const milvus::knowhere::DatasetPtr& query_dataset, - const milvus::knowhere::MetricType& metric, - const float threshold = 1.0e-5) { - auto base_vecs = base_dataset->Get(milvus::knowhere::meta::TENSOR); - auto query_vecs = query_dataset->Get(milvus::knowhere::meta::TENSOR); - auto dim = base_dataset->Get(milvus::knowhere::meta::DIM); - auto nq = result->nq; - auto k = result->topk; - for (auto i = 0; i < nq; i++) { - for (auto j = 0; j < k; ++j) { - auto dis = result->distances[i * k + j]; - auto id = result->ids[i * k + j]; - auto count_dis = CountDistance(query_vecs + i * dim, base_vecs + id * dim, dim, metric); - // assert(std::abs(dis - count_dis) < threshold); - } - } -} } // namespace using Param = std::pair; @@ -341,26 +247,8 @@ class IndexWrapperTest : public ::testing::TestWithParam { metric_type = param.second; std::tie(type_params, index_params) = generate_params(index_type, metric_type); - std::map is_binary_map = { - {milvus::knowhere::IndexEnum::INDEX_FAISS_IDMAP, false}, - {milvus::knowhere::IndexEnum::INDEX_FAISS_IVFPQ, false}, - {milvus::knowhere::IndexEnum::INDEX_FAISS_IVFFLAT, false}, - {milvus::knowhere::IndexEnum::INDEX_FAISS_IVFSQ8, false}, - {milvus::knowhere::IndexEnum::INDEX_FAISS_BIN_IVFFLAT, true}, - {milvus::knowhere::IndexEnum::INDEX_FAISS_BIN_IDMAP, true}, -#ifdef MILVUS_SUPPORT_SPTAG - {milvus::knowhere::IndexEnum::INDEX_SPTAG_KDT_RNT, false}, - {milvus::knowhere::IndexEnum::INDEX_SPTAG_BKT_RNT, false}, -#endif - {milvus::knowhere::IndexEnum::INDEX_HNSW, false}, - {milvus::knowhere::IndexEnum::INDEX_ANNOY, false}, - {milvus::knowhere::IndexEnum::INDEX_RHNSWFlat, false}, - {milvus::knowhere::IndexEnum::INDEX_RHNSWPQ, false}, - {milvus::knowhere::IndexEnum::INDEX_RHNSWSQ, false}, - {milvus::knowhere::IndexEnum::INDEX_NGTPANNG, false}, - {milvus::knowhere::IndexEnum::INDEX_NGTONNG, false}, - {milvus::knowhere::IndexEnum::INDEX_NSG, false}, - }; + std::map is_binary_map = {{milvus::knowhere::IndexEnum::INDEX_FAISS_IVFPQ, false}, + {milvus::knowhere::IndexEnum::INDEX_FAISS_BIN_IVFFLAT, true}}; is_binary = is_binary_map[index_type]; @@ -374,13 +262,9 @@ class IndexWrapperTest : public ::testing::TestWithParam { if (!is_binary) { xb_data = dataset.get_col(0); xb_dataset = milvus::knowhere::GenDataset(NB, DIM, xb_data.data()); - xq_data = dataset.get_col(0); - xq_dataset = milvus::knowhere::GenDataset(NQ, DIM, xq_data.data()); } else { xb_bin_data = dataset.get_col(0); xb_dataset = milvus::knowhere::GenDataset(NB, DIM, xb_bin_data.data()); - xq_bin_data = dataset.get_col(0); - xq_dataset = milvus::knowhere::GenDataset(NQ, DIM, xq_bin_data.data()); } } @@ -398,9 +282,6 @@ class IndexWrapperTest : public ::testing::TestWithParam { std::vector xb_data; std::vector xb_bin_data; std::vector ids; - milvus::knowhere::DatasetPtr xq_dataset; - std::vector xq_data; - std::vector xq_bin_data; }; TEST(PQ, Build) { @@ -427,47 +308,6 @@ TEST(IVFFLATNM, Build) { ASSERT_NO_THROW(index->AddWithoutIds(xb_dataset, conf)); } -TEST(IVFFLATNM, Query) { - auto index_type = milvus::knowhere::IndexEnum::INDEX_FAISS_IVFFLAT; - auto metric_type = milvus::knowhere::Metric::L2; - auto conf = generate_conf(index_type, metric_type); - auto index = milvus::knowhere::VecIndexFactory::GetInstance().CreateVecIndex(index_type); - auto dataset = GenDataset(NB, metric_type, false); - auto xb_data = dataset.get_col(0); - auto xb_dataset = milvus::knowhere::GenDataset(NB, DIM, xb_data.data()); - ASSERT_NO_THROW(index->Train(xb_dataset, conf)); - ASSERT_NO_THROW(index->AddWithoutIds(xb_dataset, conf)); - auto bs = index->Serialize(conf); - auto bptr = std::make_shared(); - bptr->data = std::shared_ptr((uint8_t*)xb_data.data(), [&](uint8_t*) {}); - bptr->size = DIM * NB * sizeof(float); - bs.Append(RAW_DATA, bptr); - index->Load(bs); - auto xq_data = dataset.get_col(0); - auto xq_dataset = milvus::knowhere::GenDataset(NQ, DIM, xq_data.data()); - auto result = index->Query(xq_dataset, conf, nullptr); -} - -TEST(NSG, Query) { - auto index_type = milvus::knowhere::IndexEnum::INDEX_NSG; - auto metric_type = milvus::knowhere::Metric::L2; - auto conf = generate_conf(index_type, metric_type); - auto index = milvus::knowhere::VecIndexFactory::GetInstance().CreateVecIndex(index_type); - auto dataset = GenDataset(NB, metric_type, false); - auto xb_data = dataset.get_col(0); - auto xb_dataset = milvus::knowhere::GenDataset(NB, DIM, xb_data.data()); - index->BuildAll(xb_dataset, conf); - auto bs = index->Serialize(conf); - auto bptr = std::make_shared(); - bptr->data = std::shared_ptr((uint8_t*)xb_data.data(), [&](uint8_t*) {}); - bptr->size = DIM * NB * sizeof(float); - bs.Append(RAW_DATA, bptr); - index->Load(bs); - auto xq_data = dataset.get_col(0); - auto xq_dataset = milvus::knowhere::GenDataset(NQ, DIM, xq_data.data()); - auto result = index->Query(xq_dataset, conf, nullptr); -} - TEST(BINFLAT, Build) { auto index_type = milvus::knowhere::IndexEnum::INDEX_FAISS_BIN_IVFFLAT; auto metric_type = milvus::knowhere::Metric::JACCARD; @@ -645,7 +485,12 @@ TEST_P(IndexWrapperTest, Dim) { TEST_P(IndexWrapperTest, BuildWithoutIds) { auto index = std::make_unique(type_params_str.c_str(), index_params_str.c_str()); - ASSERT_NO_THROW(index->BuildWithoutIds(xb_dataset)); + + if (milvus::indexbuilder::is_in_need_id_list(index_type)) { + ASSERT_ANY_THROW(index->BuildWithoutIds(xb_dataset)); + } else { + ASSERT_NO_THROW(index->BuildWithoutIds(xb_dataset)); + } } TEST_P(IndexWrapperTest, Codec) { @@ -666,16 +511,3 @@ TEST_P(IndexWrapperTest, Codec) { ASSERT_EQ(strcmp(binary.data, copy_binary.data), 0); } } - -TEST_P(IndexWrapperTest, Query) { - auto index_wrapper = - std::make_unique(type_params_str.c_str(), index_params_str.c_str()); - - index_wrapper->BuildWithoutIds(xb_dataset); - - std::unique_ptr query_result = index_wrapper->Query(xq_dataset); - ASSERT_EQ(query_result->topk, K); - ASSERT_EQ(query_result->nq, NQ); - ASSERT_EQ(query_result->distances.size(), query_result->topk * query_result->nq); - ASSERT_EQ(query_result->ids.size(), query_result->topk * query_result->nq); -} diff --git a/internal/indexbuilder/task.go b/internal/indexbuilder/task.go index a7c62df725..0c92b153e3 100644 --- a/internal/indexbuilder/task.go +++ b/internal/indexbuilder/task.go @@ -223,26 +223,16 @@ func (it *IndexBuildTask) Execute() error { for _, value := range insertData.Data { // TODO: BinaryVectorFieldData - floatVectorFieldData, fOk := value.(*storage.FloatVectorFieldData) - if fOk { - err = it.index.BuildFloatVecIndexWithoutIds(floatVectorFieldData.Data) - if err != nil { - return err - } - } - - binaryVectorFieldData, bOk := value.(*storage.BinaryVectorFieldData) - if bOk { - err = it.index.BuildBinaryVecIndexWithoutIds(binaryVectorFieldData.Data) - if err != nil { - return err - } - } - - if !fOk || !bOk { + floatVectorFieldData, ok := value.(*storage.FloatVectorFieldData) + if !ok { return errors.New("we expect FloatVectorFieldData or BinaryVectorFieldData") } + err = it.index.BuildFloatVecIndexWithoutIds(floatVectorFieldData.Data) + if err != nil { + return err + } + indexBlobs, err := it.index.Serialize() if err != nil { return err diff --git a/internal/master/segment_manager.go b/internal/master/segment_manager.go index 63ef0a11b6..e7a4b64dce 100644 --- a/internal/master/segment_manager.go +++ b/internal/master/segment_manager.go @@ -297,8 +297,7 @@ func (manager *SegmentManagerImpl) syncWriteNodeTimestamp(timeTick Timestamp) er manager.mu.Lock() defer manager.mu.Unlock() for _, status := range manager.collStatus { - for i := 0; i < len(status.segments); i++ { - segStatus := status.segments[i] + for i, segStatus := range status.segments { if !segStatus.closable { closable, err := manager.judgeSegmentClosable(segStatus) if err != nil { @@ -319,7 +318,6 @@ func (manager *SegmentManagerImpl) syncWriteNodeTimestamp(timeTick Timestamp) er continue } status.segments = append(status.segments[:i], status.segments[i+1:]...) - i-- ts, err := manager.globalTSOAllocator() if err != nil { log.Printf("allocate tso error: %s", err.Error())