admin/milvus
1
0
Fork 0
mirror of https://gitee.com/milvus-io/milvus.git synced 2026-01-07 19:31:51 +08:00
Code Issues Packages Projects Releases Wiki Activity

Remove useless index mode (#22934)

Browse Source 
Signed-off-by: Yudong Cai <yudong.cai@zilliz.com>
This commit is contained in:
Cai Yudong 2023-03-23 21:39:59 +08:00 committed by GitHub
parent 93bc805933
commit 0e9a4478e3
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
21 changed files with 18 additions and 142 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
internal/core/src/common/Types.h
View File

@ -130,8 +130,6 @@ using DatasetPtr = knowhere::DataSetPtr;
using MetricType = knowhere::MetricType;
// TODO :: type define milvus index type(vector index type and scalar index type)
using IndexType = knowhere::IndexType;
// TODO :: type define milvus index mode, add transfer func from milvus index mode to knowhere index mode
using IndexMode = knowhere::IndexMode;
// Plus 1 because we can't use greater(>) symbol
constexpr size_t REF_SIZE_THRESHOLD = 16 + 1;

1
internal/core/src/index/Index.h
View File

@ -46,7 +46,6 @@ class IndexBase {
protected:
IndexType index_type_ = "";
IndexMode index_mode_ = IndexMode::MODE_CPU;
};
using IndexBasePtr = std::unique_ptr<IndexBase>;

9
internal/core/src/index/IndexFactory.cpp
View File

@ -75,7 +75,6 @@ IndexFactory::CreateVectorIndex(const CreateIndexInfo& create_index_info,
auto data_type = create_index_info.field_type;
auto index_type = create_index_info.index_type;
auto metric_type = create_index_info.metric_type;
auto index_mode = create_index_info.index_mode;
#ifdef BUILD_DISK_ANN
// create disk index
@ -83,7 +82,7 @@ IndexFactory::CreateVectorIndex(const CreateIndexInfo& create_index_info,
switch (data_type) {
case DataType::VECTOR_FLOAT: {
return std::make_unique<VectorDiskAnnIndex<float>>(
index_type, metric_type, index_mode, file_manager);
index_type, metric_type, file_manager);
}
default:
throw std::invalid_argument(
@ -94,12 +93,10 @@ IndexFactory::CreateVectorIndex(const CreateIndexInfo& create_index_info,
#endif
if (is_in_nm_list(index_type)) {
return std::make_unique<VectorMemNMIndex>(
index_type, metric_type, index_mode);
return std::make_unique<VectorMemNMIndex>(index_type, metric_type);
}
// create mem index
return std::make_unique<VectorMemIndex>(
index_type, metric_type, index_mode);
return std::make_unique<VectorMemIndex>(index_type, metric_type);
}
} // namespace milvus::index

4
internal/core/src/index/IndexFactory.h
View File

@ -63,8 +63,8 @@ class IndexFactory {
IndexBasePtr
CreateScalarIndex(const CreateIndexInfo& create_index_info);
// IndexBasePtr
// CreateIndex(DataType dtype, const IndexType& index_type, const IndexMode& index_mode = IndexMode::MODE_CPU);
// IndexBasePtr
// CreateIndex(DataType dtype, const IndexType& index_type);
private:
template <typename T>
ScalarIndexPtr<T>

1
internal/core/src/index/IndexInfo.h
View File

@ -24,7 +24,6 @@ struct CreateIndexInfo {
DataType field_type;
IndexType index_type;
MetricType metric_type;
IndexMode index_mode = IndexMode::MODE_CPU;
};
} // namespace milvus::index

1
internal/core/src/index/Meta.h
View File

@ -31,7 +31,6 @@ constexpr const char* MARISA_TRIE_INDEX = "marisa_trie_index";
constexpr const char* MARISA_STR_IDS = "marisa_trie_str_ids";
constexpr const char* INDEX_TYPE = "index_type";
constexpr const char* INDEX_MODE = "index_mode";
constexpr const char* METRIC_TYPE = "metric_type";
// scalar index type

20
internal/core/src/index/Utils.cpp
View File

@ -122,26 +122,6 @@ GetIndexTypeFromConfig(const Config& config) {
return index_type.value();
}
IndexMode
GetIndexModeFromConfig(const Config& config) {
auto mode = GetValueFromConfig<std::string>(config, INDEX_MODE);
return mode.has_value() ? GetIndexMode(mode.value())
: knowhere::IndexMode::MODE_CPU;
}
IndexMode
GetIndexMode(const std::string_view index_mode) {
if (index_mode.compare("CPU") == 0 || index_mode.compare("cpu") == 0) {
return IndexMode::MODE_CPU;
}
if (index_mode.compare("GPU") == 0 || index_mode.compare("gpu") == 0) {
return IndexMode::MODE_GPU;
}
PanicInfo("unsupported index mode");
}
// TODO :: too ugly
storage::FieldDataMeta
GetFieldDataMetaFromConfig(const Config& config) {

6
internal/core/src/index/Utils.h
View File

@ -108,12 +108,6 @@ GetMetricTypeFromConfig(const Config& config);
std::string
GetIndexTypeFromConfig(const Config& config);
IndexMode
GetIndexModeFromConfig(const Config& config);
IndexMode
GetIndexMode(const std::string_view index_mode);
storage::FieldDataMeta
GetFieldDataMetaFromConfig(const Config& config);

3
internal/core/src/index/VectorDiskIndex.cpp
View File

@ -38,9 +38,8 @@ template <typename T>
VectorDiskAnnIndex<T>::VectorDiskAnnIndex(
const IndexType& index_type,
const MetricType& metric_type,
const IndexMode& index_mode,
storage::FileManagerImplPtr file_manager)
: VectorIndex(index_type, index_mode, metric_type) {
: VectorIndex(index_type, metric_type) {
file_manager_ =
std::dynamic_pointer_cast<storage::DiskFileManagerImpl>(file_manager);
auto& local_chunk_manager = storage::LocalChunkManager::GetInstance();

1
internal/core/src/index/VectorDiskIndex.h
View File

@ -30,7 +30,6 @@ class VectorDiskAnnIndex : public VectorIndex {
public:
explicit VectorDiskAnnIndex(const IndexType& index_type,
const MetricType& metric_type,
const IndexMode& index_mode,
storage::FileManagerImplPtr file_manager);
BinarySet
Serialize(const Config& config) override {

11
internal/core/src/index/VectorIndex.h
View File

@ -33,11 +33,8 @@ namespace milvus::index {
class VectorIndex : public IndexBase {
public:
explicit VectorIndex(const IndexType& index_type,
const IndexMode& index_mode,
const MetricType& metric_type)
: index_type_(index_type),
index_mode_(index_mode),
metric_type_(metric_type) {
: index_type_(index_type), metric_type_(metric_type) {
}
public:
@ -63,11 +60,6 @@ class VectorIndex : public IndexBase {
return metric_type_;
}
IndexMode
GetIndexMode() const {
return index_mode_;
}
int64_t
GetDim() const {
return dim_;
@ -84,7 +76,6 @@ class VectorIndex : public IndexBase {
private:
IndexType index_type_;
IndexMode index_mode_;
MetricType metric_type_;
int64_t dim_;
};

5
internal/core/src/index/VectorMemIndex.cpp
View File

@ -33,9 +33,8 @@
namespace milvus::index {
VectorMemIndex::VectorMemIndex(const IndexType& index_type,
const MetricType& metric_type,
const IndexMode& index_mode)
: VectorIndex(index_type, index_mode, metric_type) {
const MetricType& metric_type)
: VectorIndex(index_type, metric_type) {
AssertInfo(!is_unsupported(index_type, metric_type),
index_type + " doesn't support metric: " + metric_type);

3
internal/core/src/index/VectorMemIndex.h
View File

@ -29,8 +29,7 @@ namespace milvus::index {
class VectorMemIndex : public VectorIndex {
public:
explicit VectorMemIndex(const IndexType& index_type,
const MetricType& metric_type,
const IndexMode& index_mode);
const MetricType& metric_type);
BinarySet
Serialize(const Config& config) override;

5
internal/core/src/index/VectorMemNMIndex.h
View File

@ -29,9 +29,8 @@ namespace milvus::index {
class VectorMemNMIndex : public VectorMemIndex {
public:
explicit VectorMemNMIndex(const IndexType& index_type,
const MetricType& metric_type,
const IndexMode& index_mode)
: VectorMemIndex(index_type, metric_type, index_mode) {
const MetricType& metric_type)
: VectorMemIndex(index_type, metric_type) {
AssertInfo(is_in_nm_list(index_type), "not valid nm index type");
}

1
internal/core/src/indexbuilder/ScalarIndexCreator.cpp
View File

@ -43,7 +43,6 @@ ScalarIndexCreator::ScalarIndexCreator(DataType dtype,
milvus::index::CreateIndexInfo index_info;
index_info.field_type = dtype_;
index_info.index_type = index_type();
index_info.index_mode = IndexMode::MODE_CPU;
index_ =
index::IndexFactory::GetInstance().CreateIndex(index_info, nullptr);
}

1
internal/core/src/indexbuilder/VecIndexCreator.cpp
View File

@ -47,7 +47,6 @@ VecIndexCreator::VecIndexCreator(DataType data_type,
index::CreateIndexInfo index_info;
index_info.field_type = data_type_;
index_info.index_mode = index::GetIndexModeFromConfig(config_);
index_info.index_type = index::GetIndexTypeFromConfig(config_);
index_info.metric_type = index::GetMetricTypeFromConfig(config_);

4
internal/core/src/segcore/FieldIndexing.cpp
View File

@ -37,9 +37,7 @@ VectorFieldIndexing::BuildIndexRange(int64_t ack_beg,
for (int chunk_id = ack_beg; chunk_id < ack_end; chunk_id++) {
const auto& chunk = source->get_chunk(chunk_id);
auto indexing = std::make_unique<index::VectorMemNMIndex>(
knowhere::IndexEnum::INDEX_FAISS_IVFFLAT,
knowhere::metric::L2,
IndexMode::MODE_CPU);
knowhere::IndexEnum::INDEX_FAISS_IVFFLAT, knowhere::metric::L2);
auto dataset = knowhere::GenDataSet(
source->get_size_per_chunk(), dim, chunk.data());
indexing->BuildWithDataset(dataset, conf);

13
internal/core/src/segcore/load_index_c.cpp
View File

@ -132,13 +132,6 @@ appendVecIndex(CLoadIndexInfo c_load_index_info, CBinarySet c_binary_set) {
"metric type is empty");
index_info.metric_type = index_params.at("metric_type");
// set default index mode
index_info.index_mode = milvus::IndexMode::MODE_CPU;
if (index_params.count("index_mode")) {
index_info.index_mode =
milvus::index::GetIndexMode(index_params["index_mode"]);
}
// init file manager
milvus::storage::FieldDataMeta field_meta{
load_index_info->collection_id,
@ -191,12 +184,6 @@ appendScalarIndex(CLoadIndexInfo c_load_index_info, CBinarySet c_binary_set) {
milvus::index::CreateIndexInfo index_info;
index_info.field_type = milvus::DataType(field_type);
index_info.index_type = index_params["index_type"];
// set default index mode
index_info.index_mode = milvus::IndexMode::MODE_CPU;
if (index_params.count("index_mode")) {
index_info.index_mode =
milvus::index::GetIndexMode(index_params["index_mode"]);
}
load_index_info->index =
milvus::index::IndexFactory::GetInstance().CreateIndex(index_info,

1
internal/core/unittest/bench/bench_search.cpp
View File

@ -118,7 +118,6 @@ Search_Sealed(benchmark::State& state) {
info.index = std::move(indexing);
info.field_id = (*schema)[FieldName("fakevec")].get_id().get();
info.index_params["index_type"] = "IVF";
info.index_params["index_mode"] = "CPU";
info.index_params["metric_type"] = knowhere::metric::L2;
segment->DropFieldData(milvus::FieldId(100));
segment->LoadIndex(info);

65
internal/core/unittest/test_c_api.cpp
View File

@ -184,8 +184,7 @@ generate_collection_schema(std::string metric_type, int dim, bool is_binary) {
// VecIndexPtr
// generate_index(
// void* raw_data, knowhere::Config conf, int64_t dim, int64_t topK, int64_t N, knowhere::IndexType index_type) {
// auto indexing = knowhere::VecIndexFactory::GetInstance().CreateVecIndex(index_type,
// knowhere::IndexMode::MODE_CPU);
// auto indexing = knowhere::VecIndexFactory::GetInstance().CreateVecIndex(index_type);
//
// auto database = knowhere::GenDataset(N, dim, raw_data);
// indexing->Train(database, conf);
@ -1681,16 +1680,11 @@ TEST(CApiTest, LoadIndexInfo) {
std::string index_param_value1 = "IVF_PQ";
status = AppendIndexParam(
c_load_index_info, index_param_key1.data(), index_param_value1.data());
std::string index_param_key2 = "index_mode";
std::string index_param_value2 = "CPU";
std::string index_param_key2 = knowhere::meta::METRIC_TYPE;
std::string index_param_value2 = knowhere::metric::L2;
status = AppendIndexParam(
c_load_index_info, index_param_key2.data(), index_param_value2.data());
ASSERT_EQ(status.error_code, Success);
std::string index_param_key3 = knowhere::meta::METRIC_TYPE;
std::string index_param_value3 = knowhere::metric::L2;
status = AppendIndexParam(
c_load_index_info, index_param_key3.data(), index_param_value3.data());
ASSERT_EQ(status.error_code, Success);
std::string field_name = "field0";
status =
AppendFieldInfo(c_load_index_info, 0, 0, 0, 0, CDataType::FloatVector);
@ -1733,10 +1727,7 @@ TEST(CApiTest, LoadIndex_Search) {
milvus::segcore::LoadIndexInfo load_index_info;
auto& index_params = load_index_info.index_params;
index_params["index_type"] = "IVF_PQ";
index_params["index_mode"] = "CPU";
auto mode = knowhere::IndexMode::MODE_CPU;
load_index_info.index = std::make_unique<VectorMemIndex>(
index_params["index_type"], knowhere::metric::L2, mode);
load_index_info.index = std::make_unique<VectorMemIndex>(index_params["index_type"], knowhere::metric::L2);
load_index_info.index->Load(binary_set);
// search
@ -1859,15 +1850,11 @@ TEST(CApiTest, Indexing_Without_Predicate) {
ASSERT_EQ(status.error_code, Success);
std::string index_type_key = "index_type";
std::string index_type_value = "IVF_PQ";
std::string index_mode_key = "index_mode";
std::string index_mode_value = "CPU";
std::string metric_type_key = "metric_type";
std::string metric_type_value = "L2";
AppendIndexParam(
c_load_index_info, index_type_key.c_str(), index_type_value.c_str());
AppendIndexParam(
c_load_index_info, index_mode_key.c_str(), index_mode_value.c_str());
AppendIndexParam(
c_load_index_info, metric_type_key.c_str(), metric_type_value.c_str());
AppendFieldInfo(c_load_index_info, 0, 0, 0, 100, CDataType::FloatVector);
@ -2009,15 +1996,11 @@ TEST(CApiTest, Indexing_Expr_Without_Predicate) {
ASSERT_EQ(status.error_code, Success);
std::string index_type_key = "index_type";
std::string index_type_value = "IVF_PQ";
std::string index_mode_key = "index_mode";
std::string index_mode_value = "CPU";
std::string metric_type_key = "metric_type";
std::string metric_type_value = "L2";
AppendIndexParam(
c_load_index_info, index_type_key.c_str(), index_type_value.c_str());
AppendIndexParam(
c_load_index_info, index_mode_key.c_str(), index_mode_value.c_str());
AppendIndexParam(
c_load_index_info, metric_type_key.c_str(), metric_type_value.c_str());
AppendFieldInfo(c_load_index_info, 0, 0, 0, 100, CDataType::FloatVector);
@ -2175,15 +2158,11 @@ TEST(CApiTest, Indexing_With_float_Predicate_Range) {
ASSERT_EQ(status.error_code, Success);
std::string index_type_key = "index_type";
std::string index_type_value = "IVF_PQ";
std::string index_mode_key = "index_mode";
std::string index_mode_value = "CPU";
std::string metric_type_key = "metric_type";
std::string metric_type_value = "L2";
AppendIndexParam(
c_load_index_info, index_type_key.c_str(), index_type_value.c_str());
AppendIndexParam(
c_load_index_info, index_mode_key.c_str(), index_mode_value.c_str());
AppendIndexParam(
c_load_index_info, metric_type_key.c_str(), metric_type_value.c_str());
AppendFieldInfo(c_load_index_info, 0, 0, 0, 100, CDataType::FloatVector);
@ -2354,15 +2333,11 @@ TEST(CApiTest, Indexing_Expr_With_float_Predicate_Range) {
ASSERT_EQ(status.error_code, Success);
std::string index_type_key = "index_type";
std::string index_type_value = "IVF_PQ";
std::string index_mode_key = "index_mode";
std::string index_mode_value = "CPU";
std::string metric_type_key = "metric_type";
std::string metric_type_value = "L2";
AppendIndexParam(
c_load_index_info, index_type_key.c_str(), index_type_value.c_str());
AppendIndexParam(
c_load_index_info, index_mode_key.c_str(), index_mode_value.c_str());
AppendIndexParam(
c_load_index_info, metric_type_key.c_str(), metric_type_value.c_str());
AppendFieldInfo(c_load_index_info, 0, 0, 0, 100, CDataType::FloatVector);
@ -2516,15 +2491,11 @@ TEST(CApiTest, Indexing_With_float_Predicate_Term) {
ASSERT_EQ(status.error_code, Success);
std::string index_type_key = "index_type";
std::string index_type_value = "IVF_PQ";
std::string index_mode_key = "index_mode";
std::string index_mode_value = "CPU";
std::string metric_type_key = "metric_type";
std::string metric_type_value = "L2";
AppendIndexParam(
c_load_index_info, index_type_key.c_str(), index_type_value.c_str());
AppendIndexParam(
c_load_index_info, index_mode_key.c_str(), index_mode_value.c_str());
AppendIndexParam(
c_load_index_info, metric_type_key.c_str(), metric_type_value.c_str());
AppendFieldInfo(c_load_index_info, 0, 0, 0, 100, CDataType::FloatVector);
@ -2688,15 +2659,11 @@ TEST(CApiTest, Indexing_Expr_With_float_Predicate_Term) {
ASSERT_EQ(status.error_code, Success);
std::string index_type_key = "index_type";
std::string index_type_value = "IVF_PQ";
std::string index_mode_key = "index_mode";
std::string index_mode_value = "CPU";
std::string metric_type_key = "metric_type";
std::string metric_type_value = "L2";
AppendIndexParam(
c_load_index_info, index_type_key.c_str(), index_type_value.c_str());
AppendIndexParam(
c_load_index_info, index_mode_key.c_str(), index_mode_value.c_str());
AppendIndexParam(
c_load_index_info, metric_type_key.c_str(), metric_type_value.c_str());
AppendFieldInfo(c_load_index_info, 0, 0, 0, 100, CDataType::FloatVector);
@ -2853,15 +2820,11 @@ TEST(CApiTest, Indexing_With_binary_Predicate_Range) {
ASSERT_EQ(status.error_code, Success);
std::string index_type_key = "index_type";
std::string index_type_value = "BIN_IVF_FLAT";
std::string index_mode_key = "index_mode";
std::string index_mode_value = "CPU";
std::string metric_type_key = "metric_type";
std::string metric_type_value = "JACCARD";
AppendIndexParam(
c_load_index_info, index_type_key.c_str(), index_type_value.c_str());
AppendIndexParam(
c_load_index_info, index_mode_key.c_str(), index_mode_value.c_str());
AppendIndexParam(
c_load_index_info, metric_type_key.c_str(), metric_type_value.c_str());
AppendFieldInfo(c_load_index_info, 0, 0, 0, 100, CDataType::BinaryVector);
@ -3032,15 +2995,11 @@ TEST(CApiTest, Indexing_Expr_With_binary_Predicate_Range) {
ASSERT_EQ(status.error_code, Success);
std::string index_type_key = "index_type";
std::string index_type_value = "BIN_IVF_FLAT";
std::string index_mode_key = "index_mode";
std::string index_mode_value = "CPU";
std::string metric_type_key = "metric_type";
std::string metric_type_value = "JACCARD";
AppendIndexParam(
c_load_index_info, index_type_key.c_str(), index_type_value.c_str());
AppendIndexParam(
c_load_index_info, index_mode_key.c_str(), index_mode_value.c_str());
AppendIndexParam(
c_load_index_info, metric_type_key.c_str(), metric_type_value.c_str());
AppendFieldInfo(c_load_index_info, 0, 0, 0, 100, CDataType::BinaryVector);
@ -3196,15 +3155,11 @@ TEST(CApiTest, Indexing_With_binary_Predicate_Term) {
ASSERT_EQ(status.error_code, Success);
std::string index_type_key = "index_type";
std::string index_type_value = "BIN_IVF_FLAT";
std::string index_mode_key = "index_mode";
std::string index_mode_value = "CPU";
std::string metric_type_key = "metric_type";
std::string metric_type_value = "JACCARD";
AppendIndexParam(
c_load_index_info, index_type_key.c_str(), index_type_value.c_str());
AppendIndexParam(
c_load_index_info, index_mode_key.c_str(), index_mode_value.c_str());
AppendIndexParam(
c_load_index_info, metric_type_key.c_str(), metric_type_value.c_str());
AppendFieldInfo(c_load_index_info, 0, 0, 0, 100, CDataType::BinaryVector);
@ -3391,15 +3346,11 @@ TEST(CApiTest, Indexing_Expr_With_binary_Predicate_Term) {
ASSERT_EQ(status.error_code, Success);
std::string index_type_key = "index_type";
std::string index_type_value = "BIN_IVF_FLAT";
std::string index_mode_key = "index_mode";
std::string index_mode_value = "CPU";
std::string metric_type_key = "metric_type";
std::string metric_type_value = "JACCARD";
AppendIndexParam(
c_load_index_info, index_type_key.c_str(), index_type_value.c_str());
AppendIndexParam(
c_load_index_info, index_mode_key.c_str(), index_mode_value.c_str());
AppendIndexParam(
c_load_index_info, metric_type_key.c_str(), metric_type_value.c_str());
AppendFieldInfo(c_load_index_info, 0, 0, 0, 100, CDataType::BinaryVector);
@ -3578,15 +3529,11 @@ TEST(CApiTest, SealedSegment_search_float_Predicate_Range) {
ASSERT_EQ(status.error_code, Success);
std::string index_type_key = "index_type";
std::string index_type_value = "IVF_PQ";
std::string index_mode_key = "index_mode";
std::string index_mode_value = "CPU";
std::string metric_type_key = "metric_type";
std::string metric_type_value = "L2";
AppendIndexParam(
c_load_index_info, index_type_key.c_str(), index_type_value.c_str());
AppendIndexParam(
c_load_index_info, index_mode_key.c_str(), index_mode_value.c_str());
AppendIndexParam(
c_load_index_info, metric_type_key.c_str(), metric_type_value.c_str());
AppendFieldInfo(c_load_index_info, 0, 0, 0, 100, CDataType::FloatVector);
@ -3888,15 +3835,11 @@ TEST(CApiTest, SealedSegment_search_float_With_Expr_Predicate_Range) {
ASSERT_EQ(status.error_code, Success);
std::string index_type_key = "index_type";
std::string index_type_value = "IVF_PQ";
std::string index_mode_key = "index_mode";
std::string index_mode_value = "CPU";
std::string metric_type_key = "metric_type";
std::string metric_type_value = "L2";
AppendIndexParam(
c_load_index_info, index_type_key.c_str(), index_type_value.c_str());
AppendIndexParam(
c_load_index_info, index_mode_key.c_str(), index_mode_value.c_str());
AppendIndexParam(
c_load_index_info, metric_type_key.c_str(), metric_type_value.c_str());
AppendFieldInfo(c_load_index_info, 0, 0, 0, 100, CDataType::FloatVector);

3
internal/core/unittest/test_utils/DataGen.h
View File

@ -518,8 +518,7 @@ GenVecIndexing(int64_t N, int64_t dim, const float* vec) {
auto database = knowhere::GenDataSet(N, dim, vec);
auto indexing = std::make_unique<index::VectorMemNMIndex>(
knowhere::IndexEnum::INDEX_FAISS_IVFFLAT,
knowhere::metric::L2,
IndexMode::MODE_CPU);
knowhere::metric::L2);
indexing->BuildWithDataset(database, conf);
return indexing;
}