mirror of
https://gitee.com/milvus-io/milvus.git
synced 2025-12-07 01:28:27 +08:00
d6a428e880
Ref https://github.com/milvus-io/milvus/issues/42148 This PR supports create index for vector array (now, only for `DataType.FLOAT_VECTOR`) and search on it. The index type supported in this PR is `EMB_LIST_HNSW` and the metric type is `MAX_SIM` only. The way to use it: ```python milvus_client = MilvusClient("xxx:19530") schema = milvus_client.create_schema(enable_dynamic_field=True, auto_id=True) ... struct_schema = milvus_client.create_struct_array_field_schema("struct_array_field") ... struct_schema.add_field("struct_float_vec", DataType.ARRAY_OF_VECTOR, element_type=DataType.FLOAT_VECTOR, dim=128, max_capacity=1000) ... schema.add_struct_array_field(struct_schema) index_params = milvus_client.prepare_index_params() index_params.add_index(field_name="struct_float_vec", index_type="EMB_LIST_HNSW", metric_type="MAX_SIM", index_params={"nlist": 128}) ... milvus_client.create_index(COLLECTION_NAME, schema=schema, index_params=index_params) ``` Note: This PR uses `Lims` to convey offsets of the vector array to knowhere where vectors of multiple vector arrays are concatenated and we need offsets to specify which vectors belong to which vector array. --------- Signed-off-by: SpadeA <tangchenjie1210@gmail.com> Signed-off-by: SpadeA-Tang <tangchenjie1210@gmail.com>
80 lines
3.0 KiB
Go
80 lines
3.0 KiB
Go
package indexparamcheck
|
|
|
|
import (
|
|
"github.com/milvus-io/milvus/pkg/v2/common"
|
|
"github.com/milvus-io/milvus/pkg/v2/util/metric"
|
|
)
|
|
|
|
const (
|
|
MinNBits = 1
|
|
MaxNBits = 16
|
|
DefaultNBits = 8
|
|
|
|
// MinNList is the lower limit of nlist that used in Index IVFxxx
|
|
MinNList = 1
|
|
// MaxNList is the upper limit of nlist that used in Index IVFxxx
|
|
MaxNList = 65536
|
|
|
|
HNSWMinEfConstruction = 1
|
|
HNSWMaxEfConstruction = 2147483647
|
|
HNSWMinM = 1
|
|
HNSWMaxM = 2048
|
|
|
|
// DIM is a constant used to represent dimension
|
|
DIM = common.DimKey
|
|
// Metric is a constant used to metric type
|
|
Metric = common.MetricTypeKey
|
|
// NLIST is a constant used to nlist in Index IVFxxx
|
|
NLIST = "nlist"
|
|
NBITS = "nbits"
|
|
IVFM = "m"
|
|
|
|
EFConstruction = "efConstruction"
|
|
HNSWM = "M"
|
|
|
|
RaftCacheDatasetOnDevice = "cache_dataset_on_device"
|
|
|
|
// Cagra Train Param
|
|
CagraInterDegree = "intermediate_graph_degree"
|
|
CagraGraphDegree = "graph_degree"
|
|
CagraBuildAlgo = "build_algo"
|
|
|
|
CagraBuildAlgoIVFPQ = "IVF_PQ"
|
|
CagraBuildAlgoNNDESCENT = "NN_DESCENT"
|
|
|
|
// Sparse Index Param
|
|
SparseDropRatioBuild = "drop_ratio_build"
|
|
|
|
BM25K1 = "bm25_k1"
|
|
BM25B = "bm25_b"
|
|
|
|
MaxBitmapCardinalityLimit = 1000
|
|
)
|
|
|
|
var (
|
|
FloatVectorMetrics = []string{metric.L2, metric.IP, metric.COSINE} // const
|
|
SparseFloatVectorMetrics = []string{metric.IP, metric.BM25} // const
|
|
BinaryVectorMetrics = []string{metric.HAMMING, metric.JACCARD, metric.SUBSTRUCTURE, metric.SUPERSTRUCTURE, metric.MHJACCARD} // const
|
|
IntVectorMetrics = []string{metric.L2, metric.IP, metric.COSINE} // const
|
|
EmbListMetrics = []string{metric.MaxSim} // const
|
|
)
|
|
|
|
// BinIDMapMetrics is a set of all metric types supported for binary vector.
|
|
var (
|
|
BinIDMapMetrics = []string{metric.HAMMING, metric.JACCARD, metric.SUBSTRUCTURE, metric.SUPERSTRUCTURE} // const
|
|
BinIvfMetrics = []string{metric.HAMMING, metric.JACCARD} // const
|
|
HnswMetrics = []string{metric.L2, metric.IP, metric.COSINE} // const
|
|
RaftMetrics = []string{metric.L2, metric.IP}
|
|
CagraBuildAlgoTypes = []string{CagraBuildAlgoIVFPQ, CagraBuildAlgoNNDESCENT}
|
|
supportDimPerSubQuantizer = []int{32, 28, 24, 20, 16, 12, 10, 8, 6, 4, 3, 2, 1} // const
|
|
supportSubQuantizer = []int{96, 64, 56, 48, 40, 32, 28, 24, 20, 16, 12, 8, 4, 3, 2, 1} // const
|
|
SparseMetrics = []string{metric.IP, metric.BM25} // const
|
|
)
|
|
|
|
const (
|
|
FloatVectorDefaultMetricType = metric.COSINE
|
|
SparseFloatVectorDefaultMetricType = metric.IP
|
|
BinaryVectorDefaultMetricType = metric.HAMMING
|
|
IntVectorDefaultMetricType = metric.COSINE
|
|
)
|