mirror of
https://gitee.com/milvus-io/milvus.git
synced 2025-12-28 14:35:27 +08:00
3b599441fd
related: #45993 This commit extends nullable vector support to the proxy layer, querynode, and adds comprehensive validation, search reduce, and field data handling for nullable vectors with sparse storage. Proxy layer changes: - Update validate_util.go checkAligned() with getExpectedVectorRows() helper to validate nullable vector field alignment using valid data count - Update checkFloatVectorFieldData/checkSparseFloatVectorFieldData for nullable vector validation with proper row count expectations - Add FieldDataIdxComputer in typeutil/schema.go for logical-to-physical index translation during search reduce operations - Update search_reduce_util.go reduceSearchResultData to use idxComputers for correct field data indexing with nullable vectors - Update task.go, task_query.go, task_upsert.go for nullable vector handling - Update msg_pack.go with nullable vector field data processing QueryNode layer changes: - Update segments/result.go for nullable vector result handling - Update segments/search_reduce.go with nullable vector offset translation Storage and index changes: - Update data_codec.go and utils.go for nullable vector serialization - Update indexcgowrapper/dataset.go and index.go for nullable vector indexing Utility changes: - Add FieldDataIdxComputer struct with Compute() method for efficient logical-to-physical index mapping across multiple field data - Update EstimateEntitySize() and AppendFieldData() with fieldIdxs parameter - Update funcutil.go with nullable vector support functions <!-- This is an auto-generated comment: release notes by coderabbit.ai --> ## Summary by CodeRabbit * **New Features** * Full support for nullable vector fields (float, binary, float16, bfloat16, int8, sparse) across ingest, storage, indexing, search and retrieval; logical↔physical offset mapping preserves row semantics. * Client: compaction control and compaction-state APIs. * **Bug Fixes** * Improved validation for adding vector fields (nullable + dimension checks) and corrected search/query behavior for nullable vectors. * **Chores** * Persisted validity maps with indexes and on-disk formats. * **Tests** * Extensive new and updated end-to-end nullable-vector tests. <sub>✏️ Tip: You can customize this high-level summary in your review settings.</sub> <!-- end of auto-generated comment: release notes by coderabbit.ai --> --------- Signed-off-by: marcelo-cjl <marcelo.chen@zilliz.com>
2090 lines
58 KiB
Go
2090 lines
58 KiB
Go
// Licensed to the LF AI & Data foundation under one
|
|
// or more contributor license agreements. See the NOTICE file
|
|
// distributed with this work for additional information
|
|
// regarding copyright ownership. The ASF licenses this file
|
|
// to you under the Apache License, Version 2.0 (the
|
|
// "License"); you may not use this file except in compliance
|
|
// with the License. You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
package storage
|
|
|
|
import (
|
|
"encoding/binary"
|
|
"fmt"
|
|
|
|
"google.golang.org/protobuf/proto"
|
|
|
|
"github.com/milvus-io/milvus-proto/go-api/v2/schemapb"
|
|
"github.com/milvus-io/milvus/pkg/v2/common"
|
|
"github.com/milvus-io/milvus/pkg/v2/util/merr"
|
|
"github.com/milvus-io/milvus/pkg/v2/util/typeutil"
|
|
)
|
|
|
|
// TODO: fill it
|
|
// info for each blob
|
|
type BlobInfo struct {
|
|
Length int
|
|
}
|
|
|
|
// InsertData example row_schema: {float_field, int_field, float_vector_field, string_field}
|
|
// Data {<0, row_id>, <1, timestamp>, <100, float_field>, <101, int_field>, <102, float_vector_field>, <103, string_field>}
|
|
//
|
|
// system filed id:
|
|
// 0: unique row id
|
|
// 1: timestamp
|
|
// 100: first user field id
|
|
// 101: second user field id
|
|
// 102: ...
|
|
type InsertData struct {
|
|
// TODO, data should be zero copy by passing data directly to event reader or change Data to map[FieldID]FieldDataArray
|
|
Data map[FieldID]FieldData // field id to field data
|
|
Infos []BlobInfo
|
|
}
|
|
|
|
func NewInsertData(schema *schemapb.CollectionSchema) (*InsertData, error) {
|
|
return NewInsertDataWithCap(schema, 0, false)
|
|
}
|
|
|
|
func NewInsertDataWithFunctionOutputField(schema *schemapb.CollectionSchema) (*InsertData, error) {
|
|
return NewInsertDataWithCap(schema, 0, true)
|
|
}
|
|
|
|
func NewInsertDataWithCap(schema *schemapb.CollectionSchema, cap int, withFunctionOutput bool) (*InsertData, error) {
|
|
if schema == nil {
|
|
return nil, merr.WrapErrParameterMissing("collection schema")
|
|
}
|
|
|
|
idata := &InsertData{
|
|
Data: make(map[FieldID]FieldData),
|
|
}
|
|
|
|
appendField := func(field *schemapb.FieldSchema) error {
|
|
if field.IsPrimaryKey && field.GetNullable() {
|
|
return merr.WrapErrParameterInvalidMsg(fmt.Sprintf("primary key field should not be nullable (field: %s)", field.Name))
|
|
}
|
|
if field.IsPartitionKey && field.GetNullable() {
|
|
return merr.WrapErrParameterInvalidMsg(fmt.Sprintf("partition key field should not be nullable (field: %s)", field.Name))
|
|
}
|
|
if field.IsFunctionOutput {
|
|
if field.IsPrimaryKey || field.IsPartitionKey {
|
|
return merr.WrapErrParameterInvalidMsg(fmt.Sprintf("function output field should not be primary key or partition key (field: %s)", field.Name))
|
|
}
|
|
if field.GetNullable() {
|
|
return merr.WrapErrParameterInvalidMsg(fmt.Sprintf("function output field should not be nullable (field: %s)", field.Name))
|
|
}
|
|
if !withFunctionOutput {
|
|
return nil
|
|
}
|
|
}
|
|
fieldData, err := NewFieldData(field.DataType, field, cap)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
idata.Data[field.FieldID] = fieldData
|
|
return nil
|
|
}
|
|
|
|
for _, field := range schema.Fields {
|
|
if err := appendField(field); err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
for _, structField := range schema.StructArrayFields {
|
|
for _, field := range structField.GetFields() {
|
|
if err := appendField(field); err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
}
|
|
|
|
return idata, nil
|
|
}
|
|
|
|
func (iData *InsertData) IsEmpty() bool {
|
|
if iData == nil {
|
|
return true
|
|
}
|
|
|
|
timeFieldData, ok := iData.Data[common.TimeStampField]
|
|
return (!ok) || (timeFieldData.RowNum() <= 0)
|
|
}
|
|
|
|
func (i *InsertData) GetRowNum() int {
|
|
if i == nil || i.Data == nil || len(i.Data) == 0 {
|
|
return 0
|
|
}
|
|
var rowNum int
|
|
for _, data := range i.Data {
|
|
rowNum = data.RowNum()
|
|
if rowNum > 0 {
|
|
break
|
|
}
|
|
}
|
|
return rowNum
|
|
}
|
|
|
|
func (i *InsertData) GetMemorySize() int {
|
|
var size int
|
|
if i.Data == nil || len(i.Data) == 0 {
|
|
return size
|
|
}
|
|
|
|
for _, data := range i.Data {
|
|
size += data.GetMemorySize()
|
|
}
|
|
|
|
return size
|
|
}
|
|
|
|
func (i *InsertData) Append(row map[FieldID]interface{}) error {
|
|
for fID, v := range row {
|
|
field, ok := i.Data[fID]
|
|
if !ok {
|
|
return fmt.Errorf("Missing field when appending row, got %d", fID)
|
|
}
|
|
|
|
if err := field.AppendRow(v); err != nil {
|
|
return merr.WrapErrParameterInvalidMsg(fmt.Sprintf("append data for field %d failed, err=%s", fID, err.Error()))
|
|
}
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func (i *InsertData) GetRow(idx int) map[FieldID]interface{} {
|
|
res := make(map[FieldID]interface{})
|
|
for field, data := range i.Data {
|
|
res[field] = data.GetRow(idx)
|
|
}
|
|
return res
|
|
}
|
|
|
|
func (i *InsertData) GetRowSize(idx int) int {
|
|
size := 0
|
|
for _, data := range i.Data {
|
|
size += data.GetRowSize(idx)
|
|
}
|
|
return size
|
|
}
|
|
|
|
// FieldData defines field data interface
|
|
type FieldData interface {
|
|
GetMemorySize() int
|
|
RowNum() int
|
|
GetRow(i int) any
|
|
GetRowSize(i int) int
|
|
GetDataRows() any
|
|
// GetValidDataRows() any
|
|
AppendRow(row interface{}) error
|
|
AppendRows(dataRows interface{}, validDataRows interface{}) error
|
|
AppendDataRows(rows interface{}) error
|
|
AppendValidDataRows(rows interface{}) error
|
|
GetDataType() schemapb.DataType
|
|
GetNullable() bool
|
|
}
|
|
|
|
func NewFieldData(dataType schemapb.DataType, fieldSchema *schemapb.FieldSchema, cap int) (FieldData, error) {
|
|
typeParams := fieldSchema.GetTypeParams()
|
|
switch dataType {
|
|
case schemapb.DataType_Float16Vector:
|
|
dim, err := GetDimFromParams(typeParams)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
data := &Float16VectorFieldData{
|
|
Data: make([]byte, 0, cap),
|
|
Dim: dim,
|
|
Nullable: fieldSchema.GetNullable(),
|
|
}
|
|
if fieldSchema.GetNullable() {
|
|
data.ValidData = make([]bool, 0, cap)
|
|
}
|
|
return data, nil
|
|
case schemapb.DataType_BFloat16Vector:
|
|
dim, err := GetDimFromParams(typeParams)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
data := &BFloat16VectorFieldData{
|
|
Data: make([]byte, 0, cap),
|
|
Dim: dim,
|
|
Nullable: fieldSchema.GetNullable(),
|
|
}
|
|
if fieldSchema.GetNullable() {
|
|
data.ValidData = make([]bool, 0, cap)
|
|
}
|
|
return data, nil
|
|
case schemapb.DataType_FloatVector:
|
|
dim, err := GetDimFromParams(typeParams)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
data := &FloatVectorFieldData{
|
|
Data: make([]float32, 0, cap),
|
|
Dim: dim,
|
|
Nullable: fieldSchema.GetNullable(),
|
|
}
|
|
if fieldSchema.GetNullable() {
|
|
data.ValidData = make([]bool, 0, cap)
|
|
}
|
|
return data, nil
|
|
case schemapb.DataType_BinaryVector:
|
|
dim, err := GetDimFromParams(typeParams)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
data := &BinaryVectorFieldData{
|
|
Data: make([]byte, 0, cap),
|
|
Dim: dim,
|
|
Nullable: fieldSchema.GetNullable(),
|
|
}
|
|
if fieldSchema.GetNullable() {
|
|
data.ValidData = make([]bool, 0, cap)
|
|
}
|
|
return data, nil
|
|
case schemapb.DataType_SparseFloatVector:
|
|
data := &SparseFloatVectorFieldData{
|
|
Nullable: fieldSchema.GetNullable(),
|
|
}
|
|
if fieldSchema.GetNullable() {
|
|
data.ValidData = make([]bool, 0, cap)
|
|
}
|
|
return data, nil
|
|
case schemapb.DataType_Int8Vector:
|
|
dim, err := GetDimFromParams(typeParams)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
data := &Int8VectorFieldData{
|
|
Data: make([]int8, 0, cap),
|
|
Dim: dim,
|
|
Nullable: fieldSchema.GetNullable(),
|
|
}
|
|
if fieldSchema.GetNullable() {
|
|
data.ValidData = make([]bool, 0, cap)
|
|
}
|
|
return data, nil
|
|
case schemapb.DataType_Bool:
|
|
data := &BoolFieldData{
|
|
Data: make([]bool, 0, cap),
|
|
Nullable: fieldSchema.GetNullable(),
|
|
}
|
|
if fieldSchema.GetNullable() {
|
|
data.ValidData = make([]bool, 0, cap)
|
|
}
|
|
return data, nil
|
|
|
|
case schemapb.DataType_Int8:
|
|
data := &Int8FieldData{
|
|
Data: make([]int8, 0, cap),
|
|
Nullable: fieldSchema.GetNullable(),
|
|
}
|
|
if fieldSchema.GetNullable() {
|
|
data.ValidData = make([]bool, 0, cap)
|
|
}
|
|
return data, nil
|
|
|
|
case schemapb.DataType_Int16:
|
|
data := &Int16FieldData{
|
|
Data: make([]int16, 0, cap),
|
|
Nullable: fieldSchema.GetNullable(),
|
|
}
|
|
if fieldSchema.GetNullable() {
|
|
data.ValidData = make([]bool, 0, cap)
|
|
}
|
|
return data, nil
|
|
|
|
case schemapb.DataType_Int32:
|
|
data := &Int32FieldData{
|
|
Data: make([]int32, 0, cap),
|
|
Nullable: fieldSchema.GetNullable(),
|
|
}
|
|
if fieldSchema.GetNullable() {
|
|
data.ValidData = make([]bool, 0, cap)
|
|
}
|
|
return data, nil
|
|
|
|
case schemapb.DataType_Int64:
|
|
data := &Int64FieldData{
|
|
Data: make([]int64, 0, cap),
|
|
Nullable: fieldSchema.GetNullable(),
|
|
}
|
|
if fieldSchema.GetNullable() {
|
|
data.ValidData = make([]bool, 0, cap)
|
|
}
|
|
return data, nil
|
|
case schemapb.DataType_Float:
|
|
data := &FloatFieldData{
|
|
Data: make([]float32, 0, cap),
|
|
Nullable: fieldSchema.GetNullable(),
|
|
}
|
|
if fieldSchema.GetNullable() {
|
|
data.ValidData = make([]bool, 0, cap)
|
|
}
|
|
return data, nil
|
|
|
|
case schemapb.DataType_Double:
|
|
data := &DoubleFieldData{
|
|
Data: make([]float64, 0, cap),
|
|
Nullable: fieldSchema.GetNullable(),
|
|
}
|
|
if fieldSchema.GetNullable() {
|
|
data.ValidData = make([]bool, 0, cap)
|
|
}
|
|
return data, nil
|
|
case schemapb.DataType_Timestamptz:
|
|
data := &TimestamptzFieldData{
|
|
Data: make([]int64, 0, cap),
|
|
Nullable: fieldSchema.GetNullable(),
|
|
}
|
|
if fieldSchema.GetNullable() {
|
|
data.ValidData = make([]bool, 0, cap)
|
|
}
|
|
return data, nil
|
|
case schemapb.DataType_JSON:
|
|
data := &JSONFieldData{
|
|
Data: make([][]byte, 0, cap),
|
|
Nullable: fieldSchema.GetNullable(),
|
|
}
|
|
if fieldSchema.GetNullable() {
|
|
data.ValidData = make([]bool, 0, cap)
|
|
}
|
|
return data, nil
|
|
|
|
case schemapb.DataType_Geometry:
|
|
data := &GeometryFieldData{
|
|
Data: make([][]byte, 0, cap),
|
|
Nullable: fieldSchema.GetNullable(),
|
|
}
|
|
if fieldSchema.GetNullable() {
|
|
data.ValidData = make([]bool, 0, cap)
|
|
}
|
|
return data, nil
|
|
case schemapb.DataType_Array:
|
|
data := &ArrayFieldData{
|
|
Data: make([]*schemapb.ScalarField, 0, cap),
|
|
ElementType: fieldSchema.GetElementType(),
|
|
Nullable: fieldSchema.GetNullable(),
|
|
}
|
|
if fieldSchema.GetNullable() {
|
|
data.ValidData = make([]bool, 0, cap)
|
|
}
|
|
return data, nil
|
|
case schemapb.DataType_String, schemapb.DataType_VarChar:
|
|
data := &StringFieldData{
|
|
Data: make([]string, 0, cap),
|
|
DataType: dataType,
|
|
Nullable: fieldSchema.GetNullable(),
|
|
}
|
|
if fieldSchema.GetNullable() {
|
|
data.ValidData = make([]bool, 0, cap)
|
|
}
|
|
return data, nil
|
|
case schemapb.DataType_ArrayOfVector:
|
|
dim, err := GetDimFromParams(typeParams)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
data := &VectorArrayFieldData{
|
|
Dim: int64(dim),
|
|
Data: make([]*schemapb.VectorField, 0, cap),
|
|
ElementType: fieldSchema.GetElementType(),
|
|
}
|
|
return data, nil
|
|
default:
|
|
return nil, fmt.Errorf("Unexpected schema data type: %d", dataType)
|
|
}
|
|
}
|
|
|
|
type BoolFieldData struct {
|
|
Data []bool
|
|
ValidData []bool
|
|
Nullable bool
|
|
}
|
|
type Int8FieldData struct {
|
|
Data []int8
|
|
ValidData []bool
|
|
Nullable bool
|
|
}
|
|
type Int16FieldData struct {
|
|
Data []int16
|
|
ValidData []bool
|
|
Nullable bool
|
|
}
|
|
type Int32FieldData struct {
|
|
Data []int32
|
|
ValidData []bool
|
|
Nullable bool
|
|
}
|
|
type Int64FieldData struct {
|
|
Data []int64
|
|
ValidData []bool
|
|
Nullable bool
|
|
}
|
|
type FloatFieldData struct {
|
|
Data []float32
|
|
ValidData []bool
|
|
Nullable bool
|
|
}
|
|
type DoubleFieldData struct {
|
|
Data []float64
|
|
ValidData []bool
|
|
Nullable bool
|
|
}
|
|
type StringFieldData struct {
|
|
Data []string
|
|
DataType schemapb.DataType
|
|
ValidData []bool
|
|
Nullable bool
|
|
}
|
|
type ArrayFieldData struct {
|
|
ElementType schemapb.DataType
|
|
Data []*schemapb.ScalarField
|
|
ValidData []bool
|
|
Nullable bool
|
|
}
|
|
type JSONFieldData struct {
|
|
Data [][]byte
|
|
ValidData []bool
|
|
Nullable bool
|
|
}
|
|
type TimestamptzFieldData struct {
|
|
Data []int64
|
|
ValidData []bool
|
|
Nullable bool
|
|
}
|
|
type GeometryFieldData struct {
|
|
Data [][]byte
|
|
ValidData []bool
|
|
Nullable bool
|
|
}
|
|
|
|
// LogicalToPhysicalMapping maps logical offset to physical offset for nullable vector
|
|
type LogicalToPhysicalMapping struct {
|
|
validCount int
|
|
l2pMap map[int]int
|
|
}
|
|
|
|
func (m *LogicalToPhysicalMapping) GetPhysicalOffset(logicalOffset int) int {
|
|
if m.l2pMap == nil {
|
|
return logicalOffset
|
|
}
|
|
if physicalOffset, ok := m.l2pMap[logicalOffset]; ok {
|
|
return physicalOffset
|
|
}
|
|
return -1
|
|
}
|
|
|
|
func (m *LogicalToPhysicalMapping) GetMemorySize() int {
|
|
size := 8 // validCount int
|
|
size += len(m.l2pMap) * 16 // map[int]int, roughly 16 bytes per entry
|
|
return size
|
|
}
|
|
|
|
func (m *LogicalToPhysicalMapping) GetValidCount() int {
|
|
return m.validCount
|
|
}
|
|
|
|
func (m *LogicalToPhysicalMapping) Build(validData []bool, startLogical, totalCount int) {
|
|
if totalCount == 0 {
|
|
return
|
|
}
|
|
if len(validData) < totalCount {
|
|
return
|
|
}
|
|
|
|
if m.l2pMap == nil {
|
|
m.l2pMap = make(map[int]int)
|
|
}
|
|
|
|
physicalIdx := m.validCount
|
|
for i := 0; i < totalCount; i++ {
|
|
if validData[i] {
|
|
m.l2pMap[startLogical+i] = physicalIdx
|
|
physicalIdx++
|
|
}
|
|
}
|
|
m.validCount = physicalIdx
|
|
}
|
|
|
|
type BinaryVectorFieldData struct {
|
|
Data []byte
|
|
ValidData []bool
|
|
Dim int
|
|
Nullable bool
|
|
L2PMapping LogicalToPhysicalMapping
|
|
}
|
|
type FloatVectorFieldData struct {
|
|
Data []float32
|
|
ValidData []bool
|
|
Dim int
|
|
Nullable bool
|
|
L2PMapping LogicalToPhysicalMapping
|
|
}
|
|
type Float16VectorFieldData struct {
|
|
Data []byte
|
|
ValidData []bool
|
|
Dim int
|
|
Nullable bool
|
|
L2PMapping LogicalToPhysicalMapping
|
|
}
|
|
type BFloat16VectorFieldData struct {
|
|
Data []byte
|
|
ValidData []bool
|
|
Dim int
|
|
Nullable bool
|
|
L2PMapping LogicalToPhysicalMapping
|
|
}
|
|
|
|
type SparseFloatVectorFieldData struct {
|
|
schemapb.SparseFloatArray
|
|
ValidData []bool
|
|
Nullable bool
|
|
L2PMapping LogicalToPhysicalMapping
|
|
}
|
|
|
|
type Int8VectorFieldData struct {
|
|
Data []int8
|
|
ValidData []bool
|
|
Dim int
|
|
Nullable bool
|
|
L2PMapping LogicalToPhysicalMapping
|
|
}
|
|
|
|
type VectorArrayFieldData struct {
|
|
Dim int64
|
|
ElementType schemapb.DataType
|
|
Data []*schemapb.VectorField
|
|
}
|
|
|
|
func (dst *SparseFloatVectorFieldData) AppendAllRows(src *SparseFloatVectorFieldData) {
|
|
if len(src.Contents) == 0 {
|
|
return
|
|
}
|
|
if dst.Dim < src.Dim {
|
|
dst.Dim = src.Dim
|
|
}
|
|
dst.Contents = append(dst.Contents, src.Contents...)
|
|
if src.Nullable {
|
|
if dst.ValidData == nil {
|
|
dst.ValidData = make([]bool, 0, len(src.ValidData))
|
|
}
|
|
dst.L2PMapping.Build(src.ValidData, len(dst.ValidData), len(src.ValidData))
|
|
dst.ValidData = append(dst.ValidData, src.ValidData...)
|
|
dst.Nullable = true
|
|
}
|
|
}
|
|
|
|
// RowNum implements FieldData.RowNum
|
|
func (data *BoolFieldData) RowNum() int { return len(data.Data) }
|
|
func (data *Int8FieldData) RowNum() int { return len(data.Data) }
|
|
func (data *Int16FieldData) RowNum() int { return len(data.Data) }
|
|
func (data *Int32FieldData) RowNum() int { return len(data.Data) }
|
|
func (data *Int64FieldData) RowNum() int { return len(data.Data) }
|
|
func (data *FloatFieldData) RowNum() int { return len(data.Data) }
|
|
func (data *DoubleFieldData) RowNum() int { return len(data.Data) }
|
|
func (data *TimestamptzFieldData) RowNum() int { return len(data.Data) }
|
|
func (data *StringFieldData) RowNum() int { return len(data.Data) }
|
|
func (data *ArrayFieldData) RowNum() int { return len(data.Data) }
|
|
func (data *JSONFieldData) RowNum() int { return len(data.Data) }
|
|
func (data *GeometryFieldData) RowNum() int { return len(data.Data) }
|
|
func (data *BinaryVectorFieldData) RowNum() int {
|
|
if data.Nullable {
|
|
return len(data.ValidData)
|
|
}
|
|
return len(data.Data) * 8 / data.Dim
|
|
}
|
|
|
|
func (data *FloatVectorFieldData) RowNum() int {
|
|
if data.Nullable {
|
|
return len(data.ValidData)
|
|
}
|
|
return len(data.Data) / data.Dim
|
|
}
|
|
|
|
func (data *Float16VectorFieldData) RowNum() int {
|
|
if data.Nullable {
|
|
return len(data.ValidData)
|
|
}
|
|
return len(data.Data) / 2 / data.Dim
|
|
}
|
|
|
|
func (data *BFloat16VectorFieldData) RowNum() int {
|
|
if data.Nullable {
|
|
return len(data.ValidData)
|
|
}
|
|
return len(data.Data) / 2 / data.Dim
|
|
}
|
|
|
|
func (data *SparseFloatVectorFieldData) RowNum() int {
|
|
if data.Nullable {
|
|
return len(data.ValidData)
|
|
}
|
|
return len(data.Contents)
|
|
}
|
|
|
|
func (data *Int8VectorFieldData) RowNum() int {
|
|
if data.Nullable {
|
|
return len(data.ValidData)
|
|
}
|
|
return len(data.Data) / data.Dim
|
|
}
|
|
|
|
func (data *VectorArrayFieldData) RowNum() int {
|
|
return len(data.Data)
|
|
}
|
|
|
|
// GetRow implements FieldData.GetRow
|
|
func (data *BoolFieldData) GetRow(i int) any {
|
|
if data.GetNullable() && !data.ValidData[i] {
|
|
return nil
|
|
}
|
|
return data.Data[i]
|
|
}
|
|
|
|
func (data *Int8FieldData) GetRow(i int) any {
|
|
if data.GetNullable() && !data.ValidData[i] {
|
|
return nil
|
|
}
|
|
return data.Data[i]
|
|
}
|
|
|
|
func (data *Int16FieldData) GetRow(i int) any {
|
|
if data.GetNullable() && !data.ValidData[i] {
|
|
return nil
|
|
}
|
|
return data.Data[i]
|
|
}
|
|
|
|
func (data *Int32FieldData) GetRow(i int) any {
|
|
if data.GetNullable() && !data.ValidData[i] {
|
|
return nil
|
|
}
|
|
return data.Data[i]
|
|
}
|
|
|
|
func (data *Int64FieldData) GetRow(i int) any {
|
|
if data.GetNullable() && !data.ValidData[i] {
|
|
return nil
|
|
}
|
|
return data.Data[i]
|
|
}
|
|
|
|
func (data *TimestamptzFieldData) GetRow(i int) any {
|
|
if data.GetNullable() && !data.ValidData[i] {
|
|
return nil
|
|
}
|
|
return data.Data[i]
|
|
}
|
|
|
|
func (data *FloatFieldData) GetRow(i int) any {
|
|
if data.GetNullable() && !data.ValidData[i] {
|
|
return nil
|
|
}
|
|
return data.Data[i]
|
|
}
|
|
|
|
func (data *DoubleFieldData) GetRow(i int) any {
|
|
if data.GetNullable() && !data.ValidData[i] {
|
|
return nil
|
|
}
|
|
return data.Data[i]
|
|
}
|
|
|
|
func (data *StringFieldData) GetRow(i int) any {
|
|
if data.GetNullable() && !data.ValidData[i] {
|
|
return nil
|
|
}
|
|
return data.Data[i]
|
|
}
|
|
|
|
func (data *ArrayFieldData) GetRow(i int) any {
|
|
if data.GetNullable() && !data.ValidData[i] {
|
|
return nil
|
|
}
|
|
return data.Data[i]
|
|
}
|
|
|
|
func (data *JSONFieldData) GetRow(i int) any {
|
|
if data.GetNullable() && !data.ValidData[i] {
|
|
return nil
|
|
}
|
|
return data.Data[i]
|
|
}
|
|
|
|
func (data *GeometryFieldData) GetRow(i int) any {
|
|
if data.GetNullable() && !data.ValidData[i] {
|
|
return nil
|
|
}
|
|
return data.Data[i]
|
|
}
|
|
|
|
func (data *BinaryVectorFieldData) GetRow(i int) any {
|
|
if data.GetNullable() && !data.ValidData[i] {
|
|
return nil
|
|
}
|
|
physicalIdx := data.L2PMapping.GetPhysicalOffset(i)
|
|
return data.Data[physicalIdx*data.Dim/8 : (physicalIdx+1)*data.Dim/8]
|
|
}
|
|
|
|
func (data *SparseFloatVectorFieldData) GetRow(i int) interface{} {
|
|
if data.GetNullable() && !data.ValidData[i] {
|
|
return nil
|
|
}
|
|
physicalIdx := data.L2PMapping.GetPhysicalOffset(i)
|
|
return data.Contents[physicalIdx]
|
|
}
|
|
|
|
func (data *FloatVectorFieldData) GetRow(i int) interface{} {
|
|
if data.GetNullable() && !data.ValidData[i] {
|
|
return nil
|
|
}
|
|
physicalIdx := data.L2PMapping.GetPhysicalOffset(i)
|
|
return data.Data[physicalIdx*data.Dim : (physicalIdx+1)*data.Dim]
|
|
}
|
|
|
|
func (data *Float16VectorFieldData) GetRow(i int) interface{} {
|
|
if data.GetNullable() && !data.ValidData[i] {
|
|
return nil
|
|
}
|
|
physicalIdx := data.L2PMapping.GetPhysicalOffset(i)
|
|
return data.Data[physicalIdx*data.Dim*2 : (physicalIdx+1)*data.Dim*2]
|
|
}
|
|
|
|
func (data *BFloat16VectorFieldData) GetRow(i int) interface{} {
|
|
if data.GetNullable() && !data.ValidData[i] {
|
|
return nil
|
|
}
|
|
physicalIdx := data.L2PMapping.GetPhysicalOffset(i)
|
|
return data.Data[physicalIdx*data.Dim*2 : (physicalIdx+1)*data.Dim*2]
|
|
}
|
|
|
|
func (data *Int8VectorFieldData) GetRow(i int) interface{} {
|
|
if data.GetNullable() && !data.ValidData[i] {
|
|
return nil
|
|
}
|
|
physicalIdx := data.L2PMapping.GetPhysicalOffset(i)
|
|
return data.Data[physicalIdx*data.Dim : (physicalIdx+1)*data.Dim]
|
|
}
|
|
|
|
func (data *VectorArrayFieldData) GetRow(i int) interface{} {
|
|
return data.Data[i]
|
|
}
|
|
|
|
func (data *BoolFieldData) GetDataRows() any { return data.Data }
|
|
func (data *Int8FieldData) GetDataRows() any { return data.Data }
|
|
func (data *Int16FieldData) GetDataRows() any { return data.Data }
|
|
func (data *Int32FieldData) GetDataRows() any { return data.Data }
|
|
func (data *Int64FieldData) GetDataRows() any { return data.Data }
|
|
func (data *FloatFieldData) GetDataRows() any { return data.Data }
|
|
func (data *DoubleFieldData) GetDataRows() any { return data.Data }
|
|
func (data *TimestamptzFieldData) GetDataRows() any { return data.Data }
|
|
func (data *StringFieldData) GetDataRows() any { return data.Data }
|
|
func (data *ArrayFieldData) GetDataRows() any { return data.Data }
|
|
func (data *JSONFieldData) GetDataRows() any { return data.Data }
|
|
func (data *GeometryFieldData) GetDataRows() any { return data.Data }
|
|
func (data *BinaryVectorFieldData) GetDataRows() any { return data.Data }
|
|
func (data *FloatVectorFieldData) GetDataRows() any { return data.Data }
|
|
func (data *Float16VectorFieldData) GetDataRows() any { return data.Data }
|
|
func (data *BFloat16VectorFieldData) GetDataRows() any { return data.Data }
|
|
func (data *SparseFloatVectorFieldData) GetDataRows() any { return data.Contents }
|
|
func (data *Int8VectorFieldData) GetDataRows() any { return data.Data }
|
|
func (data *VectorArrayFieldData) GetDataRows() any { return data.Data }
|
|
|
|
// AppendRow implements FieldData.AppendRow
|
|
func (data *BoolFieldData) AppendRow(row interface{}) error {
|
|
if data.GetNullable() && row == nil {
|
|
data.Data = append(data.Data, make([]bool, 1)...)
|
|
data.ValidData = append(data.ValidData, false)
|
|
return nil
|
|
}
|
|
v, ok := row.(bool)
|
|
if !ok {
|
|
return merr.WrapErrParameterInvalid("bool", row, "Wrong row type")
|
|
}
|
|
data.Data = append(data.Data, v)
|
|
if data.GetNullable() {
|
|
data.ValidData = append(data.ValidData, true)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (data *Int8FieldData) AppendRow(row interface{}) error {
|
|
if data.GetNullable() && row == nil {
|
|
data.Data = append(data.Data, make([]int8, 1)...)
|
|
data.ValidData = append(data.ValidData, false)
|
|
return nil
|
|
}
|
|
v, ok := row.(int8)
|
|
if !ok {
|
|
return merr.WrapErrParameterInvalid("int8", row, "Wrong row type")
|
|
}
|
|
if data.GetNullable() {
|
|
data.ValidData = append(data.ValidData, true)
|
|
}
|
|
data.Data = append(data.Data, v)
|
|
return nil
|
|
}
|
|
|
|
func (data *Int16FieldData) AppendRow(row interface{}) error {
|
|
if data.GetNullable() && row == nil {
|
|
data.Data = append(data.Data, make([]int16, 1)...)
|
|
data.ValidData = append(data.ValidData, false)
|
|
return nil
|
|
}
|
|
v, ok := row.(int16)
|
|
if !ok {
|
|
return merr.WrapErrParameterInvalid("int16", row, "Wrong row type")
|
|
}
|
|
if data.GetNullable() {
|
|
data.ValidData = append(data.ValidData, true)
|
|
}
|
|
data.Data = append(data.Data, v)
|
|
return nil
|
|
}
|
|
|
|
func (data *Int32FieldData) AppendRow(row interface{}) error {
|
|
if data.GetNullable() && row == nil {
|
|
data.Data = append(data.Data, make([]int32, 1)...)
|
|
data.ValidData = append(data.ValidData, false)
|
|
return nil
|
|
}
|
|
v, ok := row.(int32)
|
|
if !ok {
|
|
return merr.WrapErrParameterInvalid("int32", row, "Wrong row type")
|
|
}
|
|
if data.GetNullable() {
|
|
data.ValidData = append(data.ValidData, true)
|
|
}
|
|
data.Data = append(data.Data, v)
|
|
return nil
|
|
}
|
|
|
|
func (data *Int64FieldData) AppendRow(row interface{}) error {
|
|
if data.GetNullable() && row == nil {
|
|
data.Data = append(data.Data, make([]int64, 1)...)
|
|
data.ValidData = append(data.ValidData, false)
|
|
return nil
|
|
}
|
|
v, ok := row.(int64)
|
|
if !ok {
|
|
return merr.WrapErrParameterInvalid("int64", row, "Wrong row type")
|
|
}
|
|
if data.GetNullable() {
|
|
data.ValidData = append(data.ValidData, true)
|
|
}
|
|
data.Data = append(data.Data, v)
|
|
return nil
|
|
}
|
|
|
|
func (data *FloatFieldData) AppendRow(row interface{}) error {
|
|
if data.GetNullable() && row == nil {
|
|
data.Data = append(data.Data, make([]float32, 1)...)
|
|
data.ValidData = append(data.ValidData, false)
|
|
return nil
|
|
}
|
|
v, ok := row.(float32)
|
|
if !ok {
|
|
return merr.WrapErrParameterInvalid("float32", row, "Wrong row type")
|
|
}
|
|
if data.GetNullable() {
|
|
data.ValidData = append(data.ValidData, true)
|
|
}
|
|
data.Data = append(data.Data, v)
|
|
return nil
|
|
}
|
|
|
|
func (data *DoubleFieldData) AppendRow(row interface{}) error {
|
|
if data.GetNullable() && row == nil {
|
|
data.Data = append(data.Data, make([]float64, 1)...)
|
|
data.ValidData = append(data.ValidData, false)
|
|
return nil
|
|
}
|
|
v, ok := row.(float64)
|
|
if !ok {
|
|
return merr.WrapErrParameterInvalid("float64", row, "Wrong row type")
|
|
}
|
|
if data.GetNullable() {
|
|
data.ValidData = append(data.ValidData, true)
|
|
}
|
|
data.Data = append(data.Data, v)
|
|
return nil
|
|
}
|
|
|
|
func (data *TimestamptzFieldData) AppendRow(row interface{}) error {
|
|
if data.GetNullable() && row == nil {
|
|
data.Data = append(data.Data, make([]int64, 1)...)
|
|
data.ValidData = append(data.ValidData, false)
|
|
return nil
|
|
}
|
|
v, ok := row.(int64)
|
|
if !ok {
|
|
return merr.WrapErrParameterInvalid("timestamptz", row, "Wrong row type")
|
|
}
|
|
if data.GetNullable() {
|
|
data.ValidData = append(data.ValidData, true)
|
|
}
|
|
data.Data = append(data.Data, v)
|
|
return nil
|
|
}
|
|
|
|
func (data *StringFieldData) AppendRow(row interface{}) error {
|
|
if data.GetNullable() && row == nil {
|
|
data.Data = append(data.Data, make([]string, 1)...)
|
|
data.ValidData = append(data.ValidData, false)
|
|
return nil
|
|
}
|
|
v, ok := row.(string)
|
|
if !ok {
|
|
return merr.WrapErrParameterInvalid("string", row, "Wrong row type")
|
|
}
|
|
if data.GetNullable() {
|
|
data.ValidData = append(data.ValidData, true)
|
|
}
|
|
data.Data = append(data.Data, v)
|
|
return nil
|
|
}
|
|
|
|
func (data *ArrayFieldData) AppendRow(row interface{}) error {
|
|
if data.GetNullable() && row == nil {
|
|
data.Data = append(data.Data, make([]*schemapb.ScalarField, 1)...)
|
|
data.ValidData = append(data.ValidData, false)
|
|
return nil
|
|
}
|
|
v, ok := row.(*schemapb.ScalarField)
|
|
if !ok {
|
|
return merr.WrapErrParameterInvalid("*schemapb.ScalarField", row, "Wrong row type")
|
|
}
|
|
if data.GetNullable() {
|
|
data.ValidData = append(data.ValidData, true)
|
|
}
|
|
data.Data = append(data.Data, v)
|
|
return nil
|
|
}
|
|
|
|
func (data *JSONFieldData) AppendRow(row interface{}) error {
|
|
if data.GetNullable() && row == nil {
|
|
data.Data = append(data.Data, make([][]byte, 1)...)
|
|
data.ValidData = append(data.ValidData, false)
|
|
return nil
|
|
}
|
|
v, ok := row.([]byte)
|
|
if !ok {
|
|
return merr.WrapErrParameterInvalid("[]byte", row, "Wrong row type")
|
|
}
|
|
if data.GetNullable() {
|
|
data.ValidData = append(data.ValidData, true)
|
|
}
|
|
data.Data = append(data.Data, v)
|
|
return nil
|
|
}
|
|
|
|
func (data *GeometryFieldData) AppendRow(row interface{}) error {
|
|
if data.GetNullable() && row == nil {
|
|
data.Data = append(data.Data, make([][]byte, 1)...)
|
|
data.ValidData = append(data.ValidData, false)
|
|
return nil
|
|
}
|
|
switch v := row.(type) {
|
|
case []byte:
|
|
data.Data = append(data.Data, v)
|
|
case string:
|
|
data.Data = append(data.Data, []byte(v))
|
|
default:
|
|
return merr.WrapErrParameterInvalid("[]byte", row, "Wrong row type")
|
|
}
|
|
if data.GetNullable() {
|
|
data.ValidData = append(data.ValidData, true)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (data *BinaryVectorFieldData) AppendRow(row interface{}) error {
|
|
if data.GetNullable() && row == nil {
|
|
data.L2PMapping.Build([]bool{false}, len(data.ValidData), 1)
|
|
data.ValidData = append(data.ValidData, false)
|
|
return nil
|
|
}
|
|
v, ok := row.([]byte)
|
|
if !ok || len(v) != data.Dim/8 {
|
|
return merr.WrapErrParameterInvalid("[]byte", row, "Wrong row type")
|
|
}
|
|
data.Data = append(data.Data, v...)
|
|
if data.GetNullable() {
|
|
data.L2PMapping.Build([]bool{true}, len(data.ValidData), 1)
|
|
data.ValidData = append(data.ValidData, true)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (data *FloatVectorFieldData) AppendRow(row interface{}) error {
|
|
if data.GetNullable() && row == nil {
|
|
data.L2PMapping.Build([]bool{false}, len(data.ValidData), 1)
|
|
data.ValidData = append(data.ValidData, false)
|
|
return nil
|
|
}
|
|
v, ok := row.([]float32)
|
|
if !ok || len(v) != data.Dim {
|
|
return merr.WrapErrParameterInvalid("[]float32", row, "Wrong row type")
|
|
}
|
|
data.Data = append(data.Data, v...)
|
|
if data.GetNullable() {
|
|
data.L2PMapping.Build([]bool{true}, len(data.ValidData), 1)
|
|
data.ValidData = append(data.ValidData, true)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (data *Float16VectorFieldData) AppendRow(row interface{}) error {
|
|
if data.GetNullable() && row == nil {
|
|
data.L2PMapping.Build([]bool{false}, len(data.ValidData), 1)
|
|
data.ValidData = append(data.ValidData, false)
|
|
return nil
|
|
}
|
|
v, ok := row.([]byte)
|
|
if !ok || len(v) != data.Dim*2 {
|
|
return merr.WrapErrParameterInvalid("[]byte", row, "Wrong row type")
|
|
}
|
|
data.Data = append(data.Data, v...)
|
|
if data.GetNullable() {
|
|
data.L2PMapping.Build([]bool{true}, len(data.ValidData), 1)
|
|
data.ValidData = append(data.ValidData, true)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (data *BFloat16VectorFieldData) AppendRow(row interface{}) error {
|
|
if data.GetNullable() && row == nil {
|
|
data.L2PMapping.Build([]bool{false}, len(data.ValidData), 1)
|
|
data.ValidData = append(data.ValidData, false)
|
|
return nil
|
|
}
|
|
v, ok := row.([]byte)
|
|
if !ok || len(v) != data.Dim*2 {
|
|
return merr.WrapErrParameterInvalid("[]byte", row, "Wrong row type")
|
|
}
|
|
data.Data = append(data.Data, v...)
|
|
if data.GetNullable() {
|
|
data.L2PMapping.Build([]bool{true}, len(data.ValidData), 1)
|
|
data.ValidData = append(data.ValidData, true)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (data *SparseFloatVectorFieldData) AppendRow(row interface{}) error {
|
|
if data.GetNullable() && row == nil {
|
|
data.L2PMapping.Build([]bool{false}, len(data.ValidData), 1)
|
|
data.ValidData = append(data.ValidData, false)
|
|
return nil
|
|
}
|
|
v, ok := row.([]byte)
|
|
if !ok {
|
|
return merr.WrapErrParameterInvalid("SparseFloatVectorRowData", row, "Wrong row type")
|
|
}
|
|
if err := typeutil.ValidateSparseFloatRows(v); err != nil {
|
|
return err
|
|
}
|
|
rowDim := typeutil.SparseFloatRowDim(v)
|
|
if data.Dim < rowDim {
|
|
data.Dim = rowDim
|
|
}
|
|
data.Contents = append(data.Contents, v)
|
|
if data.GetNullable() {
|
|
data.L2PMapping.Build([]bool{true}, len(data.ValidData), 1)
|
|
data.ValidData = append(data.ValidData, true)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (data *Int8VectorFieldData) AppendRow(row interface{}) error {
|
|
if data.GetNullable() && row == nil {
|
|
data.L2PMapping.Build([]bool{false}, len(data.ValidData), 1)
|
|
data.ValidData = append(data.ValidData, false)
|
|
return nil
|
|
}
|
|
v, ok := row.([]int8)
|
|
if !ok || len(v) != data.Dim {
|
|
return merr.WrapErrParameterInvalid("[]int8", row, "Wrong row type")
|
|
}
|
|
data.Data = append(data.Data, v...)
|
|
if data.GetNullable() {
|
|
data.L2PMapping.Build([]bool{true}, len(data.ValidData), 1)
|
|
data.ValidData = append(data.ValidData, true)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (data *VectorArrayFieldData) AppendRow(row interface{}) error {
|
|
v, ok := row.(*schemapb.VectorField)
|
|
if !ok {
|
|
return merr.WrapErrParameterInvalid("[]*schemapb.VectorField", row, "Wrong row type")
|
|
}
|
|
data.Data = append(data.Data, v)
|
|
return nil
|
|
}
|
|
|
|
func (data *BoolFieldData) AppendRows(dataRows interface{}, validDataRows interface{}) error {
|
|
err := data.AppendDataRows(dataRows)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return data.AppendValidDataRows(validDataRows)
|
|
}
|
|
|
|
func (data *Int8FieldData) AppendRows(dataRows interface{}, validDataRows interface{}) error {
|
|
err := data.AppendDataRows(dataRows)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return data.AppendValidDataRows(validDataRows)
|
|
}
|
|
|
|
func (data *Int16FieldData) AppendRows(dataRows interface{}, validDataRows interface{}) error {
|
|
err := data.AppendDataRows(dataRows)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return data.AppendValidDataRows(validDataRows)
|
|
}
|
|
|
|
func (data *Int32FieldData) AppendRows(dataRows interface{}, validDataRows interface{}) error {
|
|
err := data.AppendDataRows(dataRows)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return data.AppendValidDataRows(validDataRows)
|
|
}
|
|
|
|
func (data *Int64FieldData) AppendRows(dataRows interface{}, validDataRows interface{}) error {
|
|
err := data.AppendDataRows(dataRows)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return data.AppendValidDataRows(validDataRows)
|
|
}
|
|
|
|
func (data *FloatFieldData) AppendRows(dataRows interface{}, validDataRows interface{}) error {
|
|
err := data.AppendDataRows(dataRows)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return data.AppendValidDataRows(validDataRows)
|
|
}
|
|
|
|
func (data *DoubleFieldData) AppendRows(dataRows interface{}, validDataRows interface{}) error {
|
|
err := data.AppendDataRows(dataRows)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return data.AppendValidDataRows(validDataRows)
|
|
}
|
|
|
|
func (data *TimestamptzFieldData) AppendRows(dataRows interface{}, validDataRows interface{}) error {
|
|
err := data.AppendDataRows(dataRows)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return data.AppendValidDataRows(validDataRows)
|
|
}
|
|
|
|
func (data *StringFieldData) AppendRows(dataRows interface{}, validDataRows interface{}) error {
|
|
err := data.AppendDataRows(dataRows)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return data.AppendValidDataRows(validDataRows)
|
|
}
|
|
|
|
func (data *ArrayFieldData) AppendRows(dataRows interface{}, validDataRows interface{}) error {
|
|
err := data.AppendDataRows(dataRows)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return data.AppendValidDataRows(validDataRows)
|
|
}
|
|
|
|
func (data *JSONFieldData) AppendRows(dataRows interface{}, validDataRows interface{}) error {
|
|
err := data.AppendDataRows(dataRows)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return data.AppendValidDataRows(validDataRows)
|
|
}
|
|
|
|
func (data *GeometryFieldData) AppendRows(dataRows interface{}, validDataRows interface{}) error {
|
|
err := data.AppendDataRows(dataRows)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return data.AppendValidDataRows(validDataRows)
|
|
}
|
|
|
|
// AppendDataRows appends FLATTEN vectors to field data.
|
|
func (data *BinaryVectorFieldData) AppendRows(dataRows interface{}, validDataRows interface{}) error {
|
|
err := data.AppendDataRows(dataRows)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return data.AppendValidDataRows(validDataRows)
|
|
}
|
|
|
|
// AppendDataRows appends FLATTEN vectors to field data.
|
|
func (data *FloatVectorFieldData) AppendRows(dataRows interface{}, validDataRows interface{}) error {
|
|
err := data.AppendDataRows(dataRows)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return data.AppendValidDataRows(validDataRows)
|
|
}
|
|
|
|
// AppendDataRows appends FLATTEN vectors to field data.
|
|
func (data *Float16VectorFieldData) AppendRows(dataRows interface{}, validDataRows interface{}) error {
|
|
err := data.AppendDataRows(dataRows)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return data.AppendValidDataRows(validDataRows)
|
|
}
|
|
|
|
// AppendDataRows appends FLATTEN vectors to field data.
|
|
func (data *BFloat16VectorFieldData) AppendRows(dataRows interface{}, validDataRows interface{}) error {
|
|
err := data.AppendDataRows(dataRows)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return data.AppendValidDataRows(validDataRows)
|
|
}
|
|
|
|
func (data *SparseFloatVectorFieldData) AppendRows(dataRows interface{}, validDataRows interface{}) error {
|
|
err := data.AppendDataRows(dataRows)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return data.AppendValidDataRows(validDataRows)
|
|
}
|
|
|
|
func (data *Int8VectorFieldData) AppendRows(dataRows interface{}, validDataRows interface{}) error {
|
|
err := data.AppendDataRows(dataRows)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return data.AppendValidDataRows(validDataRows)
|
|
}
|
|
|
|
func (data *VectorArrayFieldData) AppendRows(dataRows interface{}, validDataRows interface{}) error {
|
|
err := data.AppendDataRows(dataRows)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return data.AppendValidDataRows(validDataRows)
|
|
}
|
|
|
|
func (data *BoolFieldData) AppendDataRows(rows interface{}) error {
|
|
v, ok := rows.([]bool)
|
|
if !ok {
|
|
return merr.WrapErrParameterInvalid("[]bool", rows, "Wrong rows type")
|
|
}
|
|
data.Data = append(data.Data, v...)
|
|
return nil
|
|
}
|
|
|
|
func (data *Int8FieldData) AppendDataRows(rows interface{}) error {
|
|
v, ok := rows.([]int8)
|
|
if !ok {
|
|
return merr.WrapErrParameterInvalid("[]int8", rows, "Wrong rows type")
|
|
}
|
|
data.Data = append(data.Data, v...)
|
|
return nil
|
|
}
|
|
|
|
func (data *Int16FieldData) AppendDataRows(rows interface{}) error {
|
|
v, ok := rows.([]int16)
|
|
if !ok {
|
|
return merr.WrapErrParameterInvalid("[]int16", rows, "Wrong rows type")
|
|
}
|
|
data.Data = append(data.Data, v...)
|
|
return nil
|
|
}
|
|
|
|
func (data *Int32FieldData) AppendDataRows(rows interface{}) error {
|
|
v, ok := rows.([]int32)
|
|
if !ok {
|
|
return merr.WrapErrParameterInvalid("[]int32", rows, "Wrong rows type")
|
|
}
|
|
data.Data = append(data.Data, v...)
|
|
return nil
|
|
}
|
|
|
|
func (data *Int64FieldData) AppendDataRows(rows interface{}) error {
|
|
v, ok := rows.([]int64)
|
|
if !ok {
|
|
return merr.WrapErrParameterInvalid("[]int64", rows, "Wrong rows type")
|
|
}
|
|
data.Data = append(data.Data, v...)
|
|
return nil
|
|
}
|
|
|
|
func (data *FloatFieldData) AppendDataRows(rows interface{}) error {
|
|
v, ok := rows.([]float32)
|
|
if !ok {
|
|
return merr.WrapErrParameterInvalid("[]float32", rows, "Wrong rows type")
|
|
}
|
|
data.Data = append(data.Data, v...)
|
|
return nil
|
|
}
|
|
|
|
func (data *DoubleFieldData) AppendDataRows(rows interface{}) error {
|
|
v, ok := rows.([]float64)
|
|
if !ok {
|
|
return merr.WrapErrParameterInvalid("[]float64", rows, "Wrong rows type")
|
|
}
|
|
data.Data = append(data.Data, v...)
|
|
return nil
|
|
}
|
|
|
|
func (data *TimestamptzFieldData) AppendDataRows(rows interface{}) error {
|
|
v, ok := rows.([]int64)
|
|
if !ok {
|
|
return merr.WrapErrParameterInvalid("[]timestamptz", rows, "Wrong rows type")
|
|
}
|
|
data.Data = append(data.Data, v...)
|
|
return nil
|
|
}
|
|
|
|
func (data *StringFieldData) AppendDataRows(rows interface{}) error {
|
|
v, ok := rows.([]string)
|
|
if !ok {
|
|
return merr.WrapErrParameterInvalid("[]string", rows, "Wrong rows type")
|
|
}
|
|
data.Data = append(data.Data, v...)
|
|
return nil
|
|
}
|
|
|
|
func (data *ArrayFieldData) AppendDataRows(rows interface{}) error {
|
|
v, ok := rows.([]*schemapb.ScalarField)
|
|
if !ok {
|
|
return merr.WrapErrParameterInvalid("[]*schemapb.ScalarField", rows, "Wrong rows type")
|
|
}
|
|
data.Data = append(data.Data, v...)
|
|
return nil
|
|
}
|
|
|
|
func (data *JSONFieldData) AppendDataRows(rows interface{}) error {
|
|
v, ok := rows.([][]byte)
|
|
if !ok {
|
|
return merr.WrapErrParameterInvalid("[][]byte", rows, "Wrong rows type")
|
|
}
|
|
data.Data = append(data.Data, v...)
|
|
return nil
|
|
}
|
|
|
|
func (data *GeometryFieldData) AppendDataRows(rows interface{}) error {
|
|
switch v := rows.(type) {
|
|
case [][]byte:
|
|
data.Data = append(data.Data, v...)
|
|
case []string:
|
|
for _, row := range v {
|
|
data.Data = append(data.Data, []byte(row))
|
|
}
|
|
default:
|
|
return merr.WrapErrParameterInvalid("[][]byte", rows, "Wrong rows type")
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// AppendDataRows appends FLATTEN vectors to field data.
|
|
func (data *BinaryVectorFieldData) AppendDataRows(rows interface{}) error {
|
|
v, ok := rows.([]byte)
|
|
if !ok {
|
|
return merr.WrapErrParameterInvalid("[]byte", rows, "Wrong rows type")
|
|
}
|
|
if len(v)%(data.Dim/8) != 0 {
|
|
return merr.WrapErrParameterInvalid(data.Dim/8, len(v), "Wrong vector size")
|
|
}
|
|
data.Data = append(data.Data, v...)
|
|
return nil
|
|
}
|
|
|
|
// AppendDataRows appends FLATTEN vectors to field data.
|
|
func (data *FloatVectorFieldData) AppendDataRows(rows interface{}) error {
|
|
v, ok := rows.([]float32)
|
|
if !ok {
|
|
return merr.WrapErrParameterInvalid("[]float32", rows, "Wrong rows type")
|
|
}
|
|
if len(v)%(data.Dim) != 0 {
|
|
return merr.WrapErrParameterInvalid(data.Dim, len(v), "Wrong vector size")
|
|
}
|
|
data.Data = append(data.Data, v...)
|
|
return nil
|
|
}
|
|
|
|
// AppendDataRows appends FLATTEN vectors to field data.
|
|
func (data *Float16VectorFieldData) AppendDataRows(rows interface{}) error {
|
|
v, ok := rows.([]byte)
|
|
if !ok {
|
|
return merr.WrapErrParameterInvalid("[]byte", rows, "Wrong rows type")
|
|
}
|
|
if len(v)%(data.Dim*2) != 0 {
|
|
return merr.WrapErrParameterInvalid(data.Dim*2, len(v), "Wrong vector size")
|
|
}
|
|
data.Data = append(data.Data, v...)
|
|
return nil
|
|
}
|
|
|
|
// AppendDataRows appends FLATTEN vectors to field data.
|
|
func (data *BFloat16VectorFieldData) AppendDataRows(rows interface{}) error {
|
|
v, ok := rows.([]byte)
|
|
if !ok {
|
|
return merr.WrapErrParameterInvalid("[]byte", rows, "Wrong rows type")
|
|
}
|
|
if len(v)%(data.Dim*2) != 0 {
|
|
return merr.WrapErrParameterInvalid(data.Dim*2, len(v), "Wrong vector size")
|
|
}
|
|
data.Data = append(data.Data, v...)
|
|
return nil
|
|
}
|
|
|
|
func (data *SparseFloatVectorFieldData) AppendDataRows(rows interface{}) error {
|
|
v, ok := rows.(*SparseFloatVectorFieldData)
|
|
if !ok {
|
|
return merr.WrapErrParameterInvalid("SparseFloatVectorFieldData", rows, "Wrong rows type")
|
|
}
|
|
data.Contents = append(data.SparseFloatArray.Contents, v.Contents...)
|
|
if data.Dim < v.Dim {
|
|
data.Dim = v.Dim
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (data *Int8VectorFieldData) AppendDataRows(rows interface{}) error {
|
|
v, ok := rows.([]int8)
|
|
if !ok {
|
|
return merr.WrapErrParameterInvalid("[]int8", rows, "Wrong rows type")
|
|
}
|
|
if len(v)%(data.Dim) != 0 {
|
|
return merr.WrapErrParameterInvalid(data.Dim, len(v), "Wrong vector size")
|
|
}
|
|
data.Data = append(data.Data, v...)
|
|
return nil
|
|
}
|
|
|
|
func (data *VectorArrayFieldData) AppendDataRows(rows interface{}) error {
|
|
v, ok := rows.([]*schemapb.VectorField)
|
|
if !ok {
|
|
return merr.WrapErrParameterInvalid("[]*schemapb.VectorField", rows, "Wrong rows type")
|
|
}
|
|
data.Data = append(data.Data, v...)
|
|
return nil
|
|
}
|
|
|
|
func (data *BoolFieldData) AppendValidDataRows(rows interface{}) error {
|
|
if rows == nil {
|
|
return nil
|
|
}
|
|
v, ok := rows.([]bool)
|
|
if !ok {
|
|
return merr.WrapErrParameterInvalid("[]bool", rows, "Wrong rows type")
|
|
}
|
|
data.ValidData = append(data.ValidData, v...)
|
|
return nil
|
|
}
|
|
|
|
func (data *Int8FieldData) AppendValidDataRows(rows interface{}) error {
|
|
if rows == nil {
|
|
return nil
|
|
}
|
|
v, ok := rows.([]bool)
|
|
if !ok {
|
|
return merr.WrapErrParameterInvalid("[]bool", rows, "Wrong rows type")
|
|
}
|
|
data.ValidData = append(data.ValidData, v...)
|
|
return nil
|
|
}
|
|
|
|
func (data *Int16FieldData) AppendValidDataRows(rows interface{}) error {
|
|
if rows == nil {
|
|
return nil
|
|
}
|
|
v, ok := rows.([]bool)
|
|
if !ok {
|
|
return merr.WrapErrParameterInvalid("[]bool", rows, "Wrong rows type")
|
|
}
|
|
data.ValidData = append(data.ValidData, v...)
|
|
return nil
|
|
}
|
|
|
|
func (data *Int32FieldData) AppendValidDataRows(rows interface{}) error {
|
|
if rows == nil {
|
|
return nil
|
|
}
|
|
v, ok := rows.([]bool)
|
|
if !ok {
|
|
return merr.WrapErrParameterInvalid("[]bool", rows, "Wrong rows type")
|
|
}
|
|
data.ValidData = append(data.ValidData, v...)
|
|
return nil
|
|
}
|
|
|
|
func (data *Int64FieldData) AppendValidDataRows(rows interface{}) error {
|
|
if rows == nil {
|
|
return nil
|
|
}
|
|
v, ok := rows.([]bool)
|
|
if !ok {
|
|
return merr.WrapErrParameterInvalid("[]bool", rows, "Wrong rows type")
|
|
}
|
|
data.ValidData = append(data.ValidData, v...)
|
|
return nil
|
|
}
|
|
|
|
func (data *FloatFieldData) AppendValidDataRows(rows interface{}) error {
|
|
if rows == nil {
|
|
return nil
|
|
}
|
|
v, ok := rows.([]bool)
|
|
if !ok {
|
|
return merr.WrapErrParameterInvalid("[]bool", rows, "Wrong rows type")
|
|
}
|
|
data.ValidData = append(data.ValidData, v...)
|
|
return nil
|
|
}
|
|
|
|
func (data *DoubleFieldData) AppendValidDataRows(rows interface{}) error {
|
|
if rows == nil {
|
|
return nil
|
|
}
|
|
v, ok := rows.([]bool)
|
|
if !ok {
|
|
return merr.WrapErrParameterInvalid("[]bool", rows, "Wrong rows type")
|
|
}
|
|
data.ValidData = append(data.ValidData, v...)
|
|
return nil
|
|
}
|
|
|
|
func (data *TimestamptzFieldData) AppendValidDataRows(rows interface{}) error {
|
|
if rows == nil {
|
|
return nil
|
|
}
|
|
v, ok := rows.([]bool)
|
|
if !ok {
|
|
return merr.WrapErrParameterInvalid("[]bool", rows, "Wrong rows type")
|
|
}
|
|
data.ValidData = append(data.ValidData, v...)
|
|
return nil
|
|
}
|
|
|
|
func (data *StringFieldData) AppendValidDataRows(rows interface{}) error {
|
|
if rows == nil {
|
|
return nil
|
|
}
|
|
v, ok := rows.([]bool)
|
|
if !ok {
|
|
return merr.WrapErrParameterInvalid("[]bool", rows, "Wrong rows type")
|
|
}
|
|
data.ValidData = append(data.ValidData, v...)
|
|
return nil
|
|
}
|
|
|
|
func (data *ArrayFieldData) AppendValidDataRows(rows interface{}) error {
|
|
if rows == nil {
|
|
return nil
|
|
}
|
|
v, ok := rows.([]bool)
|
|
if !ok {
|
|
return merr.WrapErrParameterInvalid("[]bool", rows, "Wrong rows type")
|
|
}
|
|
data.ValidData = append(data.ValidData, v...)
|
|
return nil
|
|
}
|
|
|
|
func (data *JSONFieldData) AppendValidDataRows(rows interface{}) error {
|
|
if rows == nil {
|
|
return nil
|
|
}
|
|
v, ok := rows.([]bool)
|
|
if !ok {
|
|
return merr.WrapErrParameterInvalid("[]bool", rows, "Wrong rows type")
|
|
}
|
|
data.ValidData = append(data.ValidData, v...)
|
|
return nil
|
|
}
|
|
|
|
func (data *GeometryFieldData) AppendValidDataRows(rows interface{}) error {
|
|
if rows == nil {
|
|
return nil
|
|
}
|
|
v, ok := rows.([]bool)
|
|
if !ok {
|
|
return merr.WrapErrParameterInvalid("[]bool", rows, "Wrong rows type")
|
|
}
|
|
data.ValidData = append(data.ValidData, v...)
|
|
return nil
|
|
}
|
|
|
|
// AppendValidDataRows appends FLATTEN vectors to field data.
|
|
func (data *BinaryVectorFieldData) AppendValidDataRows(rows interface{}) error {
|
|
if rows == nil {
|
|
return nil
|
|
}
|
|
v, ok := rows.([]bool)
|
|
if !ok {
|
|
return merr.WrapErrParameterInvalid("[]bool", rows, "Wrong rows type")
|
|
}
|
|
data.L2PMapping.Build(v, len(data.ValidData), len(v))
|
|
data.ValidData = append(data.ValidData, v...)
|
|
return nil
|
|
}
|
|
|
|
func (data *VectorArrayFieldData) AppendValidDataRows(rows interface{}) error {
|
|
if rows != nil {
|
|
v, ok := rows.([]bool)
|
|
if !ok {
|
|
return merr.WrapErrParameterInvalid("[]bool", rows, "Wrong rows type")
|
|
}
|
|
if len(v) != 0 {
|
|
return merr.WrapErrParameterInvalidMsg("not support Nullable in vector")
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// AppendValidDataRows appends FLATTEN vectors to field data.
|
|
func (data *FloatVectorFieldData) AppendValidDataRows(rows interface{}) error {
|
|
if rows == nil {
|
|
return nil
|
|
}
|
|
v, ok := rows.([]bool)
|
|
if !ok {
|
|
return merr.WrapErrParameterInvalid("[]bool", rows, "Wrong rows type")
|
|
}
|
|
data.L2PMapping.Build(v, len(data.ValidData), len(v))
|
|
data.ValidData = append(data.ValidData, v...)
|
|
return nil
|
|
}
|
|
|
|
// AppendValidDataRows appends FLATTEN vectors to field data.
|
|
func (data *Float16VectorFieldData) AppendValidDataRows(rows interface{}) error {
|
|
if rows == nil {
|
|
return nil
|
|
}
|
|
v, ok := rows.([]bool)
|
|
if !ok {
|
|
return merr.WrapErrParameterInvalid("[]bool", rows, "Wrong rows type")
|
|
}
|
|
data.L2PMapping.Build(v, len(data.ValidData), len(v))
|
|
data.ValidData = append(data.ValidData, v...)
|
|
return nil
|
|
}
|
|
|
|
// AppendValidDataRows appends FLATTEN vectors to field data.
|
|
func (data *BFloat16VectorFieldData) AppendValidDataRows(rows interface{}) error {
|
|
if rows == nil {
|
|
return nil
|
|
}
|
|
v, ok := rows.([]bool)
|
|
if !ok {
|
|
return merr.WrapErrParameterInvalid("[]bool", rows, "Wrong rows type")
|
|
}
|
|
data.L2PMapping.Build(v, len(data.ValidData), len(v))
|
|
data.ValidData = append(data.ValidData, v...)
|
|
return nil
|
|
}
|
|
|
|
func (data *SparseFloatVectorFieldData) AppendValidDataRows(rows interface{}) error {
|
|
if rows == nil {
|
|
return nil
|
|
}
|
|
v, ok := rows.([]bool)
|
|
if !ok {
|
|
return merr.WrapErrParameterInvalid("[]bool", rows, "Wrong rows type")
|
|
}
|
|
data.L2PMapping.Build(v, len(data.ValidData), len(v))
|
|
data.ValidData = append(data.ValidData, v...)
|
|
return nil
|
|
}
|
|
|
|
func (data *Int8VectorFieldData) AppendValidDataRows(rows interface{}) error {
|
|
if rows == nil {
|
|
return nil
|
|
}
|
|
v, ok := rows.([]bool)
|
|
if !ok {
|
|
return merr.WrapErrParameterInvalid("[]bool", rows, "Wrong rows type")
|
|
}
|
|
data.L2PMapping.Build(v, len(data.ValidData), len(v))
|
|
data.ValidData = append(data.ValidData, v...)
|
|
return nil
|
|
}
|
|
|
|
// GetMemorySize implements FieldData.GetMemorySize
|
|
func (data *BoolFieldData) GetMemorySize() int {
|
|
return binary.Size(data.Data) + binary.Size(data.ValidData) + binary.Size(data.Nullable)
|
|
}
|
|
|
|
func (data *Int8FieldData) GetMemorySize() int {
|
|
return binary.Size(data.Data) + binary.Size(data.ValidData) + binary.Size(data.Nullable)
|
|
}
|
|
|
|
func (data *Int16FieldData) GetMemorySize() int {
|
|
return binary.Size(data.Data) + binary.Size(data.ValidData) + binary.Size(data.Nullable)
|
|
}
|
|
|
|
func (data *Int32FieldData) GetMemorySize() int {
|
|
return binary.Size(data.Data) + binary.Size(data.ValidData) + binary.Size(data.Nullable)
|
|
}
|
|
|
|
func (data *Int64FieldData) GetMemorySize() int {
|
|
return binary.Size(data.Data) + binary.Size(data.ValidData) + binary.Size(data.Nullable)
|
|
}
|
|
|
|
func (data *FloatFieldData) GetMemorySize() int {
|
|
return binary.Size(data.Data) + binary.Size(data.ValidData) + binary.Size(data.Nullable)
|
|
}
|
|
|
|
func (data *DoubleFieldData) GetMemorySize() int {
|
|
return binary.Size(data.Data) + binary.Size(data.ValidData) + binary.Size(data.Nullable)
|
|
}
|
|
|
|
func (data *TimestamptzFieldData) GetMemorySize() int {
|
|
return binary.Size(data.Data) + binary.Size(data.ValidData) + binary.Size(data.Nullable)
|
|
}
|
|
|
|
func (data *BinaryVectorFieldData) GetMemorySize() int {
|
|
// Data + ValidData + Dim(4) + Nullable(1) + L2PMapping
|
|
return binary.Size(data.Data) + binary.Size(data.ValidData) + 4 + 1 + data.L2PMapping.GetMemorySize()
|
|
}
|
|
|
|
func (data *FloatVectorFieldData) GetMemorySize() int {
|
|
return binary.Size(data.Data) + binary.Size(data.ValidData) + 4 + 1 + data.L2PMapping.GetMemorySize()
|
|
}
|
|
|
|
func (data *Float16VectorFieldData) GetMemorySize() int {
|
|
return binary.Size(data.Data) + binary.Size(data.ValidData) + 4 + 1 + data.L2PMapping.GetMemorySize()
|
|
}
|
|
|
|
func (data *BFloat16VectorFieldData) GetMemorySize() int {
|
|
return binary.Size(data.Data) + binary.Size(data.ValidData) + 4 + 1 + data.L2PMapping.GetMemorySize()
|
|
}
|
|
|
|
func (data *SparseFloatVectorFieldData) GetMemorySize() int {
|
|
// TODO(SPARSE): should this be the memory size of serialzied size?
|
|
// SparseFloatArray + ValidData + Nullable(1) + L2PMapping
|
|
return proto.Size(&data.SparseFloatArray) + binary.Size(data.ValidData) + 1 + data.L2PMapping.GetMemorySize()
|
|
}
|
|
|
|
func (data *Int8VectorFieldData) GetMemorySize() int {
|
|
return binary.Size(data.Data) + binary.Size(data.ValidData) + 4 + 1 + data.L2PMapping.GetMemorySize()
|
|
}
|
|
|
|
func GetVectorSize(vector *schemapb.VectorField, vectorType schemapb.DataType) int {
|
|
size := 0
|
|
switch vectorType {
|
|
case schemapb.DataType_BinaryVector:
|
|
size += binary.Size(vector.GetBinaryVector()) + 4
|
|
case schemapb.DataType_FloatVector:
|
|
size += binary.Size(vector.GetFloatVector().Data) + 4
|
|
case schemapb.DataType_Float16Vector:
|
|
size += binary.Size(vector.GetFloat16Vector()) + 4
|
|
case schemapb.DataType_BFloat16Vector:
|
|
size += binary.Size(vector.GetBfloat16Vector()) + 4
|
|
case schemapb.DataType_SparseFloatVector:
|
|
panic("not implemented")
|
|
case schemapb.DataType_Int8Vector:
|
|
size += binary.Size(vector.GetInt8Vector()) + 4
|
|
default:
|
|
panic("unreachable")
|
|
}
|
|
return size
|
|
}
|
|
|
|
func (data *VectorArrayFieldData) GetMemorySize() int {
|
|
var size int
|
|
for _, val := range data.Data {
|
|
size += GetVectorSize(val, data.ElementType)
|
|
}
|
|
return size
|
|
}
|
|
|
|
// GetDataType implements FieldData.GetDataType
|
|
func (data *BoolFieldData) GetDataType() schemapb.DataType { return schemapb.DataType_Bool }
|
|
func (data *Int8FieldData) GetDataType() schemapb.DataType { return schemapb.DataType_Int8 }
|
|
func (data *Int16FieldData) GetDataType() schemapb.DataType { return schemapb.DataType_Int16 }
|
|
func (data *Int32FieldData) GetDataType() schemapb.DataType { return schemapb.DataType_Int32 }
|
|
func (data *Int64FieldData) GetDataType() schemapb.DataType { return schemapb.DataType_Int64 }
|
|
func (data *FloatFieldData) GetDataType() schemapb.DataType { return schemapb.DataType_Float }
|
|
func (data *DoubleFieldData) GetDataType() schemapb.DataType { return schemapb.DataType_Double }
|
|
func (data *TimestamptzFieldData) GetDataType() schemapb.DataType {
|
|
return schemapb.DataType_Timestamptz
|
|
}
|
|
func (data *StringFieldData) GetDataType() schemapb.DataType { return data.DataType }
|
|
func (data *ArrayFieldData) GetDataType() schemapb.DataType { return schemapb.DataType_Array }
|
|
func (data *JSONFieldData) GetDataType() schemapb.DataType { return schemapb.DataType_JSON }
|
|
func (data *GeometryFieldData) GetDataType() schemapb.DataType {
|
|
return schemapb.DataType_Geometry
|
|
}
|
|
|
|
func (data *BinaryVectorFieldData) GetDataType() schemapb.DataType {
|
|
return schemapb.DataType_BinaryVector
|
|
}
|
|
|
|
func (data *FloatVectorFieldData) GetDataType() schemapb.DataType {
|
|
return schemapb.DataType_FloatVector
|
|
}
|
|
|
|
func (data *Float16VectorFieldData) GetDataType() schemapb.DataType {
|
|
return schemapb.DataType_Float16Vector
|
|
}
|
|
|
|
func (data *BFloat16VectorFieldData) GetDataType() schemapb.DataType {
|
|
return schemapb.DataType_BFloat16Vector
|
|
}
|
|
|
|
func (data *SparseFloatVectorFieldData) GetDataType() schemapb.DataType {
|
|
return schemapb.DataType_SparseFloatVector
|
|
}
|
|
|
|
func (data *Int8VectorFieldData) GetDataType() schemapb.DataType {
|
|
return schemapb.DataType_Int8Vector
|
|
}
|
|
|
|
func (data *VectorArrayFieldData) GetDataType() schemapb.DataType {
|
|
return schemapb.DataType_ArrayOfVector
|
|
}
|
|
|
|
// why not binary.Size(data) directly? binary.Size(data) return -1
|
|
// binary.Size returns how many bytes Write would generate to encode the value v, which
|
|
// must be a fixed-size value or a slice of fixed-size values, or a pointer to such data.
|
|
// If v is neither of these, binary.Size returns -1.
|
|
func (data *StringFieldData) GetMemorySize() int {
|
|
var size int
|
|
for _, val := range data.Data {
|
|
size += len(val) + 16
|
|
}
|
|
return size + binary.Size(data.ValidData) + binary.Size(data.Nullable)
|
|
}
|
|
|
|
func (data *ArrayFieldData) GetMemorySize() int {
|
|
var size int
|
|
for _, val := range data.Data {
|
|
switch data.ElementType {
|
|
case schemapb.DataType_Bool:
|
|
size += binary.Size(val.GetBoolData().GetData())
|
|
case schemapb.DataType_Int8:
|
|
size += binary.Size(val.GetIntData().GetData()) / 4
|
|
case schemapb.DataType_Int16:
|
|
size += binary.Size(val.GetIntData().GetData()) / 2
|
|
case schemapb.DataType_Int32:
|
|
size += binary.Size(val.GetIntData().GetData())
|
|
case schemapb.DataType_Int64:
|
|
size += binary.Size(val.GetLongData().GetData())
|
|
case schemapb.DataType_Float:
|
|
size += binary.Size(val.GetFloatData().GetData())
|
|
case schemapb.DataType_Double:
|
|
size += binary.Size(val.GetDoubleData().GetData())
|
|
case schemapb.DataType_String, schemapb.DataType_VarChar:
|
|
size += (&StringFieldData{Data: val.GetStringData().GetData()}).GetMemorySize()
|
|
}
|
|
}
|
|
return size + binary.Size(data.ValidData) + binary.Size(data.Nullable)
|
|
}
|
|
|
|
func (data *JSONFieldData) GetMemorySize() int {
|
|
var size int
|
|
for _, val := range data.Data {
|
|
size += len(val) + 16
|
|
}
|
|
return size + binary.Size(data.ValidData) + binary.Size(data.Nullable)
|
|
}
|
|
|
|
func (data *GeometryFieldData) GetMemorySize() int {
|
|
var size int
|
|
// what's the meaning of 16?
|
|
for _, val := range data.Data {
|
|
size += len(val) + 16
|
|
}
|
|
return size + binary.Size(data.ValidData) + binary.Size(data.Nullable)
|
|
}
|
|
|
|
func (data *BoolFieldData) GetRowSize(i int) int { return 1 }
|
|
func (data *Int8FieldData) GetRowSize(i int) int { return 1 }
|
|
func (data *Int16FieldData) GetRowSize(i int) int { return 2 }
|
|
func (data *Int32FieldData) GetRowSize(i int) int { return 4 }
|
|
func (data *Int64FieldData) GetRowSize(i int) int { return 8 }
|
|
func (data *FloatFieldData) GetRowSize(i int) int { return 4 }
|
|
func (data *DoubleFieldData) GetRowSize(i int) int { return 8 }
|
|
func (data *TimestamptzFieldData) GetRowSize(i int) int { return 8 }
|
|
func (data *BinaryVectorFieldData) GetRowSize(i int) int {
|
|
if data.GetNullable() && !data.ValidData[i] {
|
|
return 0
|
|
}
|
|
return data.Dim / 8
|
|
}
|
|
|
|
func (data *FloatVectorFieldData) GetRowSize(i int) int {
|
|
if data.GetNullable() && !data.ValidData[i] {
|
|
return 0
|
|
}
|
|
return data.Dim * 4
|
|
}
|
|
|
|
func (data *Float16VectorFieldData) GetRowSize(i int) int {
|
|
if data.GetNullable() && !data.ValidData[i] {
|
|
return 0
|
|
}
|
|
return data.Dim * 2
|
|
}
|
|
|
|
func (data *BFloat16VectorFieldData) GetRowSize(i int) int {
|
|
if data.GetNullable() && !data.ValidData[i] {
|
|
return 0
|
|
}
|
|
return data.Dim * 2
|
|
}
|
|
|
|
func (data *Int8VectorFieldData) GetRowSize(i int) int {
|
|
if data.GetNullable() && !data.ValidData[i] {
|
|
return 0
|
|
}
|
|
return data.Dim
|
|
}
|
|
func (data *StringFieldData) GetRowSize(i int) int { return len(data.Data[i]) + 16 }
|
|
func (data *JSONFieldData) GetRowSize(i int) int { return len(data.Data[i]) + 16 }
|
|
func (data *GeometryFieldData) GetRowSize(i int) int { return len(data.Data[i]) + 16 }
|
|
func (data *ArrayFieldData) GetRowSize(i int) int {
|
|
switch data.ElementType {
|
|
case schemapb.DataType_Bool:
|
|
return binary.Size(data.Data[i].GetBoolData().GetData())
|
|
case schemapb.DataType_Int8:
|
|
return binary.Size(data.Data[i].GetIntData().GetData()) / 4
|
|
case schemapb.DataType_Int16:
|
|
return binary.Size(data.Data[i].GetIntData().GetData()) / 2
|
|
case schemapb.DataType_Int32:
|
|
return binary.Size(data.Data[i].GetIntData().GetData())
|
|
case schemapb.DataType_Int64:
|
|
return binary.Size(data.Data[i].GetLongData().GetData())
|
|
case schemapb.DataType_Float:
|
|
return binary.Size(data.Data[i].GetFloatData().GetData())
|
|
case schemapb.DataType_Double:
|
|
return binary.Size(data.Data[i].GetDoubleData().GetData())
|
|
case schemapb.DataType_String, schemapb.DataType_VarChar:
|
|
return (&StringFieldData{Data: data.Data[i].GetStringData().GetData()}).GetMemorySize()
|
|
}
|
|
return 0
|
|
}
|
|
|
|
func (data *SparseFloatVectorFieldData) GetRowSize(i int) int {
|
|
if data.GetNullable() && !data.ValidData[i] {
|
|
return 0
|
|
}
|
|
physicalIdx := data.L2PMapping.GetPhysicalOffset(i)
|
|
return len(data.Contents[physicalIdx])
|
|
}
|
|
|
|
func (data *VectorArrayFieldData) GetRowSize(i int) int {
|
|
return GetVectorSize(data.Data[i], data.ElementType)
|
|
}
|
|
|
|
func (data *BoolFieldData) GetNullable() bool {
|
|
return data.Nullable
|
|
}
|
|
|
|
func (data *Int8FieldData) GetNullable() bool {
|
|
return data.Nullable
|
|
}
|
|
|
|
func (data *Int16FieldData) GetNullable() bool {
|
|
return data.Nullable
|
|
}
|
|
|
|
func (data *Int32FieldData) GetNullable() bool {
|
|
return data.Nullable
|
|
}
|
|
|
|
func (data *Int64FieldData) GetNullable() bool {
|
|
return data.Nullable
|
|
}
|
|
|
|
func (data *FloatFieldData) GetNullable() bool {
|
|
return data.Nullable
|
|
}
|
|
|
|
func (data *DoubleFieldData) GetNullable() bool {
|
|
return data.Nullable
|
|
}
|
|
|
|
func (data *TimestamptzFieldData) GetNullable() bool {
|
|
return data.Nullable
|
|
}
|
|
|
|
func (data *BFloat16VectorFieldData) GetNullable() bool {
|
|
return data.Nullable
|
|
}
|
|
|
|
func (data *BinaryVectorFieldData) GetNullable() bool {
|
|
return data.Nullable
|
|
}
|
|
|
|
func (data *FloatVectorFieldData) GetNullable() bool {
|
|
return data.Nullable
|
|
}
|
|
|
|
func (data *SparseFloatVectorFieldData) GetNullable() bool {
|
|
return data.Nullable
|
|
}
|
|
|
|
func (data *Float16VectorFieldData) GetNullable() bool {
|
|
return data.Nullable
|
|
}
|
|
|
|
func (data *Int8VectorFieldData) GetNullable() bool {
|
|
return data.Nullable
|
|
}
|
|
|
|
func (data *StringFieldData) GetNullable() bool {
|
|
return data.Nullable
|
|
}
|
|
|
|
func (data *ArrayFieldData) GetNullable() bool {
|
|
return data.Nullable
|
|
}
|
|
|
|
func (data *JSONFieldData) GetNullable() bool {
|
|
return data.Nullable
|
|
}
|
|
|
|
func (data *VectorArrayFieldData) GetNullable() bool {
|
|
return false
|
|
}
|
|
|
|
func (data *GeometryFieldData) GetNullable() bool {
|
|
return data.Nullable
|
|
}
|
|
|
|
func (data *BoolFieldData) GetValidData() []bool { return data.ValidData }
|
|
func (data *Int8FieldData) GetValidData() []bool { return data.ValidData }
|
|
func (data *Int16FieldData) GetValidData() []bool { return data.ValidData }
|
|
func (data *Int32FieldData) GetValidData() []bool { return data.ValidData }
|
|
func (data *Int64FieldData) GetValidData() []bool { return data.ValidData }
|
|
func (data *FloatFieldData) GetValidData() []bool { return data.ValidData }
|
|
func (data *DoubleFieldData) GetValidData() []bool { return data.ValidData }
|
|
func (data *TimestamptzFieldData) GetValidData() []bool { return data.ValidData }
|
|
func (data *StringFieldData) GetValidData() []bool { return data.ValidData }
|
|
func (data *ArrayFieldData) GetValidData() []bool { return data.ValidData }
|
|
func (data *JSONFieldData) GetValidData() []bool { return data.ValidData }
|
|
func (data *GeometryFieldData) GetValidData() []bool { return data.ValidData }
|
|
func (data *BinaryVectorFieldData) GetValidData() []bool { return data.ValidData }
|
|
func (data *FloatVectorFieldData) GetValidData() []bool { return data.ValidData }
|
|
func (data *Float16VectorFieldData) GetValidData() []bool { return data.ValidData }
|
|
func (data *BFloat16VectorFieldData) GetValidData() []bool { return data.ValidData }
|
|
func (data *SparseFloatVectorFieldData) GetValidData() []bool { return data.ValidData }
|
|
func (data *Int8VectorFieldData) GetValidData() []bool { return data.ValidData }
|
|
func (data *VectorArrayFieldData) GetValidData() []bool { return nil }
|