mirror of
https://gitee.com/milvus-io/milvus.git
synced 2026-01-06 10:52:56 +08:00
1ce609834a
Add the following methods for convenient fp32 vector <-> fp16/bf16
vector conversion
fp32 <-> fp16/bf16 vector conversion:
- `func (fv FloatVector) ToFloat16Vector() Float16Vector`
- `func (fv FloatVector) ToBFloat16Vector() BFloat16Vector`
- `func (fv Float16Vector) ToFloat32Vector() FloatVector`
- `func (fv BFloat16Vector) ToFloat32Vector() FloatVector`
`columnBasedDataOption`:
- `func (opt *columnBasedDataOption) WithFloat16VectorColumn(colName
string, dim int, data [][]float32) *columnBasedDataOption`
- `func (opt *columnBasedDataOption) WithBFloat16VectorColumn(colName
string, dim int, data [][]float32) *columnBasedDataOption`
`ColumnFloat16Vector`/`ColumnBFloat16Vector`:
- `func NewColumnFloat16VectorFromFp32Vector(fieldName string, dim int,
data [][]float32) *ColumnFloat16Vector`
- `func NewColumnBFloat16VectorFromFp32Vector(fieldName string, dim int,
data [][]float32) *ColumnBFloat16Vector`
- support []float32 or `entity.FloatVector` in
- `func (c *ColumnFloat16Vector) AppendValue(i interface{}) error`
- `func (c *ColumnFloat16Vector) AppendValue(i interface{}) error`
issue: #37448
Signed-off-by: Yinzuo Jiang <yinzuo.jiang@zilliz.com>
Signed-off-by: Yinzuo Jiang <jiangyinzuo@foxmail.com>
93 lines
2.6 KiB
Go
93 lines
2.6 KiB
Go
// Licensed to the LF AI & Data foundation under one
|
|
// or more contributor license agreements. See the NOTICE file
|
|
// distributed with this work for additional information
|
|
// regarding copyright ownership. The ASF licenses this file
|
|
// to you under the Apache License, Version 2.0 (the
|
|
// "License"); you may not use this file except in compliance
|
|
// with the License. You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
package entity
|
|
|
|
import (
|
|
"math/rand"
|
|
"testing"
|
|
|
|
"github.com/stretchr/testify/assert"
|
|
)
|
|
|
|
func TestVectors(t *testing.T) {
|
|
dim := rand.Intn(127) + 1
|
|
|
|
t.Run("test float vector", func(t *testing.T) {
|
|
raw := make([]float32, dim)
|
|
for i := 0; i < dim; i++ {
|
|
raw[i] = rand.Float32()
|
|
}
|
|
|
|
fv := FloatVector(raw)
|
|
assert.Equal(t, dim, fv.Dim())
|
|
assert.Equal(t, dim*4, len(fv.Serialize()))
|
|
|
|
var fvConverted FloatVector
|
|
|
|
fp16v := fv.ToFloat16Vector()
|
|
assert.Equal(t, dim, fp16v.Dim())
|
|
assert.Equal(t, dim*2, len(fp16v.Serialize()))
|
|
fvConverted = fp16v.ToFloat32Vector()
|
|
assert.Equal(t, dim, fvConverted.Dim())
|
|
assert.Equal(t, dim*4, len(fvConverted.Serialize()))
|
|
|
|
bf16v := fv.ToBFloat16Vector()
|
|
assert.Equal(t, dim, bf16v.Dim())
|
|
assert.Equal(t, dim*2, len(bf16v.Serialize()))
|
|
fvConverted = bf16v.ToFloat32Vector()
|
|
assert.Equal(t, dim, fvConverted.Dim())
|
|
assert.Equal(t, dim*4, len(fvConverted.Serialize()))
|
|
})
|
|
|
|
t.Run("test fp32 <-> fp16/bf16 vector conversion", func(t *testing.T) {
|
|
raw := make([]float32, dim)
|
|
for i := 0; i < dim; i++ {
|
|
raw[i] = float32(i) * 0.1
|
|
}
|
|
|
|
fv := FloatVector(raw)
|
|
fp16v := fv.ToFloat16Vector()
|
|
bf16v := fv.ToBFloat16Vector()
|
|
|
|
assert.Equal(t, dim, fp16v.Dim())
|
|
assert.Equal(t, dim*2, len(fp16v.Serialize()))
|
|
assert.Equal(t, dim, bf16v.Dim())
|
|
assert.Equal(t, dim*2, len(bf16v.Serialize()))
|
|
|
|
fp32vFromfp16v := fp16v.ToFloat32Vector()
|
|
for i := 0; i < dim; i++ {
|
|
assert.InDelta(t, fv[i], fp32vFromfp16v[i], 0.04)
|
|
}
|
|
|
|
fp32vFrombf16v := bf16v.ToFloat32Vector()
|
|
for i := 0; i < dim; i++ {
|
|
assert.InDelta(t, fp32vFromfp16v[i], fp32vFrombf16v[i], 0.04)
|
|
}
|
|
})
|
|
|
|
t.Run("test binary vector", func(t *testing.T) {
|
|
raw := make([]byte, dim)
|
|
_, err := rand.Read(raw)
|
|
assert.Nil(t, err)
|
|
|
|
bv := BinaryVector(raw)
|
|
|
|
assert.Equal(t, dim*8, bv.Dim())
|
|
assert.ElementsMatch(t, raw, bv.Serialize())
|
|
})
|
|
}
|