admin/milvus
1
0
Fork 0
mirror of https://gitee.com/milvus-io/milvus.git synced 2025-12-08 01:58:34 +08:00
Code Issues Packages Projects Releases Wiki Activity

enhance: reduce memory when read field (#33195)

Browse Source 
Signed-off-by: lixinguo <xinguo.li@zilliz.com>
Co-authored-by: lixinguo <xinguo.li@zilliz.com>
This commit is contained in:
smellthemoon 2024-05-20 22:25:38 +08:00 committed by GitHub
parent 0d99db23b8
commit 89ad3eb0ca
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 8 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
internal/util/importutilv2/parquet/field_reader.go
View File

@ -130,15 +130,13 @@ func ReadBoolData(pcr *FieldReader, count int64) (any, error) {
data := make([]bool, 0, count) data := make([]bool, 0, count)
for _, chunk := range chunked.Chunks() { for _, chunk := range chunked.Chunks() {
dataNums := chunk.Data().Len() dataNums := chunk.Data().Len()
chunkData := make([]bool, dataNums)
boolReader, ok := chunk.(*array.Boolean) boolReader, ok := chunk.(*array.Boolean)
if !ok { if !ok {
return nil, WrapTypeErr("bool", chunk.DataType().Name(), pcr.field) return nil, WrapTypeErr("bool", chunk.DataType().Name(), pcr.field)
} }
for i := 0; i < dataNums; i++ { for i := 0; i < dataNums; i++ {
chunkData[i] = boolReader.Value(i) data = append(data, boolReader.Value(i))
} }
data = append(data, chunkData...)
} }
if len(data) == 0 { if len(data) == 0 {
return nil, nil return nil, nil
@ -154,42 +152,40 @@ func ReadIntegerOrFloatData[T constraints.Integer | constraints.Float](pcr *Fiel
data := make([]T, 0, count) data := make([]T, 0, count)
for _, chunk := range chunked.Chunks() { for _, chunk := range chunked.Chunks() {
dataNums := chunk.Data().Len() dataNums := chunk.Data().Len()
chunkData := make([]T, dataNums)
switch chunk.DataType().ID() { switch chunk.DataType().ID() {
case arrow.INT8: case arrow.INT8:
int8Reader := chunk.(*array.Int8) int8Reader := chunk.(*array.Int8)
for i := 0; i < dataNums; i++ { for i := 0; i < dataNums; i++ {
chunkData[i] = T(int8Reader.Value(i)) data = append(data, T(int8Reader.Value(i)))
} }
case arrow.INT16: case arrow.INT16:
int16Reader := chunk.(*array.Int16) int16Reader := chunk.(*array.Int16)
for i := 0; i < dataNums; i++ { for i := 0; i < dataNums; i++ {
chunkData[i] = T(int16Reader.Value(i)) data = append(data, T(int16Reader.Value(i)))
} }
case arrow.INT32: case arrow.INT32:
int32Reader := chunk.(*array.Int32) int32Reader := chunk.(*array.Int32)
for i := 0; i < dataNums; i++ { for i := 0; i < dataNums; i++ {
chunkData[i] = T(int32Reader.Value(i)) data = append(data, T(int32Reader.Value(i)))
} }
case arrow.INT64: case arrow.INT64:
int64Reader := chunk.(*array.Int64) int64Reader := chunk.(*array.Int64)
for i := 0; i < dataNums; i++ { for i := 0; i < dataNums; i++ {
chunkData[i] = T(int64Reader.Value(i)) data = append(data, T(int64Reader.Value(i)))
} }
case arrow.FLOAT32: case arrow.FLOAT32:
float32Reader := chunk.(*array.Float32) float32Reader := chunk.(*array.Float32)
for i := 0; i < dataNums; i++ { for i := 0; i < dataNums; i++ {
chunkData[i] = T(float32Reader.Value(i)) data = append(data, T(float32Reader.Value(i)))
} }
case arrow.FLOAT64: case arrow.FLOAT64:
float64Reader := chunk.(*array.Float64) float64Reader := chunk.(*array.Float64)
for i := 0; i < dataNums; i++ { for i := 0; i < dataNums; i++ {
chunkData[i] = T(float64Reader.Value(i)) data = append(data, T(float64Reader.Value(i)))
} }
default: default:
return nil, WrapTypeErr("integer|float", chunk.DataType().Name(), pcr.field) return nil, WrapTypeErr("integer|float", chunk.DataType().Name(), pcr.field)
} }
data = append(data, chunkData...)
} }
if len(data) == 0 { if len(data) == 0 {
return nil, nil return nil, nil
@ -205,15 +201,13 @@ func ReadStringData(pcr *FieldReader, count int64) (any, error) {
data := make([]string, 0, count) data := make([]string, 0, count)
for _, chunk := range chunked.Chunks() { for _, chunk := range chunked.Chunks() {
dataNums := chunk.Data().Len() dataNums := chunk.Data().Len()
chunkData := make([]string, dataNums)
stringReader, ok := chunk.(*array.String) stringReader, ok := chunk.(*array.String)
if !ok { if !ok {
return nil, WrapTypeErr("string", chunk.DataType().Name(), pcr.field) return nil, WrapTypeErr("string", chunk.DataType().Name(), pcr.field)
} }
for i := 0; i < dataNums; i++ { for i := 0; i < dataNums; i++ {
chunkData[i] = stringReader.Value(i) data = append(data, stringReader.Value(i))
} }
data = append(data, chunkData...)
} }
if len(data) == 0 { if len(data) == 0 {
return nil, nil return nil, nil