enhance: [2.4] Preallocate delete data slice to avoid growslice (#37044)

Rewritten based on master pr
pr: #37043

Related to #36887

Signed-off-by: Congqi Xia <congqi.xia@zilliz.com>
This commit is contained in:
congqixia 2024-10-22 14:15:28 +08:00 committed by GitHub
parent 55caf41169
commit 7acf1d53c1
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
3 changed files with 14 additions and 5 deletions

View File

@ -17,11 +17,11 @@
package datacoord
import (
"github.com/milvus-io/milvus/pkg/util/funcutil"
"github.com/milvus-io/milvus/pkg/util/paramtable"
"google.golang.org/protobuf/proto"
"github.com/milvus-io/milvus/internal/proto/datapb"
"github.com/milvus-io/milvus/pkg/util/funcutil"
"github.com/milvus-io/milvus/pkg/util/paramtable"
"github.com/milvus-io/milvus/pkg/util/timerecord"
)

View File

@ -1395,8 +1395,9 @@ func (loader *segmentLoader) LoadDeltaLogs(ctx context.Context, segment Segment,
return nil, err
}
blob := &storage.Blob{
Key: bLog.GetLogPath(),
Value: value,
Key: bLog.GetLogPath(),
Value: value,
RowNum: bLog.EntriesNum,
}
return blob, nil
})

View File

@ -23,6 +23,8 @@ import (
"math"
"sort"
"github.com/samber/lo"
"github.com/milvus-io/milvus-proto/go-api/v2/schemapb"
"github.com/milvus-io/milvus/internal/proto/etcdpb"
"github.com/milvus-io/milvus/pkg/common"
@ -868,7 +870,13 @@ func (deleteCodec *DeleteCodec) Deserialize(blobs []*Blob) (partitionID UniqueID
}
var pid, sid UniqueID
result := &DeleteData{}
rowNums := lo.SumBy(blobs, func(blob *Blob) int64 {
return blob.RowNum
})
result := &DeleteData{
Pks: make([]PrimaryKey, 0, rowNums),
Tss: make([]uint64, 0, rowNums),
}
deserializeBlob := func(blob *Blob) error {
binlogReader, err := NewBinlogReader(blob.Value)