From b39dfc25dceb152cb246c29a20a62b53e6956690 Mon Sep 17 00:00:00 2001 From: congqixia Date: Wed, 12 Jun 2024 20:41:57 +0800 Subject: [PATCH] enhance: Use fastjson lib for unmarshal delete log (#33787) ``` goos: linux goarch: amd64 GOMAXPROC=1 cpu: Intel(R) Core(TM) i7-8700 CPU @ 3.20GHz BenchmarkJsonSerdeStd 343872 3568 ns/op 1335 B/op 25 allocs/op BenchmarkJsonSerdeFastjson 5124177 234.9 ns/op 16 B/op 1 allocs/op ``` --------- Signed-off-by: Congqi Xia --- go.mod | 1 + go.sum | 2 ++ internal/storage/data_codec.go | 18 ++++++++++++++++-- 3 files changed, 19 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index 3412f20c40..653c0798af 100644 --- a/go.mod +++ b/go.mod @@ -208,6 +208,7 @@ require ( github.com/twmb/murmur3 v1.1.3 // indirect github.com/uber/jaeger-client-go v2.30.0+incompatible // indirect github.com/ugorji/go/codec v1.2.11 // indirect + github.com/valyala/fastjson v1.6.4 // indirect github.com/x448/float16 v0.8.4 // indirect github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 // indirect github.com/yusufpapurcu/wmi v1.2.2 // indirect diff --git a/go.sum b/go.sum index f0f45360fe..70b031afdd 100644 --- a/go.sum +++ b/go.sum @@ -893,6 +893,8 @@ github.com/ugorji/go/codec v1.2.11/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZ github.com/urfave/negroni v1.0.0/go.mod h1:Meg73S6kFm/4PpbYdq35yYWoCZ9mS/YSx+lKnmiohz4= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/valyala/fasthttp v1.6.0/go.mod h1:FstJa9V+Pj9vQ7OJie2qMHdwemEDaDiSdBnvPM1Su9w= +github.com/valyala/fastjson v1.6.4 h1:uAUNq9Z6ymTgGhcm0UynUAB6tlbakBrz6CQFax3BXVQ= +github.com/valyala/fastjson v1.6.4/go.mod h1:CLCAqky6SMuOcxStkYQvblddUtoRxhYMGLrsQns1aXY= github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ= github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio= diff --git a/internal/storage/data_codec.go b/internal/storage/data_codec.go index 61babb9227..fac23ee344 100644 --- a/internal/storage/data_codec.go +++ b/internal/storage/data_codec.go @@ -26,6 +26,7 @@ import ( "strings" "github.com/samber/lo" + "github.com/valyala/fastjson" "github.com/milvus-io/milvus-proto/go-api/v2/schemapb" "github.com/milvus-io/milvus/internal/proto/etcdpb" @@ -1059,14 +1060,18 @@ func (deleteCodec *DeleteCodec) Deserialize(blobs []*Blob) (partitionID UniqueID } defer rr.Release() + var p fastjson.Parser + deleteLog := &DeleteLog{} + for rr.Next() { rec := rr.Record() defer rec.Release() column := rec.Column(0) for i := 0; i < column.Len(); i++ { - deleteLog := &DeleteLog{} strVal := column.ValueStr(i) - if err = json.Unmarshal([]byte(strVal), deleteLog); err != nil { + + v, err := p.Parse(strVal) + if err != nil { // compatible with versions that only support int64 type primary keys // compatible with fmt.Sprintf("%d,%d", pk, ts) // compatible error info (unmarshal err invalid character ',' after top-level value) @@ -1086,6 +1091,15 @@ func (deleteCodec *DeleteCodec) Deserialize(blobs []*Blob) (partitionID UniqueID if err != nil { return err } + } else { + deleteLog.Ts = v.GetUint64("ts") + deleteLog.PkType = v.GetInt64("pkType") + switch deleteLog.PkType { + case int64(schemapb.DataType_Int64): + deleteLog.Pk = &Int64PrimaryKey{Value: v.GetInt64("pk")} + case int64(schemapb.DataType_VarChar): + deleteLog.Pk = &VarCharPrimaryKey{Value: string(v.GetStringBytes("pk"))} + } } result.Append(deleteLog.Pk, deleteLog.Ts)