From 801eeffbccc6bc4a5ad2a59cce97ad196e981400 Mon Sep 17 00:00:00 2001 From: Xiaofan <83447078+xiaofan-luan@users.noreply.github.com> Date: Wed, 30 Mar 2022 15:21:28 +0800 Subject: [PATCH] Replace cgo parquet reader to go parquet reader (#16199) Signed-off-by: xiaofan-luan --- go.mod | 18 +- go.sum | 275 ++++++-- internal/storage/binlog_reader.go | 58 +- internal/storage/binlog_writer_test.go | 2 - internal/storage/data_codec.go | 46 +- internal/storage/event_test.go | 169 ++--- internal/storage/event_writer.go | 29 - internal/storage/payload.go | 304 +-------- internal/storage/payload_cgo_test.go | 898 +++++++++++++++++++++++++ internal/storage/payload_reader.go | 441 ++++++++++++ internal/storage/payload_reader_cgo.go | 322 +++++++++ internal/storage/payload_test.go | 55 +- internal/storage/print_binlog.go | 22 +- 13 files changed, 2044 insertions(+), 595 deletions(-) create mode 100644 internal/storage/payload_cgo_test.go create mode 100644 internal/storage/payload_reader.go create mode 100644 internal/storage/payload_reader_cgo.go diff --git a/go.mod b/go.mod index 01f286b6a8..5cf23a9730 100644 --- a/go.mod +++ b/go.mod @@ -3,12 +3,13 @@ module github.com/milvus-io/milvus go 1.16 require ( - github.com/BurntSushi/toml v1.0.0 // indirect + github.com/BurntSushi/toml v1.0.0 github.com/HdrHistogram/hdrhistogram-go v1.0.1 // indirect github.com/StackExchange/wmi v1.2.1 // indirect github.com/antonmedv/expr v1.8.9 + github.com/apache/arrow/go/v8 v8.0.0-20220322092137-778b1772fd20 github.com/apache/pulsar-client-go v0.6.1-0.20210728062540-29414db801a7 - github.com/apache/thrift/lib/go/thrift v0.0.0-20210120171102-e27e82c46ba4 + github.com/apache/thrift v0.15.0 github.com/bits-and-blooms/bloom/v3 v3.0.1 github.com/containerd/cgroups v1.0.2 github.com/facebookgo/ensure v0.0.0-20200202191622-63f1cf65ac4c // indirect @@ -16,14 +17,13 @@ require ( github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4 // indirect github.com/gin-gonic/gin v1.7.7 github.com/go-basic/ipv4 v1.0.0 - github.com/gofrs/flock v0.8.1 // indirect + github.com/gofrs/flock v0.8.1 github.com/golang/mock v1.5.0 github.com/golang/protobuf v1.5.2 github.com/google/btree v1.0.1 github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 - github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 github.com/jarcoal/httpmock v1.0.8 - github.com/klauspost/compress v1.10.11 // indirect + github.com/klauspost/compress v1.14.2 github.com/lingdor/stackerror v0.0.0-20191119040541-976d8885ed76 github.com/minio/minio-go/v7 v7.0.10 github.com/mitchellh/mapstructure v1.4.1 @@ -47,10 +47,9 @@ require ( go.uber.org/atomic v1.7.0 go.uber.org/automaxprocs v1.4.0 go.uber.org/zap v1.17.0 - golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6 + golang.org/x/exp v0.0.0-20211216164055-b2b84827b756 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c - golang.org/x/tools v0.1.7 // indirect - google.golang.org/grpc v1.38.0 + google.golang.org/grpc v1.44.0 gopkg.in/natefinch/lumberjack.v2 v2.0.0 stathat.com/c/consistent v1.0.0 ) @@ -60,5 +59,4 @@ replace ( github.com/dgrijalva/jwt-go => github.com/golang-jwt/jwt v3.2.2+incompatible // Fix security alert for jwt-go 3.2.0 github.com/keybase/go-keychain => github.com/99designs/go-keychain v0.0.0-20191008050251-8e49817e8af4 google.golang.org/grpc => google.golang.org/grpc v1.38.0 -) - +) \ No newline at end of file diff --git a/go.sum b/go.sum index 94ec46ed2d..daa3c4f271 100644 --- a/go.sum +++ b/go.sum @@ -37,15 +37,14 @@ cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohl cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +dmitri.shuralyov.com/gpu/mtl v0.0.0-20201218220906-28db891af037/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +gioui.org v0.0.0-20210308172011-57750fc8a0a6/go.mod h1:RSH6KIUZ0p2xy5zHDxgAM4zumjgTw83q2ge/PI+yyw8= github.com/99designs/go-keychain v0.0.0-20191008050251-8e49817e8af4 h1:/vQbFIOMbk2FiG/kXiLl8BRyzTWDw7gX/Hz7Dd5eDMs= github.com/99designs/go-keychain v0.0.0-20191008050251-8e49817e8af4/go.mod h1:hN7oaIRCjzsZ2dE+yG5k+rsdt3qcwykqK6HVGcKwsw4= github.com/99designs/keyring v1.1.5 h1:wLv7QyzYpFIyMSwOADq1CLTF9KbjbBfcnfmOGJ64aO4= github.com/99designs/keyring v1.1.5/go.mod h1:7hsVvt2qXgtadGevGJ4ujg+u8m6SpJ5TpHqTozIPqf0= github.com/AthenZ/athenz v1.10.15 h1:8Bc2W313k/ev/SGokuthNbzpwfg9W3frg3PKq1r943I= github.com/AthenZ/athenz v1.10.15/go.mod h1:7KMpEuJ9E4+vMCMI3UQJxwWs0RZtQq7YXZ1IteUjdsc= -github.com/AthenZ/athenz v1.10.39 h1:mtwHTF/v62ewY2Z5KWhuZgVXftBej1/Tn80zx4DcawY= -github.com/AthenZ/athenz v1.10.39/go.mod h1:3Tg8HLsiQZp81BJY58JBeU2BR6B/H4/0MQGfCwhHNEA= -github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/toml v1.0.0 h1:dtDWrepsVPfW9H/4y7dDgFc2MBUSeJhlaDtK13CxFlU= github.com/BurntSushi/toml v1.0.0/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= @@ -53,38 +52,48 @@ github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym github.com/DATA-DOG/go-sqlmock v1.3.3/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= github.com/DataDog/zstd v1.4.6-0.20210211175136-c6db21d202f4 h1:++HGU87uq9UsSTlFeiOV9uZR3NpYkndUXeYyLv2DTc8= github.com/DataDog/zstd v1.4.6-0.20210211175136-c6db21d202f4/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= -github.com/DataDog/zstd v1.5.0 h1:+K/VEwIAaPcHiMtQvpLD4lqW7f0Gk3xdYZmI1hD+CXo= -github.com/DataDog/zstd v1.5.0/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= github.com/HdrHistogram/hdrhistogram-go v1.0.1 h1:GX8GAYDuhlFQnI2fRDHQhTlkHMz8bEn0jTI6LJU0mpw= github.com/HdrHistogram/hdrhistogram-go v1.0.1/go.mod h1:BWJ+nMSHY3L41Zj7CA3uXnloDp7xxV0YvstAE7nKTaM= +github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c h1:RGWPOewvKIROun94nF7v2cua9qP+thov/7M50KEoeSU= +github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c/go.mod h1:X0CRv0ky0k6m906ixxpzmDRLvX58TFUKS2eePweuyxk= +github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= +github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= github.com/StackExchange/wmi v1.2.1 h1:VIkavFPXSjcnS+O8yTq7NI32k0R5Aj+v39y29VYDOSA= github.com/StackExchange/wmi v1.2.1/go.mod h1:rcmrprowKIVzvc+NUiLncP2uuArMWLCbu9SBzvHz7e8= +github.com/VividCortex/gohistogram v1.0.0/go.mod h1:Pf5mBqqDxYaXu3hDrrU+w6nw50o/4+TcAqDqk/vUH7g= +github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5/go.mod h1:SkGFH1ia65gfNATL8TAiHDNxPzPdmEL5uirI2Uyuz6c= +github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= +github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY= +github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/antonmedv/expr v1.8.9 h1:O9stiHmHHww9b4ozhPx7T6BK7fXfOCHJ8ybxf0833zw= github.com/antonmedv/expr v1.8.9/go.mod h1:5qsM3oLGDND7sDmQGDXHkYfkjYMUX14qsgqmHhwGEk8= -github.com/apache/pulsar-client-go v0.6.1-0.20210728062540-29414db801a7 h1:mTY6GM1gkiAneYm//bRDYu2/jVqi/BnB5PF6O6Wp9QU= -github.com/apache/pulsar-client-go v0.6.1-0.20210728062540-29414db801a7/go.mod h1:A1P5VjjljsFKAD13w7/jmU3Dly2gcRvcobiULqQXhz4= -github.com/apache/pulsar-client-go v0.7.0 h1:sZBkjJPHC7akM8n8DuzkLdwioKPSzyub3efCJ1Ltw9Y= -github.com/apache/pulsar-client-go v0.7.0/go.mod h1:EauTUv9sTmP9QRznRgK9hxnzCsIVfS8fyhTfGcuJBrE= -github.com/apache/pulsar-client-go v0.7.1-0.20220114073100-244facfe21a8 h1:GiNiP7lA9zaK84PpmL/JeX5q2AtNmer+ySDL6voWSqI= -github.com/apache/pulsar-client-go v0.7.1-0.20220114073100-244facfe21a8/go.mod h1:lhsBJTq1ZUyZJ3Le5AzZJRxW4EZkfopHid19/FlUaCQ= +github.com/apache/arrow/go/v8 v8.0.0-20220322092137-778b1772fd20 h1:YcSFhAin12rxRsvvfzD6gepH7jZwtFVdikXRyhzUC2w= +github.com/apache/arrow/go/v8 v8.0.0-20220322092137-778b1772fd20/go.mod h1:UUe+gJaMnuFD6icfGSJxUjG/tX/POUbPS/wE+EFyncM= github.com/apache/pulsar-client-go/oauth2 v0.0.0-20201120111947-b8bd55bc02bd h1:P5kM7jcXJ7TaftX0/EMKiSJgvQc/ct+Fw0KMvcH3WuY= github.com/apache/pulsar-client-go/oauth2 v0.0.0-20201120111947-b8bd55bc02bd/go.mod h1:0UtvvETGDdvXNDCHa8ZQpxl+w3HbdFtfYZvDHLgWGTY= -github.com/apache/thrift/lib/go/thrift v0.0.0-20210120171102-e27e82c46ba4 h1:orNYqmQGnSjgOauLWjHEp9/qIDT98xv/0Aa4Zet3/Y8= -github.com/apache/thrift/lib/go/thrift v0.0.0-20210120171102-e27e82c46ba4/go.mod h1:V/LzksIyqd3KZuQ2SunvReTG/UkArhII1dAWY5U1sCE= +github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= +github.com/apache/thrift v0.13.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= +github.com/apache/thrift v0.15.0 h1:aGvdaR0v1t9XLgjtBYwxcBvBOTMqClzwE26CHOgjW1Y= +github.com/apache/thrift v0.15.0/go.mod h1:PHK3hniurgQaNMZYaCLEqXKsYK8upmhPbmdP2FXSqgU= github.com/ardielle/ardielle-go v1.5.2 h1:TilHTpHIQJ27R1Tl/iITBzMwiUGSlVfiVhwDNGM3Zj4= github.com/ardielle/ardielle-go v1.5.2/go.mod h1:I4hy1n795cUhaVt/ojz83SNVCYIGsAFAONtv2Dr7HUI= github.com/ardielle/ardielle-tools v1.5.4/go.mod h1:oZN+JRMnqGiIhrzkRN9l26Cej9dEx4jeNG6A+AdkShk= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= +github.com/aryann/difflib v0.0.0-20170710044230-e206f873d14a/go.mod h1:DAHtR1m6lCRdSC2Tm3DSWRPvIPr6xNKyeHdqDQSQT+A= +github.com/aws/aws-lambda-go v1.13.3/go.mod h1:4UKl9IzQMoD+QF79YdCuzCwp8VbmG4VAQwij/eHl5CU= +github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/aws/aws-sdk-go v1.32.6/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= +github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g= github.com/beefsack/go-rate v0.0.0-20180408011153-efa7637bb9b6/go.mod h1:6YNgTHLutezwnBvyneBbwvB8C82y3dcoOj5EQJIdGXA= github.com/benbjohnson/clock v1.0.3 h1:vkLuvpK4fmtSCuo60+yC63p7y0BmQ8gm5ZXGuBCJyXg= github.com/benbjohnson/clock v1.0.3/go.mod h1:bGMdMPoPVvcYyt1gHDf4J2KE153Yf9BuiUKYMaxlTDM= @@ -100,6 +109,9 @@ github.com/bits-and-blooms/bloom/v3 v3.0.1/go.mod h1:MC8muvBzzPOFsrcdND/A7kU7kMh github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= github.com/bketelsen/crypt v0.0.4/go.mod h1:aI6NrJ0pMGgvZKL1iVgXLnfIFJtfV+bKCoqOes/6LfM= github.com/bmizerany/perks v0.0.0-20141205001514-d9a9656a3a4b/go.mod h1:ac9efd0D1fsDb3EJvhqgXRbFx7bs2wqZ10HQPeU8U/Q= +github.com/boombuler/barcode v1.0.0/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8= +github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ= +github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/certifi/gocertifi v0.0.0-20191021191039-0944d244cd40/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA= github.com/certifi/gocertifi v0.0.0-20200922220541-2c3bb06c6054 h1:uH66TXeswKn5PW5zdZ39xEwfS9an067BirqA+P4QaLI= @@ -112,26 +124,33 @@ github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWR github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/cilium/ebpf v0.4.0/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJXRs= +github.com/clbanning/x2j v0.0.0-20191024224557-825249438eec/go.mod h1:jMjuTZXRI4dUb/I5gc9Hdhagfvm9+RyrPryS/auMzxE= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= github.com/cockroachdb/datadriven v0.0.0-20200714090401-bf6692d28da5 h1:xD/lrqdvwsc+O2bjSSi3YqY73Ke3LAiSCx49aCesA0E= github.com/cockroachdb/datadriven v0.0.0-20200714090401-bf6692d28da5/go.mod h1:h6jFvWxBdQXxjopDMZyH2UVceIRfR84bdzbkoKrsWNo= github.com/cockroachdb/errors v1.2.4 h1:Lap807SXTH5tri2TivECb/4abUkMZC9zRoLarvcKDqs= github.com/cockroachdb/errors v1.2.4/go.mod h1:rQD95gz6FARkaKkQXUksEje/d9a6wBJoCr5oaCLELYA= github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f h1:o/kfcElHqOiXqcou5a3rIlMc7oJbMQkeLk0VQJ7zgqY= github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI= +github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= github.com/containerd/cgroups v1.0.2 h1:mZBclaSgNDfPWtfhj2xJY28LZ9nYIgzB0pwSURPl6JM= github.com/containerd/cgroups v1.0.2/go.mod h1:qpbpJ1jmlqsR9f2IyaLPsdkCdnt0rbDVqIDlhuu5tRY= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e h1:Wf6HqHfScWJN9/ZjdUKyjop4mf3Qdd+1TvvltAvM3m8= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd/v22 v22.3.2 h1:D9/bQk5vlXQFZ6Kwuu6zaiXJ9oTPe68++AzAJc1DzSI= github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= +github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/danieljoos/wincred v1.0.2 h1:zf4bhty2iLuwgjgpraD2E9UbvO+fe54XXGJbOwe23fU= @@ -145,10 +164,16 @@ github.com/dimfeld/httptreemux v5.0.1+incompatible h1:Qj3gVcDNoOthBAqftuD596rm4w github.com/dimfeld/httptreemux v5.0.1+incompatible/go.mod h1:rbUlSV+CCpv/SuqUTP/8Bk2O3LyUV436/yaRGkhP6Z0= github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw= github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= +github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dvsekhvalnov/jose2go v0.0.0-20180829124132-7f401d37b68a h1:mq+R6XEM6lJX5VlLyZIrUSP8tSuJp82xTK89hvBwJbU= github.com/dvsekhvalnov/jose2go v0.0.0-20180829124132-7f401d37b68a/go.mod h1:7BvyPhdbLxMXIYTFPLsyJRFMsKmOZnQmzh6Gb+uquuM= +github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= +github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= +github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= +github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/facebookgo/ensure v0.0.0-20200202191622-63f1cf65ac4c h1:8ISkoahWXwZR41ois5lSJBSVw4D0OV19Ht/JSTzvSv0= @@ -158,8 +183,12 @@ github.com/facebookgo/stack v0.0.0-20160209184415-751773369052/go.mod h1:UbMTZqL github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4 h1:7HZCaLC5+BZpmbhCOZJ293Lz68O7PYrF2EzeiFMwCLk= github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4/go.mod h1:5tD+neXqOorC30/tWg0LCSkrqj/AR6gu8yY8/fpw1q0= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= +github.com/fogleman/gg v1.3.0/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= github.com/form3tech-oss/jwt-go v3.2.3+incompatible h1:7ZaBxOI7TMoYBfyA3cQHErNNyAWIKUMIwqxEtgHOs5c= github.com/form3tech-oss/jwt-go v3.2.3+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= +github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVBjqR7JHJk0brhHOZYGmfBYOrK0ZhYMEtBr4= +github.com/franela/goreq v0.0.0-20171204163338-bcd34c9993f8/go.mod h1:ZhphrRTfi2rbfLwlschooIH4+wKKDR4Pdxhh+TRoA20= github.com/frankban/quicktest v1.11.3 h1:8sXhOn0uLys67V8EsXLc6eszDs8VXWxL3iRvebPhedY= github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= @@ -176,17 +205,25 @@ github.com/gin-gonic/gin v1.7.7 h1:3DoBmSbJbZAWqXJC3SLjAPfutPJJRN1U5pALB7EeTTs= github.com/gin-gonic/gin v1.7.7/go.mod h1:axIBovoeJpVj8S3BwE0uPMTeReE4+AfFtqpqaZ1qq1U= github.com/go-basic/ipv4 v1.0.0 h1:gjyFAa1USC1hhXTkPOwBWDPfMcUaIM+tvo1XzV9EZxs= github.com/go-basic/ipv4 v1.0.0/go.mod h1:etLBnaxbidQfuqE6wgZQfs38nEWNmzALkxDZe4xY8Dg= +github.com/go-fonts/dejavu v0.1.0/go.mod h1:4Wt4I4OU2Nq9asgDCteaAaWZOV24E+0/Pwo0gppep4g= +github.com/go-fonts/latin-modern v0.2.0/go.mod h1:rQVLdDMK+mK1xscDwsqM5J8U2jrRa3T0ecnM9pNujks= +github.com/go-fonts/liberation v0.1.1/go.mod h1:K6qoJYypsmfVjWg8KOVDQhLc8UDgIK2HYqyqAO9z7GY= +github.com/go-fonts/stix v0.1.0/go.mod h1:w/c1f0ldAUlJmLBvlbkvVXLAD+tAMqobIIQpmnUIzUY= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-kit/kit v0.10.0/go.mod h1:xUsJbQ/Fp4kEt7AFgCuvyX4a71u8h9jB8tj/ORgOZ7o= github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= +github.com/go-latex/latex v0.0.0-20210118124228-b3d85cf34e07/go.mod h1:CO1AlKB2CSIqUrmQPqA0gdRIlnLEY0gK5JGjh37zN5U= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= +github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= github.com/go-ole/go-ole v1.2.5 h1:t4MGB5xEDZvXI+0rMjjsfBsD7yAgp/s9ZDkL1JndXwY= github.com/go-ole/go-ole v1.2.5/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= +github.com/go-playground/assert/v2 v2.0.1 h1:MsBgLAaY856+nPRTKrp3/OZK38U/wa0CcBYNjji3q3A= github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= github.com/go-playground/locales v0.13.0 h1:HyWk6mgj5qFqCT5fjGBuRArbVDfE4hi8+e8ceBS/t7Q= github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8= @@ -194,8 +231,11 @@ github.com/go-playground/universal-translator v0.17.0 h1:icxd5fm+REJzpZx7ZfpaD87 github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA= github.com/go-playground/validator/v10 v10.4.1 h1:pH2c5ADXtd66mxoE0Zm9SUhxE20r7aM3F26W0hOn+GE= github.com/go-playground/validator/v10 v10.4.1/go.mod h1:nlOn6nFhuKACm19sB/8EGNn9GlaMV7XkbRSipzJ0Ii4= +github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/goccy/go-json v0.7.10 h1:ulhbuNe1JqE68nMRXXTJRrUu0uhouf0VevLINxQq4Ec= +github.com/goccy/go-json v0.7.10/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2 h1:ZpnhV/YsD2/4cESfV5+Hoeu/iUR3ruzNvZ+yQfO03a0= github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2/go.mod h1:bBOAhwG1umN6/6ZUMtDFBMQR8jRg9O75tm9K00oMsK4= github.com/godbus/dbus/v5 v5.0.4 h1:9349emZab16e7zQvpmsbtjc18ykshndd8y2PG3sgJbA= @@ -203,15 +243,16 @@ github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5x github.com/gofrs/flock v0.8.1 h1:+gYjHKf32LDeiEEFhQaotPbLuUXjY5ZqxKgXy7n59aw= github.com/gofrs/flock v0.8.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= -github.com/golang-jwt/jwt v3.2.1+incompatible h1:73Z+4BJcrTC+KczS6WvTPvRGOp1WmfEP4Q1lOd9Z/+c= -github.com/golang-jwt/jwt v3.2.1+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keLg81eXfW3O+oY= github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= +github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -243,12 +284,16 @@ github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaS github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= +github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= +github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.1 h1:gK4Kx5IaGY9CD5sPJ36FHiBJ6ZXl0kilRiiCj+jdYp4= github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA= +github.com/google/flatbuffers v2.0.5+incompatible h1:ANsW0idDAXIY+mNHzIHxWRfabV2x5LUEEIIWcwsYgB8= +github.com/google/flatbuffers v2.0.5+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -259,8 +304,9 @@ github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.7 h1:81/ik6ipDQS2aGcBfIN5dHDB36BwrStyeAQquSYCV4o= +github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= @@ -277,6 +323,7 @@ github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLe github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -284,21 +331,29 @@ github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+ github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= +github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= +github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gorilla/mux v1.7.4/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= +github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= +github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 h1:+9834+KizmvFV7pXQGSXQTsaWhq2GjuNUt0aUU0YBYw= github.com/grpc-ecosystem/go-grpc-middleware v1.3.0/go.mod h1:z0ButlSOZa5vEBq9m2m2hlwIgKw+rp3sdCBRoJY+30Y= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= +github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c h1:6rhixN/i8ZofjG1Y75iExal34USq5p+wiN1tpie8IrU= github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c/go.mod h1:NMPJylDgVpX0MLRlPy15sqSwOFv/U1GZ2m21JhFfek0= github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= +github.com/hashicorp/consul/api v1.3.0/go.mod h1:MmDNSzIMUjNpY/mQ398R4bk2FnqQLoPndWW5VkKPlCE= github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= +github.com/hashicorp/consul/sdk v0.3.0/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= @@ -309,6 +364,7 @@ github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerX github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= @@ -319,25 +375,24 @@ github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0m github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/hudl/fargo v1.3.0/go.mod h1:y3CKSmjA+wD2gak7sUSXTAoopbhU08POFhmITJgmKTg= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo= github.com/jarcoal/httpmock v1.0.8 h1:8kI16SoO6LQKgPE7PvQuV+YuD/inwHd7fOOe2zMbo4k= github.com/jarcoal/httpmock v1.0.8/go.mod h1:ATjnClrvW/3tijVmpL/va5Z3aAyGvqU3gCT8nX0Txik= github.com/jawher/mow.cli v1.0.4/go.mod h1:5hQj2V8g+qYmLUVWqu4Wuja1pI57M83EChYLVZ0sMKk= github.com/jawher/mow.cli v1.2.0/go.mod h1:y+pcA3jBAdo/GIZx/0rFjw/K2bVEODP9rfZOfaiq8Ko= -github.com/jhump/gopoet v0.0.0-20190322174617-17282ff210b3/go.mod h1:me9yfT6IJSlOL3FCfrg+L6yzUEZ+5jW6WHt4Sk+UPUI= -github.com/jhump/gopoet v0.1.0/go.mod h1:me9yfT6IJSlOL3FCfrg+L6yzUEZ+5jW6WHt4Sk+UPUI= -github.com/jhump/goprotoc v0.5.0/go.mod h1:VrbvcYrQOrTi3i0Vf+m+oqQWk9l72mjkJCYo7UvLHRQ= -github.com/jhump/protoreflect v1.11.0/go.mod h1:U7aMIjN0NWq9swDP7xDdoMfRHb35uiuTd3Z9nFXJf5E= -github.com/jhump/protoreflect v1.12.0 h1:1NQ4FpWMgn3by/n1X0fbeKEUxP1wBt7+Oitpv01HR10= -github.com/jhump/protoreflect v1.12.0/go.mod h1:JytZfP5d0r8pVNLZvai7U/MCuTWITgrI4tTg7puQFKI= +github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/jonboulle/clockwork v0.2.2 h1:UOGuzwb1PwsrDAObMuhUnj0p5ULPj8V/xJ7Kx9qUBdQ= github.com/jonboulle/clockwork v0.2.2/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.11 h1:uVUAXhF2To8cbw/3xN3pxj6kk7TYKs98NIrTqPlMWAQ= @@ -348,16 +403,22 @@ github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7 github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= +github.com/jung-kurt/gofpdf v1.0.0/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes= +github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/asmfmt v1.3.1 h1:7xZi1N7s9gTLbqiM8KUv8TLyysavbTRGBT5/ly0bRtw= +github.com/klauspost/asmfmt v1.3.1/go.mod h1:AG8TuvYojzulgDAMCnYn50l/5QV3Bs/tp6j0HLHbNSE= github.com/klauspost/compress v1.10.8/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= -github.com/klauspost/compress v1.10.11 h1:K9z59aO18Aywg2b/WSgBaUX99mHy2BES18Cr5lBKZHk= -github.com/klauspost/compress v1.10.11/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= +github.com/klauspost/compress v1.14.2 h1:S0OHlFk/Gbon/yauFJ4FfJJF5V0fc5HbBTJazi28pRw= +github.com/klauspost/compress v1.14.2/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/klauspost/cpuid v1.2.3/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/klauspost/cpuid v1.3.1 h1:5JNjFYYQrZeKRJ0734q51WCEEn2huer72Dc7K+R/b6s= github.com/klauspost/cpuid v1.3.1/go.mod h1:bYW4mA6ZgKPob1/Dlai2LviZJO7KGI3uoWLd42rAQw4= +github.com/klauspost/cpuid/v2 v2.0.9 h1:lgaqFMSdTdQYdZ04uHyN2d/eKdOMyi2YLSvlQIBFYa4= +github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= @@ -372,6 +433,8 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y= github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= +github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM= +github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4= github.com/lingdor/stackerror v0.0.0-20191119040541-976d8885ed76 h1:IVlcvV0CjvfBYYod5ePe89l+3LBAl//6n9kJ9Vr2i0k= github.com/lingdor/stackerror v0.0.0-20191119040541-976d8885ed76/go.mod h1:Iu9BHUvTh8/KpbuSoKx/CaJEdJvFxSverxIy7I+nq7s= github.com/linkedin/goavro/v2 v2.9.8 h1:jN50elxBsGBDGVDEKqUlDuU1cFwJ11K/yrJCBMe/7Wg= @@ -383,8 +446,10 @@ github.com/magiconair/properties v1.8.5 h1:b6kJs+EmPFMYGkow9GiUyCyOvIwYetYJ3fSaW github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= +github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-runewidth v0.0.8/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= @@ -392,6 +457,10 @@ github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5 github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/milvus-io/pulsar-client-go v0.6.1-0.20220310065106-1ef6d309ead7 h1:oe+20W/i8wBJ05HznX+HKbk1xDk0+vuVNfNh90l6weo= github.com/milvus-io/pulsar-client-go v0.6.1-0.20220310065106-1ef6d309ead7/go.mod h1:A1P5VjjljsFKAD13w7/jmU3Dly2gcRvcobiULqQXhz4= +github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 h1:AMFGa4R4MiIpspGNG7Z948v4n35fFGB3RR3G/ry4FWs= +github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8/go.mod h1:mC1jAcsrzbxHt8iiaC+zU4b1ylILSosueou12R++wfY= +github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 h1:+n/aFZefKZp7spd8DFdX7uMikMLXX4oubIzJF4kv/wI= +github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3/go.mod h1:RagcQ7I8IeTMnF8JTXieKnO4Z6JCsikNEzj0DwauVzE= github.com/minio/md5-simd v1.1.0 h1:QPfiOqlZH+Cj9teu0t9b1nTBfPbyTl16Of5MeuShdK4= github.com/minio/md5-simd v1.1.0/go.mod h1:XpBqgZULrMYD3R+M28PcmP0CkI7PEMzB3U77ZrKZ0Gw= github.com/minio/minio-go/v7 v7.0.10 h1:1oUKe4EOPUEhw2qnPQaPsJ0lmVTYLFu03SiItauXs94= @@ -419,67 +488,103 @@ github.com/mtibben/percent v0.2.1 h1:5gssi8Nqo8QU/r2pynCm+hBQHpkB/uNK7BJCFogWdzs github.com/mtibben/percent v0.2.1/go.mod h1:KG9uO+SZkUp+VkRHsCdYQV3XSZrrSpR3O9ibNBTZrns= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/nats-io/jwt v0.3.0/go.mod h1:fRYCDE99xlTsqUzISS1Bi75UBJ6ljOJQOAAu5VglpSg= +github.com/nats-io/jwt v0.3.2/go.mod h1:/euKqTS1ZD+zzjYrY7pseZrTtWQSjujC7xjPc8wL6eU= +github.com/nats-io/nats-server/v2 v2.1.2/go.mod h1:Afk+wRZqkMQs/p45uXdrVLuab3gwv3Z8C4HTBu8GD/k= +github.com/nats-io/nats.go v1.9.1/go.mod h1:ZjDU1L/7fJ09jvUSRVBR2e7+RnLiiIQyqyzEE/Zbp4w= +github.com/nats-io/nkeys v0.1.0/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= +github.com/nats-io/nkeys v0.1.3/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= +github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= +github.com/oklog/oklog v0.3.2/go.mod h1:FCV+B7mhrz4o+ueLpx+KqkyXRGMWOYEvfiXtdGtbWGs= +github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= +github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= github.com/onsi/ginkgo v1.14.0 h1:2mOpI4JVVPBN+WQRa0WKH2eXR+Ey+uK4n7Zj0aYpIQA= github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= +github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1 h1:o0+MgICZLuZ7xjH7Vx6zS/zcu93/BEp1VwkIW1mEXCE= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= +github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= github.com/opencontainers/runtime-spec v1.0.2 h1:UfAcuLBJB9Coz72x1hgl8O5RVzTdNiaglX6v2DM6FI0= github.com/opencontainers/runtime-spec v1.0.2/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= +github.com/opentracing-contrib/go-observer v0.0.0-20170622124052-a52f23424492/go.mod h1:Ngi6UdF0k5OKD5t5wlmGhe/EDKPoUM3BXZSSfIuJbis= +github.com/opentracing/basictracer-go v1.0.0/go.mod h1:QfBfYuafItcjQuMwinw9GhYKwFXS9KnPs5lxoYwgW74= +github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs= github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc= +github.com/openzipkin-contrib/zipkin-go-opentracing v0.4.5/go.mod h1:/wsWhb9smxSfWAKL3wpBW7V8scJMt8N8gnaMCS9E/cA= +github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw= +github.com/openzipkin/zipkin-go v0.2.1/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4= +github.com/openzipkin/zipkin-go v0.2.2/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4= +github.com/pact-foundation/pact-go v1.0.4/go.mod h1:uExwJY4kCzNPcHRj+hCR/HBbOOIwwtUjcrb0b5/5kLM= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc= github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= +github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pelletier/go-toml v1.9.3 h1:zeC5b1GviRUyKYd6OJPvBU/mcVDVoL1OhT17FCt5dSQ= github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= +github.com/performancecopilot/speed v3.0.0+incompatible/go.mod h1:/CLtqpZ5gBg1M9iaPbIdPPGyKcA8hKdoy6hAWba7Yac= +github.com/phpdave11/gofpdf v1.4.2/go.mod h1:zpO6xFn9yxo3YLyMvW8HcKWVdbNqgIfOOp2dXMnm1mY= +github.com/phpdave11/gofpdi v1.0.12/go.mod h1:vBmVV0Do6hSBHC8uKUQ71JGW+ZGQq74llk/7bXwjDoI= +github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc= github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pierrec/lz4 v2.5.2+incompatible h1:WCjObylUIOlKy/+7Abdn34TLIkXiA4UWUMhxq9m9ZXI= github.com/pierrec/lz4 v2.5.2+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= +github.com/pierrec/lz4/v4 v4.1.12 h1:44l88ehTZAUGW4VlO1QC4zkilL99M6Y9MXNwEs0uzP8= +github.com/pierrec/lz4/v4 v4.1.12/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/profile v1.2.1/go.mod h1:hJw3o1OdXxsrSjjVksARp5W95eeEaEfptyVZyv6JUPA= github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= github.com/pmezard/go-difflib v0.0.0-20151028094244-d8ed2627bdf0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod h1:p2iRAGwDERtqlqzRXnrOVns+ignqQo//hLXqYxZYVNs= github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= +github.com/prometheus/client_golang v1.3.0/go.mod h1:hJaj2vgQTGQmVCsAACORcieXFeDPbaTKGT+JTgUa3og= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= github.com/prometheus/client_golang v1.11.0 h1:HNkLOAEQMIDv/K+04rukrLx6ch7msSRwf3/SASFAGtQ= github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.1.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= +github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= github.com/prometheus/common v0.26.0 h1:iMAkS2TDoNWnKM+Kopnx/8tnEStIfpYA0ur0xQzzhMQ= github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.6.0 h1:mxy4L2jP6qMonqmq+aTtOx1ifVWUgG/TAmntgbh3xv4= github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= -github.com/quasilyte/go-ruleguard v0.2.1 h1:56eRm0daAyny9UhJnmtJW/UyLZQusukBAB8oT8AHKHo= -github.com/quasilyte/go-ruleguard v0.2.1/go.mod h1:hN2rVc/uS4bQhQKTio2XaSJSafJwqBUWWwtssT3cQmc= +github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/rivo/tview v0.0.0-20200219210816-cd38d7432498/go.mod h1:6lkG1x+13OShEf0EaOCaTQYyB7d5nSbb181KtjlS+84= github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= @@ -487,8 +592,11 @@ github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6L github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rs/xid v1.2.1 h1:mhH9Nq+C1fY2l1XIpgxIiUOfNpRBYH1kKcr+qfKgjRc= github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= +github.com/rs/zerolog v1.21.0/go.mod h1:ZPhntP/xmq1nnND05hhpAh2QMhSsA4UN3MGZ6O2J3hM= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/ruudk/golang-pdf417 v0.0.0-20181029194003-1af4ab5afa58/go.mod h1:6lfFZQK844Gfx8o5WFuvpxWRwnSoipWe/p622j1v06w= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= +github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E= github.com/sanity-io/litter v1.2.0/go.mod h1:JF6pZUFgu2Q0sBZ+HSV35P8TVPI1TTzEwyu9FXAw2W4= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/shirou/gopsutil v3.21.8+incompatible h1:sh0foI8tMRlCidUJR+KzqWYWxrkuuPIGiO6Vp+KXdCU= @@ -507,6 +615,7 @@ github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9 github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/soheilhy/cmux v0.1.5 h1:jjzc5WVemNEDTLwv9tlmemhC73tI08BNOIGwBOo10Js= github.com/soheilhy/cmux v0.1.5/go.mod h1:T7TcVDs9LWfQgPlPsdngu6I6QIoyIFZDDC6sNE1GqG0= +github.com/sony/gobreaker v0.4.1/go.mod h1:ZKptC7FHNvhBz7dN2LGjPVBz2sZJmc0/PkyDJOjmxWY= github.com/soothing-rain/gorocksdb v0.0.1 h1:Eo4u7O86BYiMZNw0VGQaVDHJxHi+Jxnws23UMTL34dU= github.com/soothing-rain/gorocksdb v0.0.1/go.mod h1:OEi1TEIyGy/kDfY3ZwoHfn+/dIK0ZOOKJ2ReaYf4sao= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= @@ -520,16 +629,19 @@ github.com/spf13/cast v1.3.1 h1:nFm6S0SMdyzrzcmThSipiEubIDy8WEXKNZ0UOgiRpng= github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v1.1.3/go.mod h1:pGADOWyqRD/YMrPZigI/zbliZ2wVD/23d+is3pSWzOo= -github.com/spf13/cobra v1.2.1/go.mod h1:ExllRjgxM/piMAM+3tAZvg8fsklGAf3tPfi+i8t68Nk= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk= github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= +github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= github.com/spf13/viper v1.8.1 h1:Kq1fyeebqsBfbjZj4EL7gj2IO0mMaiyjYUWcUsl2O44= github.com/spf13/viper v1.8.1/go.mod h1:o0Pch8wJ9BVSWGQMbra6iw0oQ5oktSIBaujf1rJH9Ns= +github.com/streadway/amqp v0.0.0-20190404075320-75d898a42a94/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= +github.com/streadway/amqp v0.0.0-20190827072141-edfb9018d271/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= +github.com/streadway/handy v0.0.0-20190108123426-d5acb3125c2a/go.mod h1:qNTQ5P5JnDBl6z3cMAg/SywNDC5ABu5ApDIw6lUbRmI= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.2.0 h1:Hbg2NidpLE8veEBkEZTL3CvlkUIVzuU9jDplZO54c48= @@ -544,12 +656,11 @@ github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5Cc github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= -github.com/tecbot/gorocksdb v0.0.0-20191217155057-f0fad39f321c h1:g+WoO5jjkqGAzHWCjJB1zZfXPIAaDpzXIEJ0eS6B5Ok= -github.com/tecbot/gorocksdb v0.0.0-20191217155057-f0fad39f321c/go.mod h1:ahpPrc7HpcfEWDQRZEmnXMzHY03mLDYMCxeDzy46i+8= github.com/tklauser/go-sysconf v0.3.9 h1:JeUVdAOWhhxVcU6Eqr/ATFHgXk/mmiItdKeJPev3vTo= github.com/tklauser/go-sysconf v0.3.9/go.mod h1:11DU/5sG7UexIrp/O6g35hrWzu0JxlwQ3LSFUzyeuhs= github.com/tklauser/numcpus v0.3.0 h1:ILuRUQBtssgnxw0XXIjKUC56fgnOrFoQQ/4+DeU2biQ= github.com/tklauser/numcpus v0.3.0/go.mod h1:yFGUr7TUHQRAhyqBcEg0Ge34zDBAsIvJJcyE6boqnA8= +github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 h1:uruHq4dN7GR16kFc5fp3d1RIYzJW5onx8Ybykw2YQFA= github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= @@ -561,6 +672,8 @@ github.com/ugorji/go v1.1.7 h1:/68gy2h+1mWMrwZFeD1kQialdSzAb432dtpeJ42ovdo= github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw= github.com/ugorji/go/codec v1.1.7 h1:2SvQaVZ1ouYrrKKwoSk2pzd4A9evlKJb9oTL+OaLUSs= github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY= +github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= +github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/urfave/cli v1.22.2/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= @@ -569,10 +682,15 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= -github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/zeebo/xxh3 v1.0.1 h1:FMSRIbkrLikb/0hZxmltpg84VkqDAT5M8ufXynuhXsI= +github.com/zeebo/xxh3 v1.0.1/go.mod h1:8VHV24/3AZLn3b6Mlp/KuC33LWH687Wq6EnziEB+rsA= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= +go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.6 h1:/ecaJf0sk1l4l6V4awd65v2C3ILy7MSj+s/x1ADCIMU= go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4= +go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738 h1:VcrIfasaLFkyjk6KNlXQSzO+B0fZcnECiDrKJsfxka0= +go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg= go.etcd.io/etcd/api/v3 v3.5.0 h1:GsV3S+OfZEOCNXdtNkBSR7kgLobAa/SO6tCxRa0GAYw= go.etcd.io/etcd/api/v3 v3.5.0/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs= go.etcd.io/etcd/client/pkg/v3 v3.5.0 h1:2aQv6F436YnN7I4VbI8PPYrBhu+SmrTaADcf8Mi/6PU= @@ -587,6 +705,8 @@ go.etcd.io/etcd/raft/v3 v3.5.0 h1:kw2TmO3yFTgE+F0mdKkG7xMxkit2duBDa2Hu6D/HMlw= go.etcd.io/etcd/raft/v3 v3.5.0/go.mod h1:UFOHSIvO/nKwd4lhkwabrTD3cqW5yVyYYf/KlD00Szc= go.etcd.io/etcd/server/v3 v3.5.0 h1:jk8D/lwGEDlQU9kZXUFMSANkE22Sg5+mW27ip8xcF9E= go.etcd.io/etcd/server/v3 v3.5.0/go.mod h1:3Ah5ruV+M+7RZr0+Y/5mNLwC+eQlni+mQmOVdCRJoS4= +go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= +go.opencensus.io v0.20.2/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= @@ -616,7 +736,10 @@ go.opentelemetry.io/otel/trace v0.20.0 h1:1DL6EXUdcg95gukhuRRvLDO/4X5THh/5dIV52l go.opentelemetry.io/otel/trace v0.20.0/go.mod h1:6GjCW8zgDjwGHGa6GkyeB8+/5vjT16gUEi0Nf1iBdgw= go.opentelemetry.io/proto/otlp v0.7.0 h1:rwOQPCuKAKmwGKq2aVNnYIibI6wnV7EvzgfTCzcdGg8= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= +go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= +go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/automaxprocs v1.4.0 h1:CpDZl6aOlLhReez+8S3eEotD7Jx0Os++lemPlMULQP0= @@ -624,9 +747,14 @@ go.uber.org/automaxprocs v1.4.0/go.mod h1:/mTEdr7LvHhs0v7mjdxDreTz1OG5zdZGqgOnhW go.uber.org/goleak v1.1.10 h1:z+mqJhf6ss6BSfSM671tgKyZBFPTTJM+HLxnhPC3wu0= go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= +go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= +go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= +go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= +go.uber.org/zap v1.16.0/go.mod h1:MA8QOfq0BHJwdXa996Y4dYkAqRKB8/1K1QMMZVaNZjQ= go.uber.org/zap v1.17.0 h1:MTjgFu6ZLKvY6Pvaqk97GlxNBuMpV4Hy/3P6tRGlI2U= go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -639,21 +767,35 @@ golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200709230013-948cd5f35899/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0 h1:hb9wdF1z5waM+dSIICn1l0DkLVDT3hqhhQsDNUmHPRE= golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 h1:7I4JAnoQBe7ZtJcBaYHi5UtiO8tQHbUSXxL+pnGRANg= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= +golang.org/x/exp v0.0.0-20190731235908-ec7cb31e5a56/go.mod h1:JhuoJpWY28nO4Vef9tZUw9qufEGTyX1+7lmHxV5q5G4= golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= +golang.org/x/exp v0.0.0-20191002040644-a1355ae1e2c3/go.mod h1:NOZ3BPKG0ec/BKJQgnvsSFpcKLM5xXVWnvZS97DWHgE= golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= -golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6 h1:QE6XYQK6naiK1EPAe1g/ILLxN5RBoH5xkJk3CqlMI/Y= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= +golang.org/x/exp v0.0.0-20211216164055-b2b84827b756 h1:/5Bs7sWi0i3rOVO5KnM55OwugpsD4bRW1zywKoZjbkI= +golang.org/x/exp v0.0.0-20211216164055-b2b84827b756/go.mod h1:b9TAUYHmRtqA6klRHApnXMnj+OyLce4yF5cZCUbk2ps= +golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/image v0.0.0-20190910094157-69e4b8554b2a/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/image v0.0.0-20200119044424-58c23975cae1/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/image v0.0.0-20200430140353-33d19683fad8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/image v0.0.0-20200618115811-c13761719519/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/image v0.0.0-20201208152932-35266b937fa6/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/image v0.0.0-20210216034530-4410531fe030/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= @@ -668,16 +810,20 @@ golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 h1:VLliZ0d+/avPrXXH+OakdXhp golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= +golang.org/x/mobile v0.0.0-20201217150744-e6ae53a27f4f/go.mod h1:skQtrUTUwhdJvXM/2KKJzY8pDgNr9I/FOMqDVRPBUS4= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.1.1-0.20191209134235-331c550502dd/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.2 h1:Gz96sIWK3OalVv/I/qNygP42zyoKp3xptRVCWRFEBvo= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= +golang.org/x/mod v0.6.0-dev.0.20211013180041-c96bc1413d57 h1:LQmS1nU0twXLA96Kt7U9qtHJEbBk3z6Q0V4UXjZkpr4= +golang.org/x/mod v0.6.0-dev.0.20211013180041-c96bc1413d57/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -685,6 +831,7 @@ golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73r golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190125091013-d26f9f9a57f3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -695,6 +842,7 @@ golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -720,9 +868,9 @@ golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d h1:20cMwl2fHAzkJMEA+8J4JgqBQcQGzbisXo31MIeenXI= -golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd h1:O7DYs+zxREGLKzKoMQrtrEacpb0ZVXA5rIwylE2Xchk= +golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -754,6 +902,7 @@ golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -765,6 +914,7 @@ golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190626150813-e07cf5db2756/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190712062909-fae7ac547cb7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -772,6 +922,7 @@ golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -800,6 +951,7 @@ golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210304124612-50617c2ba197/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -808,10 +960,15 @@ golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210816074244-15123e1e1f71 h1:ikCpsnYR+Ew0vu99XlDp55lGgDJdIMx3f4a18jfse/s= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210816074244-15123e1e1f71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27 h1:XDXtA5hveEEV8JB2l7nhMTp3t3cHp9ZpwcdjqyEWLlo= +golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 h1:JGgROgKl9N8DuW20oFS5gxc+lE67/N3FcwmBPMe7ArY= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -819,16 +976,20 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba h1:O8mE0/t419eoIwhTFpKVkHiTs/Igowgfkj25AcZrtiE= golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190206041539-40960b6deb8e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= @@ -842,7 +1003,10 @@ golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgw golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190927191325-030b2cf1153e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -852,6 +1016,8 @@ golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200103221440-774c71fcf114/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200117012304-6edc0a871e69/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= @@ -870,7 +1036,6 @@ golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roY golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200812195022-5ae4c3c160a0/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= @@ -880,13 +1045,22 @@ golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.7 h1:6j8CgantCy3yc8JGBqkDLMKWqZ0RDU2g1HVgacojGWQ= -golang.org/x/tools v0.1.7/go.mod h1:LGqMHiF4EqQNHR1JncWGqT5BVaXmza+X+BDGol+dOxo= +golang.org/x/tools v0.1.8-0.20211029000441-d6a9af8af023/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU= +golang.org/x/tools v0.1.9 h1:j9KsMiaP1c3B0OTQGth0/k+miLGTgLsAFUCrF2vLcF8= +golang.org/x/tools v0.1.9/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gonum.org/v1/gonum v0.0.0-20180816165407-929014505bf4/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo= +gonum.org/v1/gonum v0.8.2/go.mod h1:oe/vMfY3deqTw+1EZJhuvEW2iwGF1bW9wwu7XCu0+v0= +gonum.org/v1/gonum v0.9.3 h1:DnoIG+QAMaF5NvxnGe/oKsgKcAc6PcUyl8q0VetfQ8s= +gonum.org/v1/gonum v0.9.3/go.mod h1:TZumC3NeyVQskjXqmyWt4S3bINhy7B4eYwW69EbyX+0= +gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= +gonum.org/v1/plot v0.0.0-20190515093506-e2840ee46a6b/go.mod h1:Wt8AAjI+ypCyYX3nZBvf6cAIx93T+c/OS2HFAYskSZc= +gonum.org/v1/plot v0.9.0/go.mod h1:3Pcqqmp6RHvJI72kgb8fThyUnav364FOsdDo2aGW5lY= +google.golang.org/api v0.3.1/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= @@ -909,6 +1083,7 @@ google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjR google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU= google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94= google.golang.org/api v0.44.0/go.mod h1:EBOGZqzyhtvMDoxwS97ctnh0zUmYY6CxqXsc1AvkYD8= +google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= @@ -921,6 +1096,7 @@ google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRn google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190530194941-fb225487d101/go.mod h1:z3L6/3dTEVtUr6QSP8miRzeRqwQOioJ9I66odjN4I7s= google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= @@ -957,8 +1133,9 @@ google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= -google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c h1:wtujag7C+4D6KMoulW9YauvK2lgdvCMS260jsqqBXr0= google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= +google.golang.org/genproto v0.0.0-20220126215142-9970aeb2e350 h1:YxHp5zqIcAShDEvRr5/0rVESVS+njYF68PSdazrNLJo= +google.golang.org/genproto v0.0.0-20220126215142-9970aeb2e350/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/grpc v1.38.0 h1:/9BgsAsa5nWe26HqOlvlgJnqBuktYOLCgjCPqsa56W0= google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= @@ -972,16 +1149,19 @@ google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpAD google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.26.0 h1:bxAC2xTBsZGibn2RTntX0oH50xLsqy1OxA9tTL3p/lk= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ= +google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/gcfg.v1 v1.2.3/go.mod h1:yesOnuUOFQAhST5vPY4nbZsb/huCgGGXlipJsBn0b3o= gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.57.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.62.0 h1:duBzk771uxoUuOlyRLkHsygud9+5lrlGjdFBb4mSKDU= @@ -992,6 +1172,7 @@ gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/square/go-jose.v2 v2.4.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= @@ -1012,10 +1193,14 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +honnef.co/go/tools v0.1.3/go.mod h1:NgwopIslSNH47DimFoV78dnkksY2EFtX0ajyb3K/las= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= +rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= +sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q= sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= +sourcegraph.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0/go.mod h1:hI742Nqp5OhwiqlzhgfbWU4mW4yO10fP+LoT9WOswdU= stathat.com/c/consistent v1.0.0 h1:ezyc51EGcRPJUxfHGSgJjWzJdj3NiMU9pNfLNGiXV0c= stathat.com/c/consistent v1.0.0/go.mod h1:QkzMWzcbB+yQBL2AttO6sgsQS/JSTapcDISJalmCDS0= diff --git a/internal/storage/binlog_reader.go b/internal/storage/binlog_reader.go index 6f3f395bef..9ae7657455 100644 --- a/internal/storage/binlog_reader.go +++ b/internal/storage/binlog_reader.go @@ -18,7 +18,12 @@ package storage import ( "bytes" + "encoding/binary" "errors" + "fmt" + "io" + + "github.com/milvus-io/milvus/internal/common" ) // BinlogReader is an object to read binlog file. Binlog file's format can be @@ -26,9 +31,9 @@ import ( type BinlogReader struct { magicNumber int32 descriptorEvent - buffer *bytes.Buffer - eventList []*EventReader - isClose bool + buffer *bytes.Buffer + eventReader *EventReader + isClose bool } // NextEventReader iters all events reader to read the binlog file. @@ -39,12 +44,16 @@ func (reader *BinlogReader) NextEventReader() (*EventReader, error) { if reader.buffer.Len() <= 0 { return nil, nil } - eventReader, err := newEventReader(reader.descriptorEvent.PayloadDataType, reader.buffer) + // close the previous reader + if reader.eventReader != nil { + reader.eventReader.Close() + } + var err error + reader.eventReader, err = newEventReader(reader.descriptorEvent.PayloadDataType, reader.buffer) if err != nil { return nil, err } - reader.eventList = append(reader.eventList, eventReader) - return eventReader, nil + return reader.eventReader, nil } func (reader *BinlogReader) readMagicNumber() (int32, error) { @@ -62,14 +71,42 @@ func (reader *BinlogReader) readDescriptorEvent() (*descriptorEvent, error) { return &reader.descriptorEvent, nil } +func readMagicNumber(buffer io.Reader) (int32, error) { + var magicNumber int32 + if err := binary.Read(buffer, common.Endian, &magicNumber); err != nil { + return -1, err + } + if magicNumber != MagicNumber { + return -1, fmt.Errorf("parse magic number failed, expected: %d, actual: %d", MagicNumber, magicNumber) + } + + return magicNumber, nil +} + +// ReadDescriptorEvent reads a descriptorEvent from buffer +func ReadDescriptorEvent(buffer io.Reader) (*descriptorEvent, error) { + header, err := readDescriptorEventHeader(buffer) + if err != nil { + return nil, err + } + data, err := readDescriptorEventData(buffer) + if err != nil { + return nil, err + } + return &descriptorEvent{ + descriptorEventHeader: *header, + descriptorEventData: *data, + }, nil +} + // Close closes the BinlogReader object. // It mainly calls the Close method of the internal events, reclaims resources, and marks itself as closed. func (reader *BinlogReader) Close() { if reader.isClose { return } - for _, e := range reader.eventList { - e.Close() + if reader.eventReader != nil { + reader.eventReader.Close() } reader.isClose = true } @@ -77,9 +114,8 @@ func (reader *BinlogReader) Close() { // NewBinlogReader creates binlogReader to read binlog file. func NewBinlogReader(data []byte) (*BinlogReader, error) { reader := &BinlogReader{ - buffer: bytes.NewBuffer(data), - eventList: []*EventReader{}, - isClose: false, + buffer: bytes.NewBuffer(data), + isClose: false, } if _, err := reader.readMagicNumber(); err != nil { diff --git a/internal/storage/binlog_writer_test.go b/internal/storage/binlog_writer_test.go index dd6e49b4bb..6b5d5bcf2e 100644 --- a/internal/storage/binlog_writer_test.go +++ b/internal/storage/binlog_writer_test.go @@ -18,7 +18,6 @@ package storage import ( "fmt" - "strconv" "testing" "github.com/milvus-io/milvus/internal/proto/schemapb" @@ -59,7 +58,6 @@ func TestBinlogWriterReader(t *testing.T) { buffer, err := binlogWriter.GetBuffer() assert.Nil(t, err) - fmt.Println("reader offset : " + strconv.Itoa(len(buffer))) binlogWriter.Close() binlogReader, err := NewBinlogReader(buffer) diff --git a/internal/storage/data_codec.go b/internal/storage/data_codec.go index 1170d1f34c..83a5758945 100644 --- a/internal/storage/data_codec.go +++ b/internal/storage/data_codec.go @@ -249,6 +249,7 @@ type BlobInfo struct { // InsertData example row_schema: {float_field, int_field, float_vector_field, string_field} // Data {<0, row_id>, <1, timestamp>, <100, float_field>, <101, int_field>, <102, float_vector_field>, <103, string_field>} type InsertData struct { + // Todo, data should be zero copy by passing data directly to event reader or change Data to map[FieldID]FieldDataArray Data map[FieldID]FieldData // field id to field data Infos []BlobInfo } @@ -633,14 +634,14 @@ func (insertCodec *InsertCodec) DeserializeAll(blobs []*Blob) ( } totalLength += length stringFieldData.NumRows = append(stringFieldData.NumRows, int64(length)) - for i := 0; i < length; i++ { - singleString, err := eventReader.GetOneStringFromPayload(i) - if err != nil { - eventReader.Close() - binlogReader.Close() - return InvalidUniqueID, InvalidUniqueID, InvalidUniqueID, nil, err - } - stringFieldData.Data = append(stringFieldData.Data, singleString) + stringPayload, err := eventReader.GetStringFromPayload() + if err != nil { + eventReader.Close() + binlogReader.Close() + return InvalidUniqueID, InvalidUniqueID, InvalidUniqueID, nil, err + } + for idx := range stringPayload { + stringFieldData.Data = append(stringFieldData.Data, stringPayload[idx]) } resultData.Data[fieldID] = stringFieldData case schemapb.DataType_BinaryVector: @@ -828,19 +829,18 @@ func (deleteCodec *DeleteCodec) Deserialize(blobs []*Blob) (partitionID UniqueID return InvalidUniqueID, InvalidUniqueID, nil, err } + stringarray, err := eventReader.GetStringFromPayload() + if err != nil { + eventReader.Close() + binlogReader.Close() + return InvalidUniqueID, InvalidUniqueID, nil, err + } for i := 0; i < length; i++ { - singleString, err := eventReader.GetOneStringFromPayload(i) - if err != nil { - eventReader.Close() - binlogReader.Close() - return InvalidUniqueID, InvalidUniqueID, nil, err - } - - splits := strings.Split(singleString, ",") + splits := strings.Split(stringarray[i], ",") if len(splits) != 2 { eventReader.Close() binlogReader.Close() - return InvalidUniqueID, InvalidUniqueID, nil, fmt.Errorf("the format of delta log is incorrect") + return InvalidUniqueID, InvalidUniqueID, nil, fmt.Errorf("the format of delta log is incorrect, %v can not be split", stringarray[i]) } pk, err := strconv.ParseInt(splits[0], 10, 64) @@ -1044,20 +1044,14 @@ func (dataDefinitionCodec *DataDefinitionCodec) Deserialize(blobs []*Blob) (ts [ resultTs = append(resultTs, Timestamp(singleTs)) } case schemapb.DataType_String: - length, err := eventReader.GetPayloadLengthFromReader() + stringPayload, err := eventReader.GetStringFromPayload() if err != nil { eventReader.Close() binlogReader.Close() return nil, nil, err } - for i := 0; i < length; i++ { - singleString, err := eventReader.GetOneStringFromPayload(i) - if err != nil { - eventReader.Close() - binlogReader.Close() - return nil, nil, err - } - requestsStrings = append(requestsStrings, singleString) + for idx := range stringPayload { + requestsStrings = append(requestsStrings, stringPayload[idx]) } } eventReader.Close() diff --git a/internal/storage/event_test.go b/internal/storage/event_test.go index a45b7eafb0..857afafdf2 100644 --- a/internal/storage/event_test.go +++ b/internal/storage/event_test.go @@ -333,34 +333,21 @@ func TestInsertEvent(t *testing.T) { pR, err := NewPayloadReader(schemapb.DataType_String, pBuf) assert.Nil(t, err) - s0, err := pR.GetOneStringFromPayload(0) + s, err := pR.GetStringFromPayload() assert.Nil(t, err) - assert.Equal(t, s0, "1234") - - s1, err := pR.GetOneStringFromPayload(1) - assert.Nil(t, err) - assert.Equal(t, s1, "567890") - - s2, err := pR.GetOneStringFromPayload(2) - assert.Nil(t, err) - assert.Equal(t, s2, "abcdefg") - + assert.Equal(t, s[0], "1234") + assert.Equal(t, s[1], "567890") + assert.Equal(t, s[2], "abcdefg") pR.Close() r, err := newEventReader(schemapb.DataType_String, bytes.NewBuffer(wBuf)) assert.Nil(t, err) - s0, err = r.GetOneStringFromPayload(0) + s, err = pR.GetStringFromPayload() assert.Nil(t, err) - assert.Equal(t, s0, "1234") - - s1, err = r.GetOneStringFromPayload(1) - assert.Nil(t, err) - assert.Equal(t, s1, "567890") - - s2, err = r.GetOneStringFromPayload(2) - assert.Nil(t, err) - assert.Equal(t, s2, "abcdefg") + assert.Equal(t, s[0], "1234") + assert.Equal(t, s[1], "567890") + assert.Equal(t, s[2], "abcdefg") r.Close() }) @@ -573,34 +560,22 @@ func TestDeleteEvent(t *testing.T) { pR, err := NewPayloadReader(schemapb.DataType_String, pBuf) assert.Nil(t, err) - s0, err := pR.GetOneStringFromPayload(0) + s, err := pR.GetStringFromPayload() assert.Nil(t, err) - assert.Equal(t, s0, "1234") - - s1, err := pR.GetOneStringFromPayload(1) - assert.Nil(t, err) - assert.Equal(t, s1, "567890") - - s2, err := pR.GetOneStringFromPayload(2) - assert.Nil(t, err) - assert.Equal(t, s2, "abcdefg") + assert.Equal(t, s[0], "1234") + assert.Equal(t, s[1], "567890") + assert.Equal(t, s[2], "abcdefg") pR.Close() r, err := newEventReader(schemapb.DataType_String, bytes.NewBuffer(wBuf)) assert.Nil(t, err) - s0, err = r.GetOneStringFromPayload(0) + s, err = pR.GetStringFromPayload() assert.Nil(t, err) - assert.Equal(t, s0, "1234") - - s1, err = r.GetOneStringFromPayload(1) - assert.Nil(t, err) - assert.Equal(t, s1, "567890") - - s2, err = r.GetOneStringFromPayload(2) - assert.Nil(t, err) - assert.Equal(t, s2, "abcdefg") + assert.Equal(t, s[0], "1234") + assert.Equal(t, s[1], "567890") + assert.Equal(t, s[2], "abcdefg") r.Close() }) @@ -687,34 +662,22 @@ func TestCreateCollectionEvent(t *testing.T) { pR, err := NewPayloadReader(schemapb.DataType_String, pBuf) assert.Nil(t, err) - s0, err := pR.GetOneStringFromPayload(0) + s, err := pR.GetStringFromPayload() assert.Nil(t, err) - assert.Equal(t, s0, "1234") - - s1, err := pR.GetOneStringFromPayload(1) - assert.Nil(t, err) - assert.Equal(t, s1, "567890") - - s2, err := pR.GetOneStringFromPayload(2) - assert.Nil(t, err) - assert.Equal(t, s2, "abcdefg") + assert.Equal(t, s[0], "1234") + assert.Equal(t, s[1], "567890") + assert.Equal(t, s[2], "abcdefg") pR.Close() r, err := newEventReader(schemapb.DataType_String, bytes.NewBuffer(wBuf)) assert.Nil(t, err) - s0, err = r.GetOneStringFromPayload(0) + s, err = pR.GetStringFromPayload() assert.Nil(t, err) - assert.Equal(t, s0, "1234") - - s1, err = r.GetOneStringFromPayload(1) - assert.Nil(t, err) - assert.Equal(t, s1, "567890") - - s2, err = r.GetOneStringFromPayload(2) - assert.Nil(t, err) - assert.Equal(t, s2, "abcdefg") + assert.Equal(t, s[0], "1234") + assert.Equal(t, s[1], "567890") + assert.Equal(t, s[2], "abcdefg") r.Close() }) @@ -801,34 +764,22 @@ func TestDropCollectionEvent(t *testing.T) { pR, err := NewPayloadReader(schemapb.DataType_String, pBuf) assert.Nil(t, err) - s0, err := pR.GetOneStringFromPayload(0) + s, err := pR.GetStringFromPayload() assert.Nil(t, err) - assert.Equal(t, s0, "1234") - - s1, err := pR.GetOneStringFromPayload(1) - assert.Nil(t, err) - assert.Equal(t, s1, "567890") - - s2, err := pR.GetOneStringFromPayload(2) - assert.Nil(t, err) - assert.Equal(t, s2, "abcdefg") + assert.Equal(t, s[0], "1234") + assert.Equal(t, s[1], "567890") + assert.Equal(t, s[2], "abcdefg") pR.Close() r, err := newEventReader(schemapb.DataType_String, bytes.NewBuffer(wBuf)) assert.Nil(t, err) - s0, err = r.GetOneStringFromPayload(0) + s, err = r.GetStringFromPayload() assert.Nil(t, err) - assert.Equal(t, s0, "1234") - - s1, err = r.GetOneStringFromPayload(1) - assert.Nil(t, err) - assert.Equal(t, s1, "567890") - - s2, err = r.GetOneStringFromPayload(2) - assert.Nil(t, err) - assert.Equal(t, s2, "abcdefg") + assert.Equal(t, s[0], "1234") + assert.Equal(t, s[1], "567890") + assert.Equal(t, s[2], "abcdefg") r.Close() }) @@ -915,34 +866,22 @@ func TestCreatePartitionEvent(t *testing.T) { pR, err := NewPayloadReader(schemapb.DataType_String, pBuf) assert.Nil(t, err) - s0, err := pR.GetOneStringFromPayload(0) + s, err := pR.GetStringFromPayload() assert.Nil(t, err) - assert.Equal(t, s0, "1234") - - s1, err := pR.GetOneStringFromPayload(1) - assert.Nil(t, err) - assert.Equal(t, s1, "567890") - - s2, err := pR.GetOneStringFromPayload(2) - assert.Nil(t, err) - assert.Equal(t, s2, "abcdefg") + assert.Equal(t, s[0], "1234") + assert.Equal(t, s[1], "567890") + assert.Equal(t, s[2], "abcdefg") pR.Close() r, err := newEventReader(schemapb.DataType_String, bytes.NewBuffer(wBuf)) assert.Nil(t, err) - s0, err = r.GetOneStringFromPayload(0) + s, err = pR.GetStringFromPayload() assert.Nil(t, err) - assert.Equal(t, s0, "1234") - - s1, err = r.GetOneStringFromPayload(1) - assert.Nil(t, err) - assert.Equal(t, s1, "567890") - - s2, err = r.GetOneStringFromPayload(2) - assert.Nil(t, err) - assert.Equal(t, s2, "abcdefg") + assert.Equal(t, s[0], "1234") + assert.Equal(t, s[1], "567890") + assert.Equal(t, s[2], "abcdefg") r.Close() }) @@ -1029,34 +968,22 @@ func TestDropPartitionEvent(t *testing.T) { pR, err := NewPayloadReader(schemapb.DataType_String, pBuf) assert.Nil(t, err) - s0, err := pR.GetOneStringFromPayload(0) + s, err := pR.GetStringFromPayload() assert.Nil(t, err) - assert.Equal(t, s0, "1234") - - s1, err := pR.GetOneStringFromPayload(1) - assert.Nil(t, err) - assert.Equal(t, s1, "567890") - - s2, err := pR.GetOneStringFromPayload(2) - assert.Nil(t, err) - assert.Equal(t, s2, "abcdefg") + assert.Equal(t, s[0], "1234") + assert.Equal(t, s[1], "567890") + assert.Equal(t, s[2], "abcdefg") pR.Close() r, err := newEventReader(schemapb.DataType_String, bytes.NewBuffer(wBuf)) assert.Nil(t, err) - s0, err = r.GetOneStringFromPayload(0) + s, err = pR.GetStringFromPayload() assert.Nil(t, err) - assert.Equal(t, s0, "1234") - - s1, err = r.GetOneStringFromPayload(1) - assert.Nil(t, err) - assert.Equal(t, s1, "567890") - - s2, err = r.GetOneStringFromPayload(2) - assert.Nil(t, err) - assert.Equal(t, s2, "abcdefg") + assert.Equal(t, s[0], "1234") + assert.Equal(t, s[1], "567890") + assert.Equal(t, s[2], "abcdefg") r.Close() }) diff --git a/internal/storage/event_writer.go b/internal/storage/event_writer.go index 43255adf0d..d6d8e8f41d 100644 --- a/internal/storage/event_writer.go +++ b/internal/storage/event_writer.go @@ -20,7 +20,6 @@ import ( "bytes" "encoding/binary" "errors" - "fmt" "io" "github.com/milvus-io/milvus/internal/common" @@ -89,34 +88,6 @@ func (event *descriptorEvent) Write(buffer io.Writer) error { return nil } -func readMagicNumber(buffer io.Reader) (int32, error) { - var magicNumber int32 - if err := binary.Read(buffer, common.Endian, &magicNumber); err != nil { - return -1, err - } - if magicNumber != MagicNumber { - return -1, fmt.Errorf("parse magic number failed, expected: %d, actual: %d", MagicNumber, magicNumber) - } - - return magicNumber, nil -} - -// ReadDescriptorEvent reads a descriptorEvent from buffer -func ReadDescriptorEvent(buffer io.Reader) (*descriptorEvent, error) { - header, err := readDescriptorEventHeader(buffer) - if err != nil { - return nil, err - } - data, err := readDescriptorEventData(buffer) - if err != nil { - return nil, err - } - return &descriptorEvent{ - descriptorEventHeader: *header, - descriptorEventData: *data, - }, nil -} - // EventWriter abstracts event writer type EventWriter interface { PayloadWriterInterface diff --git a/internal/storage/payload.go b/internal/storage/payload.go index f89b00bf9f..b80bd868d7 100644 --- a/internal/storage/payload.go +++ b/internal/storage/payload.go @@ -28,12 +28,9 @@ package storage import "C" import ( "errors" - "fmt" "reflect" "unsafe" - "github.com/milvus-io/milvus/internal/log" - "github.com/milvus-io/milvus/internal/proto/commonpb" "github.com/milvus-io/milvus/internal/proto/schemapb" ) @@ -60,7 +57,7 @@ type PayloadWriterInterface interface { // PayloadReaderInterface abstracts PayloadReader type PayloadReaderInterface interface { - GetDataFromPayload(idx ...int) (interface{}, int, error) + GetDataFromPayload() (interface{}, int, error) GetBoolFromPayload() ([]bool, error) GetByteFromPayload() ([]byte, error) GetInt8FromPayload() ([]int8, error) @@ -69,7 +66,7 @@ type PayloadReaderInterface interface { GetInt64FromPayload() ([]int64, error) GetFloatFromPayload() ([]float32, error) GetDoubleFromPayload() ([]float64, error) - GetOneStringFromPayload(idx int) (string, error) + GetStringFromPayload() ([]string, error) GetBinaryVectorFromPayload() ([]byte, int, error) GetFloatVectorFromPayload() ([]float32, int, error) GetPayloadLengthFromReader() (int, error) @@ -83,12 +80,6 @@ type PayloadWriter struct { colType schemapb.DataType } -// PayloadReader reads data from payload -type PayloadReader struct { - payloadReaderPtr C.CPayloadReader - colType schemapb.DataType -} - // NewPayloadWriter is constructor of PayloadWriter func NewPayloadWriter(colType schemapb.DataType) (*PayloadWriter, error) { w := C.NewPayloadWriter(C.int(colType)) @@ -364,294 +355,3 @@ func (w *PayloadWriter) ReleasePayloadWriter() { func (w *PayloadWriter) Close() { w.ReleasePayloadWriter() } - -func NewPayloadReader(colType schemapb.DataType, buf []byte) (*PayloadReader, error) { - if len(buf) == 0 { - return nil, errors.New("create Payload reader failed, buffer is empty") - } - r := C.NewPayloadReader(C.int(colType), (*C.uint8_t)(unsafe.Pointer(&buf[0])), C.int64_t(len(buf))) - if r == nil { - return nil, errors.New("failed to read parquet from buffer") - } - return &PayloadReader{payloadReaderPtr: r, colType: colType}, nil -} - -// GetDataFromPayload returns data,length from payload, returns err if failed -// Params: -// `idx`: String index -// Return: -// `interface{}`: all types. -// `int`: length, only meaningful to FLOAT/BINARY VECTOR type. -// `error`: error. -func (r *PayloadReader) GetDataFromPayload(idx ...int) (interface{}, int, error) { - switch len(idx) { - case 1: - switch r.colType { - case schemapb.DataType_String, schemapb.DataType_VarChar: - val, err := r.GetOneStringFromPayload(idx[0]) - return val, 0, err - default: - return nil, 0, errors.New("unknown type") - } - case 0: - switch r.colType { - case schemapb.DataType_Bool: - val, err := r.GetBoolFromPayload() - return val, 0, err - case schemapb.DataType_Int8: - val, err := r.GetInt8FromPayload() - return val, 0, err - case schemapb.DataType_Int16: - val, err := r.GetInt16FromPayload() - return val, 0, err - case schemapb.DataType_Int32: - val, err := r.GetInt32FromPayload() - return val, 0, err - case schemapb.DataType_Int64: - val, err := r.GetInt64FromPayload() - return val, 0, err - case schemapb.DataType_Float: - val, err := r.GetFloatFromPayload() - return val, 0, err - case schemapb.DataType_Double: - val, err := r.GetDoubleFromPayload() - return val, 0, err - case schemapb.DataType_BinaryVector: - return r.GetBinaryVectorFromPayload() - case schemapb.DataType_FloatVector: - return r.GetFloatVectorFromPayload() - default: - return nil, 0, errors.New("unknown type") - } - default: - return nil, 0, errors.New("incorrect number of index") - } -} - -// ReleasePayloadReader release payload reader. -func (r *PayloadReader) ReleasePayloadReader() { - C.ReleasePayloadReader(r.payloadReaderPtr) -} - -// GetBoolFromPayload returns bool slice from payload. -func (r *PayloadReader) GetBoolFromPayload() ([]bool, error) { - if r.colType != schemapb.DataType_Bool { - return nil, errors.New("incorrect data type") - } - - var cMsg *C.bool - var cSize C.int - - status := C.GetBoolFromPayload(r.payloadReaderPtr, &cMsg, &cSize) - if err := HandleCStatus(&status, "GetBoolFromPayload failed"); err != nil { - return nil, err - } - - slice := (*[1 << 28]bool)(unsafe.Pointer(cMsg))[:cSize:cSize] - return slice, nil -} - -// GetByteFromPayload returns byte slice from payload -func (r *PayloadReader) GetByteFromPayload() ([]byte, error) { - if r.colType != schemapb.DataType_Int8 { - return nil, errors.New("incorrect data type") - } - - var cMsg *C.int8_t - var cSize C.int - - status := C.GetInt8FromPayload(r.payloadReaderPtr, &cMsg, &cSize) - if err := HandleCStatus(&status, "GetInt8FromPayload failed"); err != nil { - return nil, err - } - - slice := (*[1 << 28]byte)(unsafe.Pointer(cMsg))[:cSize:cSize] - return slice, nil -} - -// GetInt8FromPayload returns int8 slice from payload -func (r *PayloadReader) GetInt8FromPayload() ([]int8, error) { - if r.colType != schemapb.DataType_Int8 { - return nil, errors.New("incorrect data type") - } - - var cMsg *C.int8_t - var cSize C.int - - status := C.GetInt8FromPayload(r.payloadReaderPtr, &cMsg, &cSize) - if err := HandleCStatus(&status, "GetInt8FromPayload failed"); err != nil { - return nil, err - } - - slice := (*[1 << 28]int8)(unsafe.Pointer(cMsg))[:cSize:cSize] - return slice, nil -} - -func (r *PayloadReader) GetInt16FromPayload() ([]int16, error) { - if r.colType != schemapb.DataType_Int16 { - return nil, errors.New("incorrect data type") - } - - var cMsg *C.int16_t - var cSize C.int - - status := C.GetInt16FromPayload(r.payloadReaderPtr, &cMsg, &cSize) - if err := HandleCStatus(&status, "GetInt16FromPayload failed"); err != nil { - return nil, err - } - - slice := (*[1 << 28]int16)(unsafe.Pointer(cMsg))[:cSize:cSize] - return slice, nil -} - -func (r *PayloadReader) GetInt32FromPayload() ([]int32, error) { - if r.colType != schemapb.DataType_Int32 { - return nil, errors.New("incorrect data type") - } - - var cMsg *C.int32_t - var cSize C.int - - status := C.GetInt32FromPayload(r.payloadReaderPtr, &cMsg, &cSize) - if err := HandleCStatus(&status, "GetInt32FromPayload failed"); err != nil { - return nil, err - } - - slice := (*[1 << 28]int32)(unsafe.Pointer(cMsg))[:cSize:cSize] - return slice, nil -} - -func (r *PayloadReader) GetInt64FromPayload() ([]int64, error) { - if r.colType != schemapb.DataType_Int64 { - return nil, errors.New("incorrect data type") - } - - var cMsg *C.int64_t - var cSize C.int - - status := C.GetInt64FromPayload(r.payloadReaderPtr, &cMsg, &cSize) - if err := HandleCStatus(&status, "GetInt64FromPayload failed"); err != nil { - return nil, err - } - - slice := (*[1 << 28]int64)(unsafe.Pointer(cMsg))[:cSize:cSize] - return slice, nil -} - -func (r *PayloadReader) GetFloatFromPayload() ([]float32, error) { - if r.colType != schemapb.DataType_Float { - return nil, errors.New("incorrect data type") - } - - var cMsg *C.float - var cSize C.int - - status := C.GetFloatFromPayload(r.payloadReaderPtr, &cMsg, &cSize) - if err := HandleCStatus(&status, "GetFloatFromPayload failed"); err != nil { - return nil, err - } - - slice := (*[1 << 28]float32)(unsafe.Pointer(cMsg))[:cSize:cSize] - return slice, nil -} - -func (r *PayloadReader) GetDoubleFromPayload() ([]float64, error) { - if r.colType != schemapb.DataType_Double { - return nil, errors.New("incorrect data type") - } - - var cMsg *C.double - var cSize C.int - - status := C.GetDoubleFromPayload(r.payloadReaderPtr, &cMsg, &cSize) - if err := HandleCStatus(&status, "GetDoubleFromPayload failed"); err != nil { - return nil, err - } - - slice := (*[1 << 28]float64)(unsafe.Pointer(cMsg))[:cSize:cSize] - return slice, nil -} - -func (r *PayloadReader) GetOneStringFromPayload(idx int) (string, error) { - if r.colType != schemapb.DataType_String && r.colType != schemapb.DataType_VarChar { - return "", errors.New("incorrect data type") - } - - var cStr *C.char - var cSize C.int - - status := C.GetOneStringFromPayload(r.payloadReaderPtr, C.int(idx), &cStr, &cSize) - if err := HandleCStatus(&status, "GetOneStringFromPayload failed"); err != nil { - return "", err - } - return C.GoStringN(cStr, cSize), nil -} - -// GetBinaryVectorFromPayload returns vector, dimension, error -func (r *PayloadReader) GetBinaryVectorFromPayload() ([]byte, int, error) { - if r.colType != schemapb.DataType_BinaryVector { - return nil, 0, errors.New("incorrect data type") - } - - var cMsg *C.uint8_t - var cDim C.int - var cLen C.int - - status := C.GetBinaryVectorFromPayload(r.payloadReaderPtr, &cMsg, &cDim, &cLen) - if err := HandleCStatus(&status, "GetBinaryVectorFromPayload failed"); err != nil { - return nil, 0, err - } - length := (cDim / 8) * cLen - - slice := (*[1 << 28]byte)(unsafe.Pointer(cMsg))[:length:length] - return slice, int(cDim), nil -} - -// GetFloatVectorFromPayload returns vector, dimension, error -func (r *PayloadReader) GetFloatVectorFromPayload() ([]float32, int, error) { - if r.colType != schemapb.DataType_FloatVector { - return nil, 0, errors.New("incorrect data type") - } - - var cMsg *C.float - var cDim C.int - var cLen C.int - - status := C.GetFloatVectorFromPayload(r.payloadReaderPtr, &cMsg, &cDim, &cLen) - if err := HandleCStatus(&status, "GetFloatVectorFromPayload failed"); err != nil { - return nil, 0, err - } - length := cDim * cLen - - slice := (*[1 << 28]float32)(unsafe.Pointer(cMsg))[:length:length] - return slice, int(cDim), nil -} - -func (r *PayloadReader) GetPayloadLengthFromReader() (int, error) { - length := C.GetPayloadLengthFromReader(r.payloadReaderPtr) - return int(length), nil -} - -// Close closes the payload reader -func (r *PayloadReader) Close() { - r.ReleasePayloadReader() -} - -// HandleCStatus deal with the error returned from CGO -func HandleCStatus(status *C.CStatus, extraInfo string) error { - if status.error_code == 0 { - return nil - } - errorCode := status.error_code - errorName, ok := commonpb.ErrorCode_name[int32(errorCode)] - if !ok { - errorName = "UnknownError" - } - errorMsg := C.GoString(status.error_msg) - defer C.free(unsafe.Pointer(status.error_msg)) - - finalMsg := fmt.Sprintf("[%s] %s", errorName, errorMsg) - logMsg := fmt.Sprintf("%s, C Runtime Exception: %s\n", extraInfo, finalMsg) - log.Warn(logMsg) - return errors.New(finalMsg) -} diff --git a/internal/storage/payload_cgo_test.go b/internal/storage/payload_cgo_test.go new file mode 100644 index 0000000000..6ac0bc3997 --- /dev/null +++ b/internal/storage/payload_cgo_test.go @@ -0,0 +1,898 @@ +// Licensed to the LF AI & Data foundation under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package storage + +import ( + "fmt" + "testing" + + "github.com/milvus-io/milvus/internal/proto/schemapb" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TODO Cgo test can not go through on mac, fix it +func TestPayload_CGO_ReaderandWriter(t *testing.T) { + + t.Run("TestBool", func(t *testing.T) { + w, err := NewPayloadWriter(schemapb.DataType_Bool) + require.Nil(t, err) + require.NotNil(t, w) + + err = w.AddBoolToPayload([]bool{false, false, false, false}) + assert.Nil(t, err) + err = w.AddDataToPayload([]bool{false, false, false, false}) + assert.Nil(t, err) + err = w.FinishPayloadWriter() + assert.Nil(t, err) + + length, err := w.GetPayloadLengthFromWriter() + assert.Nil(t, err) + assert.Equal(t, 8, length) + defer w.ReleasePayloadWriter() + + buffer, err := w.GetPayloadBufferFromWriter() + assert.Nil(t, err) + + r, err := NewPayloadReaderGgo(schemapb.DataType_Bool, buffer) + require.Nil(t, err) + length, err = r.GetPayloadLengthFromReader() + assert.Nil(t, err) + assert.Equal(t, length, 8) + bools, err := r.GetBoolFromPayload() + assert.Nil(t, err) + assert.ElementsMatch(t, []bool{false, false, false, false, false, false, false, false}, bools) + ibools, _, err := r.GetDataFromPayload() + bools = ibools.([]bool) + assert.Nil(t, err) + assert.ElementsMatch(t, []bool{false, false, false, false, false, false, false, false}, bools) + defer r.ReleasePayloadReader() + + }) + + t.Run("TestInt8", func(t *testing.T) { + w, err := NewPayloadWriter(schemapb.DataType_Int8) + require.Nil(t, err) + require.NotNil(t, w) + + err = w.AddInt8ToPayload([]int8{1, 2, 3}) + assert.Nil(t, err) + err = w.AddDataToPayload([]int8{4, 5, 6}) + assert.Nil(t, err) + err = w.FinishPayloadWriter() + assert.Nil(t, err) + + length, err := w.GetPayloadLengthFromWriter() + assert.Nil(t, err) + assert.Equal(t, 6, length) + defer w.ReleasePayloadWriter() + + buffer, err := w.GetPayloadBufferFromWriter() + assert.Nil(t, err) + + r, err := NewPayloadReaderGgo(schemapb.DataType_Int8, buffer) + require.Nil(t, err) + length, err = r.GetPayloadLengthFromReader() + assert.Nil(t, err) + assert.Equal(t, length, 6) + + int8s, err := r.GetInt8FromPayload() + assert.Nil(t, err) + assert.ElementsMatch(t, []int8{1, 2, 3, 4, 5, 6}, int8s) + + iint8s, _, err := r.GetDataFromPayload() + int8s = iint8s.([]int8) + assert.Nil(t, err) + + assert.ElementsMatch(t, []int8{1, 2, 3, 4, 5, 6}, int8s) + defer r.ReleasePayloadReader() + }) + + t.Run("TestInt16", func(t *testing.T) { + w, err := NewPayloadWriter(schemapb.DataType_Int16) + require.Nil(t, err) + require.NotNil(t, w) + + err = w.AddInt16ToPayload([]int16{1, 2, 3}) + assert.Nil(t, err) + err = w.AddDataToPayload([]int16{1, 2, 3}) + assert.Nil(t, err) + err = w.FinishPayloadWriter() + assert.Nil(t, err) + + length, err := w.GetPayloadLengthFromWriter() + assert.Nil(t, err) + assert.Equal(t, 6, length) + defer w.ReleasePayloadWriter() + + buffer, err := w.GetPayloadBufferFromWriter() + assert.Nil(t, err) + + r, err := NewPayloadReaderGgo(schemapb.DataType_Int16, buffer) + require.Nil(t, err) + length, err = r.GetPayloadLengthFromReader() + assert.Nil(t, err) + assert.Equal(t, length, 6) + int16s, err := r.GetInt16FromPayload() + assert.Nil(t, err) + assert.ElementsMatch(t, []int16{1, 2, 3, 1, 2, 3}, int16s) + + iint16s, _, err := r.GetDataFromPayload() + int16s = iint16s.([]int16) + assert.Nil(t, err) + assert.ElementsMatch(t, []int16{1, 2, 3, 1, 2, 3}, int16s) + defer r.ReleasePayloadReader() + }) + + t.Run("TestInt32", func(t *testing.T) { + w, err := NewPayloadWriter(schemapb.DataType_Int32) + require.Nil(t, err) + require.NotNil(t, w) + + err = w.AddInt32ToPayload([]int32{1, 2, 3}) + assert.Nil(t, err) + err = w.AddDataToPayload([]int32{1, 2, 3}) + assert.Nil(t, err) + err = w.FinishPayloadWriter() + assert.Nil(t, err) + + length, err := w.GetPayloadLengthFromWriter() + assert.Nil(t, err) + assert.Equal(t, 6, length) + defer w.ReleasePayloadWriter() + + buffer, err := w.GetPayloadBufferFromWriter() + assert.Nil(t, err) + + r, err := NewPayloadReaderGgo(schemapb.DataType_Int32, buffer) + require.Nil(t, err) + length, err = r.GetPayloadLengthFromReader() + assert.Nil(t, err) + assert.Equal(t, length, 6) + + int32s, err := r.GetInt32FromPayload() + assert.Nil(t, err) + assert.ElementsMatch(t, []int32{1, 2, 3, 1, 2, 3}, int32s) + + iint32s, _, err := r.GetDataFromPayload() + int32s = iint32s.([]int32) + assert.Nil(t, err) + assert.ElementsMatch(t, []int32{1, 2, 3, 1, 2, 3}, int32s) + defer r.ReleasePayloadReader() + }) + + t.Run("TestInt64", func(t *testing.T) { + w, err := NewPayloadWriter(schemapb.DataType_Int64) + require.Nil(t, err) + require.NotNil(t, w) + + err = w.AddInt64ToPayload([]int64{1, 2, 3}) + assert.Nil(t, err) + err = w.AddDataToPayload([]int64{1, 2, 3}) + assert.Nil(t, err) + err = w.FinishPayloadWriter() + assert.Nil(t, err) + + length, err := w.GetPayloadLengthFromWriter() + assert.Nil(t, err) + assert.Equal(t, 6, length) + defer w.ReleasePayloadWriter() + + buffer, err := w.GetPayloadBufferFromWriter() + assert.Nil(t, err) + + r, err := NewPayloadReaderGgo(schemapb.DataType_Int64, buffer) + require.Nil(t, err) + length, err = r.GetPayloadLengthFromReader() + assert.Nil(t, err) + assert.Equal(t, length, 6) + + int64s, err := r.GetInt64FromPayload() + assert.Nil(t, err) + assert.ElementsMatch(t, []int64{1, 2, 3, 1, 2, 3}, int64s) + + iint64s, _, err := r.GetDataFromPayload() + int64s = iint64s.([]int64) + assert.Nil(t, err) + assert.ElementsMatch(t, []int64{1, 2, 3, 1, 2, 3}, int64s) + defer r.ReleasePayloadReader() + }) + + t.Run("TestFloat32", func(t *testing.T) { + w, err := NewPayloadWriter(schemapb.DataType_Float) + require.Nil(t, err) + require.NotNil(t, w) + + err = w.AddFloatToPayload([]float32{1.0, 2.0, 3.0}) + assert.Nil(t, err) + err = w.AddDataToPayload([]float32{1.0, 2.0, 3.0}) + assert.Nil(t, err) + err = w.FinishPayloadWriter() + assert.Nil(t, err) + + length, err := w.GetPayloadLengthFromWriter() + assert.Nil(t, err) + assert.Equal(t, 6, length) + defer w.ReleasePayloadWriter() + + buffer, err := w.GetPayloadBufferFromWriter() + assert.Nil(t, err) + + r, err := NewPayloadReaderGgo(schemapb.DataType_Float, buffer) + require.Nil(t, err) + length, err = r.GetPayloadLengthFromReader() + assert.Nil(t, err) + assert.Equal(t, length, 6) + + float32s, err := r.GetFloatFromPayload() + assert.Nil(t, err) + assert.ElementsMatch(t, []float32{1.0, 2.0, 3.0, 1.0, 2.0, 3.0}, float32s) + + ifloat32s, _, err := r.GetDataFromPayload() + float32s = ifloat32s.([]float32) + assert.Nil(t, err) + assert.ElementsMatch(t, []float32{1.0, 2.0, 3.0, 1.0, 2.0, 3.0}, float32s) + defer r.ReleasePayloadReader() + }) + + t.Run("TestDouble", func(t *testing.T) { + w, err := NewPayloadWriter(schemapb.DataType_Double) + require.Nil(t, err) + require.NotNil(t, w) + + err = w.AddDoubleToPayload([]float64{1.0, 2.0, 3.0}) + assert.Nil(t, err) + err = w.AddDataToPayload([]float64{1.0, 2.0, 3.0}) + assert.Nil(t, err) + err = w.FinishPayloadWriter() + assert.Nil(t, err) + + length, err := w.GetPayloadLengthFromWriter() + assert.Nil(t, err) + assert.Equal(t, 6, length) + defer w.ReleasePayloadWriter() + + buffer, err := w.GetPayloadBufferFromWriter() + assert.Nil(t, err) + + r, err := NewPayloadReaderGgo(schemapb.DataType_Double, buffer) + require.Nil(t, err) + length, err = r.GetPayloadLengthFromReader() + assert.Nil(t, err) + assert.Equal(t, length, 6) + + float64s, err := r.GetDoubleFromPayload() + assert.Nil(t, err) + assert.ElementsMatch(t, []float64{1.0, 2.0, 3.0, 1.0, 2.0, 3.0}, float64s) + + ifloat64s, _, err := r.GetDataFromPayload() + float64s = ifloat64s.([]float64) + assert.Nil(t, err) + assert.ElementsMatch(t, []float64{1.0, 2.0, 3.0, 1.0, 2.0, 3.0}, float64s) + defer r.ReleasePayloadReader() + }) + + t.Run("TestAddString", func(t *testing.T) { + w, err := NewPayloadWriter(schemapb.DataType_String) + require.Nil(t, err) + require.NotNil(t, w) + + err = w.AddOneStringToPayload("hello0") + assert.Nil(t, err) + err = w.AddOneStringToPayload("hello1") + assert.Nil(t, err) + err = w.AddOneStringToPayload("hello2") + assert.Nil(t, err) + err = w.AddDataToPayload("hello3") + assert.Nil(t, err) + err = w.FinishPayloadWriter() + assert.Nil(t, err) + length, err := w.GetPayloadLengthFromWriter() + assert.Nil(t, err) + assert.Equal(t, length, 4) + buffer, err := w.GetPayloadBufferFromWriter() + assert.Nil(t, err) + + r, err := NewPayloadReaderGgo(schemapb.DataType_String, buffer) + assert.Nil(t, err) + length, err = r.GetPayloadLengthFromReader() + assert.Nil(t, err) + assert.Equal(t, length, 4) + + str, err := r.GetStringFromPayload() + assert.Nil(t, err) + + assert.Equal(t, str[0], "hello0") + assert.Equal(t, str[1], "hello1") + assert.Equal(t, str[2], "hello2") + assert.Equal(t, str[3], "hello3") + + istr, _, err := r.GetDataFromPayload() + strArray := istr.([]string) + assert.Nil(t, err) + assert.Equal(t, strArray[0], "hello0") + assert.Equal(t, strArray[1], "hello1") + assert.Equal(t, strArray[2], "hello2") + assert.Equal(t, strArray[3], "hello3") + r.ReleasePayloadReader() + w.ReleasePayloadWriter() + }) + + t.Run("TestBinaryVector", func(t *testing.T) { + w, err := NewPayloadWriter(schemapb.DataType_BinaryVector) + require.Nil(t, err) + require.NotNil(t, w) + + in := make([]byte, 16) + for i := 0; i < 16; i++ { + in[i] = 1 + } + in2 := make([]byte, 8) + for i := 0; i < 8; i++ { + in2[i] = 1 + } + + err = w.AddBinaryVectorToPayload(in, 8) + assert.Nil(t, err) + err = w.AddDataToPayload(in2, 8) + assert.Nil(t, err) + err = w.FinishPayloadWriter() + assert.Nil(t, err) + + length, err := w.GetPayloadLengthFromWriter() + assert.Nil(t, err) + assert.Equal(t, 24, length) + defer w.ReleasePayloadWriter() + + buffer, err := w.GetPayloadBufferFromWriter() + assert.Nil(t, err) + + r, err := NewPayloadReaderGgo(schemapb.DataType_BinaryVector, buffer) + require.Nil(t, err) + length, err = r.GetPayloadLengthFromReader() + assert.Nil(t, err) + assert.Equal(t, length, 24) + + binVecs, dim, err := r.GetBinaryVectorFromPayload() + assert.Nil(t, err) + assert.Equal(t, 8, dim) + assert.Equal(t, 24, len(binVecs)) + fmt.Println(binVecs) + + ibinVecs, dim, err := r.GetDataFromPayload() + assert.Nil(t, err) + binVecs = ibinVecs.([]byte) + assert.Equal(t, 8, dim) + assert.Equal(t, 24, len(binVecs)) + defer r.ReleasePayloadReader() + }) + + t.Run("TestFloatVector", func(t *testing.T) { + w, err := NewPayloadWriter(schemapb.DataType_FloatVector) + require.Nil(t, err) + require.NotNil(t, w) + + err = w.AddFloatVectorToPayload([]float32{1.0, 2.0}, 1) + assert.Nil(t, err) + err = w.AddDataToPayload([]float32{3.0, 4.0}, 1) + assert.Nil(t, err) + err = w.FinishPayloadWriter() + assert.Nil(t, err) + + length, err := w.GetPayloadLengthFromWriter() + assert.Nil(t, err) + assert.Equal(t, 4, length) + defer w.ReleasePayloadWriter() + + buffer, err := w.GetPayloadBufferFromWriter() + assert.Nil(t, err) + + r, err := NewPayloadReaderGgo(schemapb.DataType_FloatVector, buffer) + require.Nil(t, err) + length, err = r.GetPayloadLengthFromReader() + assert.Nil(t, err) + assert.Equal(t, length, 4) + + floatVecs, dim, err := r.GetFloatVectorFromPayload() + assert.Nil(t, err) + assert.Equal(t, 1, dim) + assert.Equal(t, 4, len(floatVecs)) + assert.ElementsMatch(t, []float32{1.0, 2.0, 3.0, 4.0}, floatVecs) + + ifloatVecs, dim, err := r.GetDataFromPayload() + assert.Nil(t, err) + floatVecs = ifloatVecs.([]float32) + assert.Equal(t, 1, dim) + assert.Equal(t, 4, len(floatVecs)) + assert.ElementsMatch(t, []float32{1.0, 2.0, 3.0, 4.0}, floatVecs) + defer r.ReleasePayloadReader() + }) + + t.Run("TestAddDataToPayload", func(t *testing.T) { + w, err := NewPayloadWriter(schemapb.DataType_Bool) + w.colType = 999 + require.Nil(t, err) + require.NotNil(t, w) + + err = w.AddDataToPayload([]bool{false, false, false, false}) + assert.NotNil(t, err) + + err = w.AddDataToPayload([]bool{false, false, false, false}, 0) + assert.NotNil(t, err) + + err = w.AddDataToPayload([]bool{false, false, false, false}, 0, 0) + assert.NotNil(t, err) + + err = w.AddBoolToPayload([]bool{}) + assert.NotNil(t, err) + err = w.FinishPayloadWriter() + assert.Nil(t, err) + err = w.AddBoolToPayload([]bool{false}) + assert.NotNil(t, err) + }) + + t.Run("TestAddBoolAfterFinish", func(t *testing.T) { + w, err := NewPayloadWriter(schemapb.DataType_Bool) + require.Nil(t, err) + require.NotNil(t, w) + + _, err = w.GetPayloadBufferFromWriter() + assert.NotNil(t, err) + + err = w.AddBoolToPayload([]bool{}) + assert.NotNil(t, err) + err = w.FinishPayloadWriter() + assert.Nil(t, err) + err = w.AddBoolToPayload([]bool{false}) + assert.NotNil(t, err) + }) + + t.Run("TestAddInt8AfterFinish", func(t *testing.T) { + w, err := NewPayloadWriter(schemapb.DataType_Int8) + require.Nil(t, err) + require.NotNil(t, w) + defer w.Close() + + _, err = w.GetPayloadBufferFromWriter() + assert.NotNil(t, err) + + err = w.AddInt8ToPayload([]int8{}) + assert.NotNil(t, err) + err = w.FinishPayloadWriter() + assert.Nil(t, err) + err = w.AddInt8ToPayload([]int8{0}) + assert.NotNil(t, err) + }) + t.Run("TestAddInt16AfterFinish", func(t *testing.T) { + w, err := NewPayloadWriter(schemapb.DataType_Int16) + require.Nil(t, err) + require.NotNil(t, w) + defer w.Close() + + _, err = w.GetPayloadBufferFromWriter() + assert.NotNil(t, err) + + err = w.AddInt16ToPayload([]int16{}) + assert.NotNil(t, err) + err = w.FinishPayloadWriter() + assert.Nil(t, err) + err = w.AddInt16ToPayload([]int16{0}) + assert.NotNil(t, err) + }) + t.Run("TestAddInt32AfterFinish", func(t *testing.T) { + w, err := NewPayloadWriter(schemapb.DataType_Int32) + require.Nil(t, err) + require.NotNil(t, w) + defer w.Close() + + _, err = w.GetPayloadBufferFromWriter() + assert.NotNil(t, err) + + err = w.AddInt32ToPayload([]int32{}) + assert.NotNil(t, err) + err = w.FinishPayloadWriter() + assert.Nil(t, err) + err = w.AddInt32ToPayload([]int32{0}) + assert.NotNil(t, err) + }) + t.Run("TestAddInt64AfterFinish", func(t *testing.T) { + w, err := NewPayloadWriter(schemapb.DataType_Int64) + require.Nil(t, err) + require.NotNil(t, w) + defer w.Close() + + _, err = w.GetPayloadBufferFromWriter() + assert.NotNil(t, err) + + err = w.AddInt64ToPayload([]int64{}) + assert.NotNil(t, err) + err = w.FinishPayloadWriter() + assert.Nil(t, err) + err = w.AddInt64ToPayload([]int64{0}) + assert.NotNil(t, err) + }) + t.Run("TestAddFloatAfterFinish", func(t *testing.T) { + w, err := NewPayloadWriter(schemapb.DataType_Float) + require.Nil(t, err) + require.NotNil(t, w) + defer w.Close() + + _, err = w.GetPayloadBufferFromWriter() + assert.NotNil(t, err) + + err = w.AddFloatToPayload([]float32{}) + assert.NotNil(t, err) + err = w.FinishPayloadWriter() + assert.Nil(t, err) + err = w.AddFloatToPayload([]float32{0.0}) + assert.NotNil(t, err) + }) + t.Run("TestAddDoubleAfterFinish", func(t *testing.T) { + w, err := NewPayloadWriter(schemapb.DataType_Double) + require.Nil(t, err) + require.NotNil(t, w) + defer w.Close() + + _, err = w.GetPayloadBufferFromWriter() + assert.NotNil(t, err) + + err = w.AddDoubleToPayload([]float64{}) + assert.NotNil(t, err) + err = w.FinishPayloadWriter() + assert.Nil(t, err) + err = w.AddDoubleToPayload([]float64{0.0}) + assert.NotNil(t, err) + }) + t.Run("TestAddOneStringAfterFinish", func(t *testing.T) { + w, err := NewPayloadWriter(schemapb.DataType_String) + require.Nil(t, err) + require.NotNil(t, w) + defer w.Close() + + _, err = w.GetPayloadBufferFromWriter() + assert.NotNil(t, err) + + err = w.AddOneStringToPayload("") + assert.NotNil(t, err) + err = w.FinishPayloadWriter() + assert.Nil(t, err) + err = w.AddOneStringToPayload("c") + assert.NotNil(t, err) + }) + t.Run("TestAddBinVectorAfterFinish", func(t *testing.T) { + w, err := NewPayloadWriter(schemapb.DataType_BinaryVector) + require.Nil(t, err) + require.NotNil(t, w) + defer w.Close() + + _, err = w.GetPayloadBufferFromWriter() + assert.NotNil(t, err) + + err = w.FinishPayloadWriter() + assert.NotNil(t, err) + + err = w.AddBinaryVectorToPayload([]byte{}, 8) + assert.NotNil(t, err) + err = w.AddBinaryVectorToPayload([]byte{1}, 0) + assert.NotNil(t, err) + + err = w.AddBinaryVectorToPayload([]byte{1, 0, 0, 0, 0, 0, 0, 0}, 8) + assert.Nil(t, err) + err = w.FinishPayloadWriter() + assert.Nil(t, err) + err = w.AddBinaryVectorToPayload([]byte{1, 0, 0, 0, 0, 0, 0, 0}, 8) + assert.NotNil(t, err) + }) + t.Run("TestAddFloatVectorAfterFinish", func(t *testing.T) { + w, err := NewPayloadWriter(schemapb.DataType_FloatVector) + require.Nil(t, err) + require.NotNil(t, w) + defer w.Close() + + err = w.FinishPayloadWriter() + assert.NotNil(t, err) + + err = w.AddFloatVectorToPayload([]float32{}, 8) + assert.NotNil(t, err) + err = w.AddFloatVectorToPayload([]float32{1.0}, 0) + assert.NotNil(t, err) + + err = w.AddFloatVectorToPayload([]float32{1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, 8) + assert.Nil(t, err) + err = w.FinishPayloadWriter() + assert.Nil(t, err) + err = w.AddFloatVectorToPayload([]float32{1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, 8) + assert.NotNil(t, err) + }) + + t.Run("TestNewReadError", func(t *testing.T) { + buffer := []byte{0} + r, err := NewPayloadReaderGgo(999, buffer) + assert.NotNil(t, err) + assert.Nil(t, r) + }) + t.Run("TestGetDataError", func(t *testing.T) { + r := PayloadReader{} + r.colType = 999 + + _, _, err := r.GetDataFromPayload() + assert.NotNil(t, err) + }) + t.Run("TestGetBoolError", func(t *testing.T) { + w, err := NewPayloadWriter(schemapb.DataType_Int8) + require.Nil(t, err) + require.NotNil(t, w) + + err = w.AddInt8ToPayload([]int8{1, 2, 3}) + assert.Nil(t, err) + + err = w.FinishPayloadWriter() + assert.Nil(t, err) + + buffer, err := w.GetPayloadBufferFromWriter() + assert.Nil(t, err) + + r, err := NewPayloadReaderGgo(schemapb.DataType_Bool, buffer) + assert.Nil(t, err) + + _, err = r.GetBoolFromPayload() + assert.NotNil(t, err) + + r.colType = 999 + _, err = r.GetBoolFromPayload() + assert.NotNil(t, err) + }) + t.Run("TestGetInt8Error", func(t *testing.T) { + w, err := NewPayloadWriter(schemapb.DataType_Bool) + require.Nil(t, err) + require.NotNil(t, w) + + err = w.AddBoolToPayload([]bool{false, true, true}) + assert.Nil(t, err) + + err = w.FinishPayloadWriter() + assert.Nil(t, err) + + buffer, err := w.GetPayloadBufferFromWriter() + assert.Nil(t, err) + + r, err := NewPayloadReaderGgo(schemapb.DataType_Int8, buffer) + assert.Nil(t, err) + + _, err = r.GetInt8FromPayload() + assert.NotNil(t, err) + + r.colType = 999 + _, err = r.GetInt8FromPayload() + assert.NotNil(t, err) + }) + t.Run("TestGetInt16Error", func(t *testing.T) { + w, err := NewPayloadWriter(schemapb.DataType_Bool) + require.Nil(t, err) + require.NotNil(t, w) + + err = w.AddBoolToPayload([]bool{false, true, true}) + assert.Nil(t, err) + + err = w.FinishPayloadWriter() + assert.Nil(t, err) + + buffer, err := w.GetPayloadBufferFromWriter() + assert.Nil(t, err) + + r, err := NewPayloadReaderGgo(schemapb.DataType_Int16, buffer) + assert.Nil(t, err) + + _, err = r.GetInt16FromPayload() + assert.NotNil(t, err) + + r.colType = 999 + _, err = r.GetInt16FromPayload() + assert.NotNil(t, err) + }) + t.Run("TestGetInt32Error", func(t *testing.T) { + w, err := NewPayloadWriter(schemapb.DataType_Bool) + require.Nil(t, err) + require.NotNil(t, w) + + err = w.AddBoolToPayload([]bool{false, true, true}) + assert.Nil(t, err) + + err = w.FinishPayloadWriter() + assert.Nil(t, err) + + buffer, err := w.GetPayloadBufferFromWriter() + assert.Nil(t, err) + + r, err := NewPayloadReaderGgo(schemapb.DataType_Int32, buffer) + assert.Nil(t, err) + + _, err = r.GetInt32FromPayload() + assert.NotNil(t, err) + + r.colType = 999 + _, err = r.GetInt32FromPayload() + assert.NotNil(t, err) + }) + t.Run("TestGetInt64Error", func(t *testing.T) { + w, err := NewPayloadWriter(schemapb.DataType_Bool) + require.Nil(t, err) + require.NotNil(t, w) + + err = w.AddBoolToPayload([]bool{false, true, true}) + assert.Nil(t, err) + + err = w.FinishPayloadWriter() + assert.Nil(t, err) + + buffer, err := w.GetPayloadBufferFromWriter() + assert.Nil(t, err) + + r, err := NewPayloadReaderGgo(schemapb.DataType_Int64, buffer) + assert.Nil(t, err) + + _, err = r.GetInt64FromPayload() + assert.NotNil(t, err) + + r.colType = 999 + _, err = r.GetInt64FromPayload() + assert.NotNil(t, err) + }) + t.Run("TestGetFloatError", func(t *testing.T) { + w, err := NewPayloadWriter(schemapb.DataType_Bool) + require.Nil(t, err) + require.NotNil(t, w) + + err = w.AddBoolToPayload([]bool{false, true, true}) + assert.Nil(t, err) + + err = w.FinishPayloadWriter() + assert.Nil(t, err) + + buffer, err := w.GetPayloadBufferFromWriter() + assert.Nil(t, err) + + r, err := NewPayloadReaderGgo(schemapb.DataType_Float, buffer) + assert.Nil(t, err) + + _, err = r.GetFloatFromPayload() + assert.NotNil(t, err) + + r.colType = 999 + _, err = r.GetFloatFromPayload() + assert.NotNil(t, err) + }) + t.Run("TestGetDoubleError", func(t *testing.T) { + w, err := NewPayloadWriter(schemapb.DataType_Bool) + require.Nil(t, err) + require.NotNil(t, w) + + err = w.AddBoolToPayload([]bool{false, true, true}) + assert.Nil(t, err) + + err = w.FinishPayloadWriter() + assert.Nil(t, err) + + buffer, err := w.GetPayloadBufferFromWriter() + assert.Nil(t, err) + + r, err := NewPayloadReaderGgo(schemapb.DataType_Double, buffer) + assert.Nil(t, err) + + _, err = r.GetDoubleFromPayload() + assert.NotNil(t, err) + + r.colType = 999 + _, err = r.GetDoubleFromPayload() + assert.NotNil(t, err) + }) + t.Run("TestGetStringError", func(t *testing.T) { + w, err := NewPayloadWriter(schemapb.DataType_Bool) + require.Nil(t, err) + require.NotNil(t, w) + + err = w.AddBoolToPayload([]bool{false, true, true}) + assert.Nil(t, err) + + err = w.FinishPayloadWriter() + assert.Nil(t, err) + + buffer, err := w.GetPayloadBufferFromWriter() + assert.Nil(t, err) + + r, err := NewPayloadReaderGgo(schemapb.DataType_String, buffer) + assert.Nil(t, err) + + _, err = r.GetStringFromPayload() + assert.NotNil(t, err) + + r.colType = 999 + _, err = r.GetStringFromPayload() + assert.NotNil(t, err) + }) + t.Run("TestGetBinaryVectorError", func(t *testing.T) { + w, err := NewPayloadWriter(schemapb.DataType_Bool) + require.Nil(t, err) + require.NotNil(t, w) + + err = w.AddBoolToPayload([]bool{false, true, true}) + assert.Nil(t, err) + + err = w.FinishPayloadWriter() + assert.Nil(t, err) + + buffer, err := w.GetPayloadBufferFromWriter() + assert.Nil(t, err) + + r, err := NewPayloadReaderGgo(schemapb.DataType_BinaryVector, buffer) + assert.Nil(t, err) + + _, _, err = r.GetBinaryVectorFromPayload() + assert.NotNil(t, err) + + r.colType = 999 + _, _, err = r.GetBinaryVectorFromPayload() + assert.NotNil(t, err) + }) + t.Run("TestGetFloatVectorError", func(t *testing.T) { + w, err := NewPayloadWriter(schemapb.DataType_Bool) + require.Nil(t, err) + require.NotNil(t, w) + + err = w.AddBoolToPayload([]bool{false, true, true}) + assert.Nil(t, err) + + err = w.FinishPayloadWriter() + assert.Nil(t, err) + + buffer, err := w.GetPayloadBufferFromWriter() + assert.Nil(t, err) + + r, err := NewPayloadReaderGgo(schemapb.DataType_FloatVector, buffer) + assert.Nil(t, err) + + _, _, err = r.GetFloatVectorFromPayload() + assert.NotNil(t, err) + + r.colType = 999 + _, _, err = r.GetFloatVectorFromPayload() + assert.NotNil(t, err) + }) + + t.Run("TestWriteLargeSizeData", func(t *testing.T) { + t.Skip("Large data skip for online ut") + size := 1 << 29 // 512M + var vec []float32 + for i := 0; i < size/4; i++ { + vec = append(vec, 1) + } + + w, err := NewPayloadWriter(schemapb.DataType_FloatVector) + assert.Nil(t, err) + + err = w.AddFloatVectorToPayload(vec, 128) + assert.Nil(t, err) + + err = w.FinishPayloadWriter() + assert.Nil(t, err) + + _, err = w.GetPayloadBufferFromWriter() + assert.Nil(t, err) + + w.ReleasePayloadWriter() + }) +} diff --git a/internal/storage/payload_reader.go b/internal/storage/payload_reader.go new file mode 100644 index 0000000000..9ba0813121 --- /dev/null +++ b/internal/storage/payload_reader.go @@ -0,0 +1,441 @@ +package storage + +import ( + "bytes" + "errors" + "fmt" + "reflect" + + "github.com/apache/arrow/go/v8/arrow" + "github.com/apache/arrow/go/v8/parquet" + "github.com/apache/arrow/go/v8/parquet/file" + "github.com/milvus-io/milvus/internal/proto/schemapb" +) + +// PayloadReader reads data from payload +type PayloadReader struct { + reader *file.Reader + colType schemapb.DataType + numRows int64 +} + +func NewPayloadReader(colType schemapb.DataType, buf []byte) (*PayloadReader, error) { + if len(buf) == 0 { + return nil, errors.New("create Payload reader failed, buffer is empty") + } + parquetReader, err := file.NewParquetReader(bytes.NewReader(buf)) + if err != nil { + return nil, err + } + return &PayloadReader{reader: parquetReader, colType: colType, numRows: parquetReader.NumRows()}, nil +} + +// GetDataFromPayload returns data,length from payload, returns err if failed +// Return: +// `interface{}`: all types. +// `int`: dim, only meaningful to FLOAT/BINARY VECTOR type. +// `error`: error. +func (r *PayloadReader) GetDataFromPayload() (interface{}, int, error) { + switch r.colType { + case schemapb.DataType_Bool: + val, err := r.GetBoolFromPayload() + return val, 0, err + case schemapb.DataType_Int8: + val, err := r.GetInt8FromPayload() + return val, 0, err + case schemapb.DataType_Int16: + val, err := r.GetInt16FromPayload() + return val, 0, err + case schemapb.DataType_Int32: + val, err := r.GetInt32FromPayload() + return val, 0, err + case schemapb.DataType_Int64: + val, err := r.GetInt64FromPayload() + return val, 0, err + case schemapb.DataType_Float: + val, err := r.GetFloatFromPayload() + return val, 0, err + case schemapb.DataType_Double: + val, err := r.GetDoubleFromPayload() + return val, 0, err + case schemapb.DataType_BinaryVector: + return r.GetBinaryVectorFromPayload() + case schemapb.DataType_FloatVector: + return r.GetFloatVectorFromPayload() + case schemapb.DataType_String: + val, err := r.GetStringFromPayload() + return val, 0, err + default: + return nil, 0, errors.New("unknown type") + } +} + +// ReleasePayloadReader release payload reader. +func (r *PayloadReader) ReleasePayloadReader() { + r.Close() +} + +// GetBoolFromPayload returns bool slice from payload. +func (r *PayloadReader) GetBoolFromPayload() ([]bool, error) { + if r.colType != schemapb.DataType_Bool { + return nil, fmt.Errorf("failed to get bool from datatype %v", r.colType.String()) + } + dumper, err := r.createDumper() + if err != nil { + return nil, err + } + ret := make([]bool, r.numRows) + var i int64 + for i = 0; i < r.numRows; i++ { + v, hasValue := dumper.Next() + if !hasValue { + return nil, fmt.Errorf("unmatched row number: expect %v, actual %v", r.numRows, i) + } + ret[i] = v.(bool) + } + return ret, nil +} + +// GetByteFromPayload returns byte slice from payload +func (r *PayloadReader) GetByteFromPayload() ([]byte, error) { + if r.colType != schemapb.DataType_Int8 { + return nil, fmt.Errorf("failed to get byte from datatype %v", r.colType.String()) + } + dumper, err := r.createDumper() + if err != nil { + return nil, err + } + ret := make([]byte, r.numRows) + var i int64 + for i = 0; i < r.numRows; i++ { + v, hasValue := dumper.Next() + if !hasValue { + return nil, fmt.Errorf("unmatched row number: expect %v, actual %v", r.numRows, i) + } + ret[i] = byte(v.(int32)) + } + return ret, nil +} + +// GetInt8FromPayload returns int8 slice from payload +func (r *PayloadReader) GetInt8FromPayload() ([]int8, error) { + if r.colType != schemapb.DataType_Int8 { + return nil, fmt.Errorf("failed to get int8 from datatype %v", r.colType.String()) + } + dumper, err := r.createDumper() + if err != nil { + return nil, err + } + ret := make([]int8, r.numRows) + var i int64 + for i = 0; i < r.numRows; i++ { + v, hasValue := dumper.Next() + if !hasValue { + return nil, fmt.Errorf("unmatched row number: expect %v, actual %v", r.numRows, i) + } + // need to trasfer because parquet didn't support int8 + ret[i] = int8(v.(int32)) + } + return ret, nil +} + +func (r *PayloadReader) GetInt16FromPayload() ([]int16, error) { + if r.colType != schemapb.DataType_Int16 { + return nil, fmt.Errorf("failed to get int16 from datatype %v", r.colType.String()) + } + dumper, err := r.createDumper() + if err != nil { + return nil, err + } + ret := make([]int16, r.numRows) + var i int64 + for i = 0; i < r.numRows; i++ { + v, hasValue := dumper.Next() + if !hasValue { + return nil, fmt.Errorf("unmatched row number: expect %v, actual %v", r.numRows, i) + } + // need to trasfer because parquet didn't support int16 + ret[i] = int16(v.(int32)) + } + return ret, nil +} + +func (r *PayloadReader) GetInt32FromPayload() ([]int32, error) { + if r.colType != schemapb.DataType_Int32 { + return nil, fmt.Errorf("failed to get int32 from datatype %v", r.colType.String()) + } + dumper, err := r.createDumper() + if err != nil { + return nil, err + } + ret := make([]int32, r.numRows) + var i int64 + for i = 0; i < r.numRows; i++ { + v, hasValue := dumper.Next() + if !hasValue { + return nil, fmt.Errorf("unmatched row number: expect %v, actual %v", r.numRows, i) + } + ret[i] = v.(int32) + } + return ret, nil +} + +func (r *PayloadReader) GetInt64FromPayload() ([]int64, error) { + if r.colType != schemapb.DataType_Int64 { + return nil, fmt.Errorf("failed to get int64 from datatype %v", r.colType.String()) + } + dumper, err := r.createDumper() + if err != nil { + return nil, err + } + ret := make([]int64, r.numRows) + var i int64 + for i = 0; i < r.numRows; i++ { + v, hasValue := dumper.Next() + if !hasValue { + return nil, fmt.Errorf("unmatched row number: expect %v, actual %v", r.numRows, i) + } + ret[i] = v.(int64) + } + return ret, nil +} + +func (r *PayloadReader) GetFloatFromPayload() ([]float32, error) { + if r.colType != schemapb.DataType_Float { + return nil, fmt.Errorf("failed to get float32 from datatype %v", r.colType.String()) + } + dumper, err := r.createDumper() + if err != nil { + return nil, err + } + ret := make([]float32, r.numRows) + var i int64 + for i = 0; i < r.numRows; i++ { + v, hasValue := dumper.Next() + if !hasValue { + return nil, fmt.Errorf("unmatched row number: expect %v, actual %v", r.numRows, i) + } + ret[i] = v.(float32) + } + return ret, nil +} + +func (r *PayloadReader) GetDoubleFromPayload() ([]float64, error) { + if r.colType != schemapb.DataType_Double { + return nil, fmt.Errorf("failed to get double from datatype %v", r.colType.String()) + } + dumper, err := r.createDumper() + if err != nil { + return nil, err + } + ret := make([]float64, r.numRows) + var i int64 + for i = 0; i < r.numRows; i++ { + v, hasValue := dumper.Next() + if !hasValue { + return nil, fmt.Errorf("unmatched row number: expect %v, actual %v", r.numRows, i) + } + ret[i] = v.(float64) + } + return ret, nil +} + +func (r *PayloadReader) GetStringFromPayload() ([]string, error) { + if r.colType != schemapb.DataType_String && r.colType != schemapb.DataType_VarChar { + return nil, fmt.Errorf("failed to get string from datatype %v", r.colType.String()) + } + dumper, err := r.createDumper() + if err != nil { + return nil, err + } + ret := make([]string, r.numRows) + var i int64 + for i = 0; i < r.numRows; i++ { + v, hasValue := dumper.Next() + if !hasValue { + return nil, fmt.Errorf("unmatched row number: expect %v, actual %v", r.numRows, i) + } + ret[i] = v.(parquet.ByteArray).String() + } + return ret, nil +} + +// GetBinaryVectorFromPayload returns vector, dimension, error +func (r *PayloadReader) GetBinaryVectorFromPayload() ([]byte, int, error) { + if r.colType != schemapb.DataType_BinaryVector { + return nil, -1, fmt.Errorf("failed to get binary vector from datatype %v", r.colType.String()) + } + dumper, err := r.createDumper() + if err != nil { + return nil, -1, err + } + + dim := r.reader.RowGroup(0).Column(0).Descriptor().TypeLength() + ret := make([]byte, int64(dim)*r.numRows) + for i := 0; i < int(r.numRows); i++ { + v, ok := dumper.Next() + if !ok { + return nil, -1, fmt.Errorf("unmatched row number: row %v, dim %v", r.numRows, dim) + } + parquetArray := v.(parquet.FixedLenByteArray) + copy(ret[i*dim:(i+1)*dim], parquetArray) + } + return ret, dim * 8, nil +} + +// GetFloatVectorFromPayload returns vector, dimension, error +func (r *PayloadReader) GetFloatVectorFromPayload() ([]float32, int, error) { + if r.colType != schemapb.DataType_FloatVector { + return nil, -1, fmt.Errorf("failed to get float vector from datatype %v", r.colType.String()) + } + dumper, err := r.createDumper() + if err != nil { + return nil, -1, err + } + + dim := r.reader.RowGroup(0).Column(0).Descriptor().TypeLength() / 4 + ret := make([]float32, int64(dim)*r.numRows) + for i := 0; i < int(r.numRows); i++ { + v, ok := dumper.Next() + if !ok { + return nil, -1, fmt.Errorf("unmatched row number: row %v, dim %v", r.numRows, dim) + } + parquetArray := v.(parquet.FixedLenByteArray) + copy(arrow.Float32Traits.CastToBytes(ret[i*dim:(i+1)*dim]), parquetArray) + } + return ret, dim, nil +} + +func (r *PayloadReader) GetPayloadLengthFromReader() (int, error) { + return int(r.numRows), nil +} + +// Close closes the payload reader +func (r *PayloadReader) Close() { + r.reader.Close() +} + +type Dumper struct { + reader file.ColumnChunkReader + batchSize int64 + valueOffset int + valuesBuffered int + + levelOffset int64 + levelsBuffered int64 + defLevels []int16 + repLevels []int16 + + valueBuffer interface{} +} + +func (r *PayloadReader) createDumper() (*Dumper, error) { + var valueBuffer interface{} + switch r.reader.RowGroup(0).Column(0).(type) { + case *file.BooleanColumnChunkReader: + if r.colType != schemapb.DataType_Bool { + return nil, errors.New("incorrect data type") + } + valueBuffer = make([]bool, r.numRows) + case *file.Int32ColumnChunkReader: + if r.colType != schemapb.DataType_Int32 && r.colType != schemapb.DataType_Int16 && r.colType != schemapb.DataType_Int8 { + return nil, fmt.Errorf("incorrect data type, expect int32/int16/int8 but find %v", r.colType.String()) + } + valueBuffer = make([]int32, r.numRows) + case *file.Int64ColumnChunkReader: + if r.colType != schemapb.DataType_Int64 { + return nil, fmt.Errorf("incorrect data type, expect int64 but find %v", r.colType.String()) + } + valueBuffer = make([]int64, r.numRows) + case *file.Float32ColumnChunkReader: + if r.colType != schemapb.DataType_Float { + return nil, fmt.Errorf("incorrect data type, expect float32 but find %v", r.colType.String()) + } + valueBuffer = make([]float32, r.numRows) + case *file.Float64ColumnChunkReader: + if r.colType != schemapb.DataType_Double { + return nil, fmt.Errorf("incorrect data type, expect float64 but find %v", r.colType.String()) + } + valueBuffer = make([]float64, r.numRows) + case *file.ByteArrayColumnChunkReader: + if r.colType != schemapb.DataType_String && r.colType != schemapb.DataType_VarChar { + return nil, fmt.Errorf("incorrect data type, expect string/varchar but find %v", r.colType.String()) + } + valueBuffer = make([]parquet.ByteArray, r.numRows) + case *file.FixedLenByteArrayColumnChunkReader: + if r.colType != schemapb.DataType_FloatVector && r.colType != schemapb.DataType_BinaryVector { + return nil, fmt.Errorf("incorrect data type, expect floavector/binaryvector but find %v", r.colType.String()) + } + valueBuffer = make([]parquet.FixedLenByteArray, r.numRows) + } + + return &Dumper{ + reader: r.reader.RowGroup(0).Column(0), + batchSize: r.numRows, + defLevels: make([]int16, r.numRows), + repLevels: make([]int16, r.numRows), + valueBuffer: valueBuffer, + }, nil +} + +func (dump *Dumper) readNextBatch() { + switch reader := dump.reader.(type) { + case *file.BooleanColumnChunkReader: + values := dump.valueBuffer.([]bool) + dump.levelsBuffered, dump.valuesBuffered, _ = reader.ReadBatch(dump.batchSize, values, dump.defLevels, dump.repLevels) + case *file.Int32ColumnChunkReader: + values := dump.valueBuffer.([]int32) + dump.levelsBuffered, dump.valuesBuffered, _ = reader.ReadBatch(dump.batchSize, values, dump.defLevels, dump.repLevels) + case *file.Int64ColumnChunkReader: + values := dump.valueBuffer.([]int64) + dump.levelsBuffered, dump.valuesBuffered, _ = reader.ReadBatch(dump.batchSize, values, dump.defLevels, dump.repLevels) + case *file.Float32ColumnChunkReader: + values := dump.valueBuffer.([]float32) + dump.levelsBuffered, dump.valuesBuffered, _ = reader.ReadBatch(dump.batchSize, values, dump.defLevels, dump.repLevels) + case *file.Float64ColumnChunkReader: + values := dump.valueBuffer.([]float64) + dump.levelsBuffered, dump.valuesBuffered, _ = reader.ReadBatch(dump.batchSize, values, dump.defLevels, dump.repLevels) + case *file.Int96ColumnChunkReader: + values := dump.valueBuffer.([]parquet.Int96) + dump.levelsBuffered, dump.valuesBuffered, _ = reader.ReadBatch(dump.batchSize, values, dump.defLevels, dump.repLevels) + case *file.ByteArrayColumnChunkReader: + values := dump.valueBuffer.([]parquet.ByteArray) + dump.levelsBuffered, dump.valuesBuffered, _ = reader.ReadBatch(dump.batchSize, values, dump.defLevels, dump.repLevels) + case *file.FixedLenByteArrayColumnChunkReader: + values := dump.valueBuffer.([]parquet.FixedLenByteArray) + dump.levelsBuffered, dump.valuesBuffered, _ = reader.ReadBatch(dump.batchSize, values, dump.defLevels, dump.repLevels) + } + + dump.valueOffset = 0 + dump.levelOffset = 0 +} + +func (dump *Dumper) hasNext() bool { + return dump.levelOffset < dump.levelsBuffered || dump.reader.HasNext() +} + +func (dump *Dumper) Next() (interface{}, bool) { + if dump.levelOffset == dump.levelsBuffered { + if !dump.hasNext() { + return nil, false + } + dump.readNextBatch() + if dump.levelsBuffered == 0 { + return nil, false + } + } + + defLevel := dump.defLevels[int(dump.levelOffset)] + // repLevel := dump.repLevels[int(dump.levelOffset)] + dump.levelOffset++ + + if defLevel < dump.reader.Descriptor().MaxDefinitionLevel() { + return nil, true + } + + vb := reflect.ValueOf(dump.valueBuffer) + v := vb.Index(dump.valueOffset).Interface() + dump.valueOffset++ + + return v, true +} diff --git a/internal/storage/payload_reader_cgo.go b/internal/storage/payload_reader_cgo.go new file mode 100644 index 0000000000..0a063773e9 --- /dev/null +++ b/internal/storage/payload_reader_cgo.go @@ -0,0 +1,322 @@ +package storage + +/* +#cgo CFLAGS: -I${SRCDIR}/cwrapper + +#cgo linux LDFLAGS: -L${SRCDIR}/cwrapper/output/lib -L${SRCDIR}/cwrapper/output/lib64 -lwrapper -lparquet -larrow -larrow_bundled_dependencies -lstdc++ -lm +#cgo darwin LDFLAGS: -L${SRCDIR}/cwrapper/output/lib -lwrapper -lparquet -larrow -larrow_bundled_dependencies -lstdc++ -lm +#cgo windows LDFLAGS: -L${SRCDIR}/cwrapper/output/lib -lwrapper -lparquet -larrow -lstdc++ +#include +#include "ParquetWrapper.h" +*/ +import "C" +import ( + "errors" + "fmt" + "github.com/milvus-io/milvus/internal/log" + "github.com/milvus-io/milvus/internal/proto/commonpb" + "github.com/milvus-io/milvus/internal/proto/schemapb" + "unsafe" +) + +// PayloadReader reads data from payload +type PayloadReaderCgo struct { + payloadReaderPtr C.CPayloadReader + colType schemapb.DataType +} + +func NewPayloadReaderGgo(colType schemapb.DataType, buf []byte) (*PayloadReaderCgo, error) { + if len(buf) == 0 { + return nil, errors.New("create Payload reader failed, buffer is empty") + } + r := C.NewPayloadReader(C.int(colType), (*C.uint8_t)(unsafe.Pointer(&buf[0])), C.int64_t(len(buf))) + if r == nil { + return nil, errors.New("failed to read parquet from buffer") + } + return &PayloadReaderCgo{payloadReaderPtr: r, colType: colType}, nil +} + +// GetDataFromPayload returns data,length from payload, returns err if failed +// Params: +// `idx`: String index +// Return: +// `interface{}`: all types. +// `int`: length, only meaningful to FLOAT/BINARY VECTOR type. +// `error`: error. +func (r *PayloadReaderCgo) GetDataFromPayload() (interface{}, int, error) { + switch r.colType { + case schemapb.DataType_Bool: + val, err := r.GetBoolFromPayload() + return val, 0, err + case schemapb.DataType_Int8: + val, err := r.GetInt8FromPayload() + return val, 0, err + case schemapb.DataType_Int16: + val, err := r.GetInt16FromPayload() + return val, 0, err + case schemapb.DataType_Int32: + val, err := r.GetInt32FromPayload() + return val, 0, err + case schemapb.DataType_Int64: + val, err := r.GetInt64FromPayload() + return val, 0, err + case schemapb.DataType_Float: + val, err := r.GetFloatFromPayload() + return val, 0, err + case schemapb.DataType_Double: + val, err := r.GetDoubleFromPayload() + return val, 0, err + case schemapb.DataType_BinaryVector: + return r.GetBinaryVectorFromPayload() + case schemapb.DataType_FloatVector: + return r.GetFloatVectorFromPayload() + case schemapb.DataType_String: + val, err := r.GetStringFromPayload() + return val, 0, err + default: + return nil, 0, errors.New("unknown type") + } +} + +// ReleasePayloadReader release payload reader. +func (r *PayloadReaderCgo) ReleasePayloadReader() { + C.ReleasePayloadReader(r.payloadReaderPtr) +} + +// GetBoolFromPayload returns bool slice from payload. +func (r *PayloadReaderCgo) GetBoolFromPayload() ([]bool, error) { + if r.colType != schemapb.DataType_Bool { + return nil, errors.New("incorrect data type") + } + + var cMsg *C.bool + var cSize C.int + + status := C.GetBoolFromPayload(r.payloadReaderPtr, &cMsg, &cSize) + if err := HandleCStatus(&status, "GetBoolFromPayload failed"); err != nil { + return nil, err + } + + slice := (*[1 << 28]bool)(unsafe.Pointer(cMsg))[:cSize:cSize] + return slice, nil +} + +// GetByteFromPayload returns byte slice from payload +func (r *PayloadReaderCgo) GetByteFromPayload() ([]byte, error) { + if r.colType != schemapb.DataType_Int8 { + return nil, errors.New("incorrect data type") + } + + var cMsg *C.int8_t + var cSize C.int + + status := C.GetInt8FromPayload(r.payloadReaderPtr, &cMsg, &cSize) + if err := HandleCStatus(&status, "GetInt8FromPayload failed"); err != nil { + return nil, err + } + + slice := (*[1 << 28]byte)(unsafe.Pointer(cMsg))[:cSize:cSize] + return slice, nil +} + +// GetInt8FromPayload returns int8 slice from payload +func (r *PayloadReaderCgo) GetInt8FromPayload() ([]int8, error) { + if r.colType != schemapb.DataType_Int8 { + return nil, errors.New("incorrect data type") + } + + var cMsg *C.int8_t + var cSize C.int + + status := C.GetInt8FromPayload(r.payloadReaderPtr, &cMsg, &cSize) + if err := HandleCStatus(&status, "GetInt8FromPayload failed"); err != nil { + return nil, err + } + + slice := (*[1 << 28]int8)(unsafe.Pointer(cMsg))[:cSize:cSize] + return slice, nil +} + +func (r *PayloadReaderCgo) GetInt16FromPayload() ([]int16, error) { + if r.colType != schemapb.DataType_Int16 { + return nil, errors.New("incorrect data type") + } + + var cMsg *C.int16_t + var cSize C.int + + status := C.GetInt16FromPayload(r.payloadReaderPtr, &cMsg, &cSize) + if err := HandleCStatus(&status, "GetInt16FromPayload failed"); err != nil { + return nil, err + } + + slice := (*[1 << 28]int16)(unsafe.Pointer(cMsg))[:cSize:cSize] + return slice, nil +} + +func (r *PayloadReaderCgo) GetInt32FromPayload() ([]int32, error) { + if r.colType != schemapb.DataType_Int32 { + return nil, errors.New("incorrect data type") + } + + var cMsg *C.int32_t + var cSize C.int + + status := C.GetInt32FromPayload(r.payloadReaderPtr, &cMsg, &cSize) + if err := HandleCStatus(&status, "GetInt32FromPayload failed"); err != nil { + return nil, err + } + + slice := (*[1 << 28]int32)(unsafe.Pointer(cMsg))[:cSize:cSize] + return slice, nil +} + +func (r *PayloadReaderCgo) GetInt64FromPayload() ([]int64, error) { + if r.colType != schemapb.DataType_Int64 { + return nil, errors.New("incorrect data type") + } + + var cMsg *C.int64_t + var cSize C.int + + status := C.GetInt64FromPayload(r.payloadReaderPtr, &cMsg, &cSize) + if err := HandleCStatus(&status, "GetInt64FromPayload failed"); err != nil { + return nil, err + } + + slice := (*[1 << 28]int64)(unsafe.Pointer(cMsg))[:cSize:cSize] + return slice, nil +} + +func (r *PayloadReaderCgo) GetFloatFromPayload() ([]float32, error) { + if r.colType != schemapb.DataType_Float { + return nil, errors.New("incorrect data type") + } + + var cMsg *C.float + var cSize C.int + + status := C.GetFloatFromPayload(r.payloadReaderPtr, &cMsg, &cSize) + if err := HandleCStatus(&status, "GetFloatFromPayload failed"); err != nil { + return nil, err + } + + slice := (*[1 << 28]float32)(unsafe.Pointer(cMsg))[:cSize:cSize] + return slice, nil +} + +func (r *PayloadReaderCgo) GetDoubleFromPayload() ([]float64, error) { + if r.colType != schemapb.DataType_Double { + return nil, errors.New("incorrect data type") + } + + var cMsg *C.double + var cSize C.int + + status := C.GetDoubleFromPayload(r.payloadReaderPtr, &cMsg, &cSize) + if err := HandleCStatus(&status, "GetDoubleFromPayload failed"); err != nil { + return nil, err + } + + slice := (*[1 << 28]float64)(unsafe.Pointer(cMsg))[:cSize:cSize] + return slice, nil +} + +func (r *PayloadReaderCgo) GetStringFromPayload() ([]string, error) { + length, err := r.GetPayloadLengthFromReader() + if err != nil { + return nil, err + } + ret := make([]string, length) + for i := 0; i < length; i++ { + ret[i], err = r.GetOneStringFromPayload(i) + if err != nil { + return nil, err + } + } + return ret, nil +} + +func (r *PayloadReaderCgo) GetOneStringFromPayload(idx int) (string, error) { + if r.colType != schemapb.DataType_String { + return "", errors.New("incorrect data type") + } + + var cStr *C.char + var cSize C.int + + status := C.GetOneStringFromPayload(r.payloadReaderPtr, C.int(idx), &cStr, &cSize) + if err := HandleCStatus(&status, "GetOneStringFromPayload failed"); err != nil { + return "", err + } + return C.GoStringN(cStr, cSize), nil +} + +// GetBinaryVectorFromPayload returns vector, dimension, error +func (r *PayloadReaderCgo) GetBinaryVectorFromPayload() ([]byte, int, error) { + if r.colType != schemapb.DataType_BinaryVector { + return nil, 0, errors.New("incorrect data type") + } + + var cMsg *C.uint8_t + var cDim C.int + var cLen C.int + + status := C.GetBinaryVectorFromPayload(r.payloadReaderPtr, &cMsg, &cDim, &cLen) + if err := HandleCStatus(&status, "GetBinaryVectorFromPayload failed"); err != nil { + return nil, 0, err + } + length := (cDim / 8) * cLen + + slice := (*[1 << 28]byte)(unsafe.Pointer(cMsg))[:length:length] + return slice, int(cDim), nil +} + +// GetFloatVectorFromPayload returns vector, dimension, error +func (r *PayloadReaderCgo) GetFloatVectorFromPayload() ([]float32, int, error) { + if r.colType != schemapb.DataType_FloatVector { + return nil, 0, errors.New("incorrect data type") + } + + var cMsg *C.float + var cDim C.int + var cLen C.int + + status := C.GetFloatVectorFromPayload(r.payloadReaderPtr, &cMsg, &cDim, &cLen) + if err := HandleCStatus(&status, "GetFloatVectorFromPayload failed"); err != nil { + return nil, 0, err + } + length := cDim * cLen + + slice := (*[1 << 28]float32)(unsafe.Pointer(cMsg))[:length:length] + return slice, int(cDim), nil +} + +func (r *PayloadReaderCgo) GetPayloadLengthFromReader() (int, error) { + length := C.GetPayloadLengthFromReader(r.payloadReaderPtr) + return int(length), nil +} + +// Close closes the payload reader +func (r *PayloadReaderCgo) Close() { + r.ReleasePayloadReader() +} + +// HandleCStatus deal with the error returned from CGO +func HandleCStatus(status *C.CStatus, extraInfo string) error { + if status.error_code == 0 { + return nil + } + errorCode := status.error_code + errorName, ok := commonpb.ErrorCode_name[int32(errorCode)] + if !ok { + errorName = "UnknownError" + } + errorMsg := C.GoString(status.error_msg) + defer C.free(unsafe.Pointer(status.error_msg)) + + finalMsg := fmt.Sprintf("[%s] %s", errorName, errorMsg) + logMsg := fmt.Sprintf("%s, C Runtime Exception: %s\n", extraInfo, finalMsg) + log.Warn(logMsg) + return errors.New(finalMsg) +} diff --git a/internal/storage/payload_test.go b/internal/storage/payload_test.go index a6222db9ee..6e5f78c2b1 100644 --- a/internal/storage/payload_test.go +++ b/internal/storage/payload_test.go @@ -285,7 +285,7 @@ func TestPayload_ReaderandWriter(t *testing.T) { defer r.ReleasePayloadReader() }) - t.Run("TestAddOneString", func(t *testing.T) { + t.Run("TestAddString", func(t *testing.T) { w, err := NewPayloadWriter(schemapb.DataType_String) require.Nil(t, err) require.NotNil(t, w) @@ -311,39 +311,22 @@ func TestPayload_ReaderandWriter(t *testing.T) { length, err = r.GetPayloadLengthFromReader() assert.Nil(t, err) assert.Equal(t, length, 4) - str0, err := r.GetOneStringFromPayload(0) - assert.Nil(t, err) - assert.Equal(t, str0, "hello0") - str1, err := r.GetOneStringFromPayload(1) - assert.Nil(t, err) - assert.Equal(t, str1, "hello1") - str2, err := r.GetOneStringFromPayload(2) - assert.Nil(t, err) - assert.Equal(t, str2, "hello2") - str3, err := r.GetOneStringFromPayload(3) - assert.Nil(t, err) - assert.Equal(t, str3, "hello3") - istr0, _, err := r.GetDataFromPayload(0) - str0 = istr0.(string) + str, err := r.GetStringFromPayload() assert.Nil(t, err) - assert.Equal(t, str0, "hello0") - istr1, _, err := r.GetDataFromPayload(1) - str1 = istr1.(string) + assert.Equal(t, str[0], "hello0") + assert.Equal(t, str[1], "hello1") + assert.Equal(t, str[2], "hello2") + assert.Equal(t, str[3], "hello3") + + istr, _, err := r.GetDataFromPayload() + strArray := istr.([]string) assert.Nil(t, err) - assert.Equal(t, str1, "hello1") - - istr2, _, err := r.GetDataFromPayload(2) - str2 = istr2.(string) - assert.Nil(t, err) - assert.Equal(t, str2, "hello2") - - istr3, _, err := r.GetDataFromPayload(3) - str3 = istr3.(string) - assert.Nil(t, err) - assert.Equal(t, str3, "hello3") - + assert.Equal(t, strArray[0], "hello0") + assert.Equal(t, strArray[1], "hello1") + assert.Equal(t, strArray[2], "hello2") + assert.Equal(t, strArray[3], "hello3") r.ReleasePayloadReader() w.ReleasePayloadWriter() }) @@ -647,12 +630,6 @@ func TestPayload_ReaderandWriter(t *testing.T) { _, _, err := r.GetDataFromPayload() assert.NotNil(t, err) - - _, _, err = r.GetDataFromPayload(1) - assert.NotNil(t, err) - - _, _, err = r.GetDataFromPayload(1, 1) - assert.NotNil(t, err) }) t.Run("TestGetBoolError", func(t *testing.T) { w, err := NewPayloadWriter(schemapb.DataType_Int8) @@ -822,7 +799,7 @@ func TestPayload_ReaderandWriter(t *testing.T) { _, err = r.GetDoubleFromPayload() assert.NotNil(t, err) }) - t.Run("TestGetOneStringError", func(t *testing.T) { + t.Run("TestGetStringError", func(t *testing.T) { w, err := NewPayloadWriter(schemapb.DataType_Bool) require.Nil(t, err) require.NotNil(t, w) @@ -839,11 +816,11 @@ func TestPayload_ReaderandWriter(t *testing.T) { r, err := NewPayloadReader(schemapb.DataType_String, buffer) assert.Nil(t, err) - _, err = r.GetOneStringFromPayload(0) + _, err = r.GetStringFromPayload() assert.NotNil(t, err) r.colType = 999 - _, err = r.GetOneStringFromPayload(0) + _, err = r.GetStringFromPayload() assert.NotNil(t, err) }) t.Run("TestGetBinaryVectorError", func(t *testing.T) { diff --git a/internal/storage/print_binlog.go b/internal/storage/print_binlog.go index 4c308aa0ed..408e0957ee 100644 --- a/internal/storage/print_binlog.go +++ b/internal/storage/print_binlog.go @@ -282,12 +282,13 @@ func printPayloadValues(colType schemapb.DataType, reader PayloadReaderInterface if err != nil { return err } + + val, err := reader.GetStringFromPayload() + if err != nil { + return err + } for i := 0; i < rows; i++ { - val, err := reader.GetOneStringFromPayload(i) - if err != nil { - return err - } - fmt.Printf("\t\t%d : %s\n", i, val) + fmt.Printf("\t\t%d : %s\n", i, val[i]) } case schemapb.DataType_BinaryVector: val, dim, err := reader.GetBinaryVectorFromPayload() @@ -341,12 +342,13 @@ func printDDLPayloadValues(eventType EventTypeCode, colType schemapb.DataType, r if err != nil { return err } + + val, err := reader.GetStringFromPayload() + if err != nil { + return err + } for i := 0; i < rows; i++ { - val, err := reader.GetOneStringFromPayload(i) - valBytes := []byte(val) - if err != nil { - return err - } + valBytes := []byte(val[i]) switch eventType { case CreateCollectionEventType: var req internalpb.CreateCollectionRequest