diff --git a/.gitignore b/.gitignore index ef85f5678c..9c71802f8f 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,7 @@ **/cmake-build-release/* internal/core/output/* internal/core/build/* +internal/kv/rocksdb/cwrapper/output/* **/.idea/* pulsar/client-cpp/build/ pulsar/client-cpp/build/* diff --git a/Makefile b/Makefile index 48f90b3fbf..4d8a81973f 100644 --- a/Makefile +++ b/Makefile @@ -57,8 +57,14 @@ lint:tools/bin/revive @echo "Running $@ check" @tools/bin/revive -formatter friendly -config tools/check/revive.toml ./... +get-rocksdb: + @go env -w CGO_CFLAGS="-I$(PWD)/internal/kv/rocksdb/cwrapper/output/include" + @go env -w CGO_LDFLAGS="-L$(PWD)/internal/kv/rocksdb/cwrapper/output/lib -l:librocksdb.a -lstdc++ -lm -lz" + @(env bash $(PWD)/internal/kv/rocksdb/cwrapper/build.sh) + @go get github.com/tecbot/gorocksdb + #TODO: Check code specifications by golangci-lint -static-check: +static-check:get-rocksdb @echo "Running $@ check" @GO111MODULE=on ${GOPATH}/bin/golangci-lint cache clean @GO111MODULE=on ${GOPATH}/bin/golangci-lint run --timeout=30m --config ./.golangci.yml ./internal/... @@ -76,7 +82,7 @@ else @${GOPATH}/bin/ruleguard -rules ruleguard.rules.go ./tests/go/... endif -verifiers: getdeps cppcheck fmt static-check ruleguard +verifiers: getdeps cppcheck fmt static-check # Builds various components locally. build-go: build-cpp @@ -106,7 +112,7 @@ build-cpp-with-unittest: unittest: test-cpp test-go #TODO: proxy master query node writer's unittest -test-go: +test-go:get-rocksdb @echo "Running go unittests..." @(env bash $(PWD)/scripts/run_go_unittest.sh) diff --git a/docs/developer_guides/chap04_message_stream.md b/docs/developer_guides/chap04_message_stream.md index f3a5ffc2a7..19fefbd344 100644 --- a/docs/developer_guides/chap04_message_stream.md +++ b/docs/developer_guides/chap04_message_stream.md @@ -1,3 +1,5 @@ + + ## 8. Message Stream Service @@ -48,6 +50,8 @@ type ChannelDescriptions struct { + + #### A.3 Message Stream ``` go @@ -150,55 +154,3 @@ func NewUnmarshalDispatcher() *UnmarshalDispatcher ``` - -#### A.4 RocksMQ - -RocksMQ is a RocksDB-based messaging/streaming library. - -```go -type ProducerMessage struct { - Key string - Payload []byte -} -``` - -```go -type ConsumerMessage struct { - MsgID MessageID - Key string - Payload []byte -} -``` - - - -```GO -type RocksMQ struct { - CreateChannel(channelName string) error - DestroyChannel(channelName string) error - CreateConsumerGroup(groupName string) error - DestroyConsumerGroup(groupName string) error - - Produce(channelName string, messages []ProducerMessage) error - Consume(groupName string, channelName string, n int) ([]ConsumerMessage, error) - Seek(groupName string, channelName string, msgID MessageID) error -} -``` - - - -##### A.4.1 Meta - -* channel meta - -```go -"$(channel_name)/start_id", MessageID -"$(channel_name)/end_id", MessageID -``` - -* consumer group meta - -```go -"$(group_name)/$(channel_name)/current_id", MessageID -``` - diff --git a/go.mod b/go.mod index bb426c8ba0..be98747570 100644 --- a/go.mod +++ b/go.mod @@ -10,9 +10,11 @@ require ( github.com/aws/aws-sdk-go v1.30.8 // indirect github.com/coreos/etcd v3.3.25+incompatible // indirect github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 // indirect + github.com/facebookgo/ensure v0.0.0-20200202191622-63f1cf65ac4c // indirect + github.com/facebookgo/stack v0.0.0-20160209184415-751773369052 // indirect + github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4 // indirect github.com/frankban/quicktest v1.10.2 // indirect github.com/fsnotify/fsnotify v1.4.9 // indirect - github.com/git-hooks/git-hooks v1.3.1 // indirect github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect github.com/golang/mock v1.3.1 github.com/golang/protobuf v1.3.2 @@ -32,13 +34,14 @@ require ( github.com/prometheus/client_golang v1.5.1 // indirect github.com/prometheus/common v0.10.0 // indirect github.com/prometheus/procfs v0.1.3 // indirect - github.com/russross/blackfriday/v2 v2.1.0 // indirect + github.com/quasilyte/go-ruleguard v0.2.1 // indirect github.com/sirupsen/logrus v1.6.0 // indirect github.com/spaolacci/murmur3 v1.1.0 github.com/spf13/cast v1.3.0 github.com/spf13/viper v1.7.1 github.com/stretchr/testify v1.6.1 - github.com/tikv/client-go v0.0.0-20200824032810-95774393107b // indirect + github.com/tecbot/gorocksdb v0.0.0-20191217155057-f0fad39f321c + github.com/tikv/client-go v0.0.0-20200824032810-95774393107b github.com/uber/jaeger-client-go v2.25.0+incompatible github.com/uber/jaeger-lib v2.4.0+incompatible // indirect github.com/urfave/cli v1.22.5 // indirect diff --git a/go.sum b/go.sum index 21b39b35f6..aaf70aaa9c 100644 --- a/go.sum +++ b/go.sum @@ -44,12 +44,8 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= -github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ= -github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= github.com/bmizerany/perks v0.0.0-20141205001514-d9a9656a3a4b/go.mod h1:ac9efd0D1fsDb3EJvhqgXRbFx7bs2wqZ10HQPeU8U/Q= github.com/boynton/repl v0.0.0-20170116235056-348863958e3e/go.mod h1:Crc/GCZ3NXDVCio7Yr0o+SSrytpcFhLmVCIzi0s49t4= -github.com/cattail/go-exclude v0.0.0-20141118090525-7e63167c2dab h1:1WOH7EEbhb6OZWcIU5RpQx5rmHm1xEUda8Qiw4UzNlU= -github.com/cattail/go-exclude v0.0.0-20141118090525-7e63167c2dab/go.mod h1:5MSsYMW59C/HfIUsthTRDxRoMQctcmAVb1JnNSQXERA= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= @@ -72,8 +68,6 @@ github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfc github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f h1:lBNOc5arjvs8E5mO2tbpBpLoyyu8B6e44T7hJy6potg= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= -github.com/cpuguy83/go-md2man/v2 v2.0.0 h1:EoUDS0afbrsXAZ9YQ9jdu/mZ2sXgT1/2yyNng4PGlyM= -github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 h1:iwZdTE0PVqJCos1vaoKsclOGD3ADKpshg3SRtYBbwso= @@ -93,6 +87,12 @@ github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385 h1:clC1lXBpe2kTj2VHdaI github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385/go.mod h1:0vRUJqYpeSZifjYj7uP3BG/gKcuzL9xWVV/Y+cK33KM= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/facebookgo/ensure v0.0.0-20200202191622-63f1cf65ac4c h1:8ISkoahWXwZR41ois5lSJBSVw4D0OV19Ht/JSTzvSv0= +github.com/facebookgo/ensure v0.0.0-20200202191622-63f1cf65ac4c/go.mod h1:Yg+htXGokKKdzcwhuNDwVvN+uBxDGXJ7G/VN1d8fa64= +github.com/facebookgo/stack v0.0.0-20160209184415-751773369052 h1:JWuenKqqX8nojtoVVWjGfOF9635RETekkoH6Cc9SX0A= +github.com/facebookgo/stack v0.0.0-20160209184415-751773369052/go.mod h1:UbMTZqLaRiH3MsBH8va0n7s1pQYcu3uTb8G4tygF4Zg= +github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4 h1:7HZCaLC5+BZpmbhCOZJ293Lz68O7PYrF2EzeiFMwCLk= +github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4/go.mod h1:5tD+neXqOorC30/tWg0LCSkrqj/AR6gu8yY8/fpw1q0= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/frankban/quicktest v1.10.2 h1:19ARM85nVi4xH7xPXuc5eM/udya5ieh7b/Sv+d844Tk= github.com/frankban/quicktest v1.10.2/go.mod h1:K+q6oSqb0W0Ininfk863uOk1lMy69l/P6txr3mVT54s= @@ -100,8 +100,6 @@ github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMo github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= -github.com/git-hooks/git-hooks v1.3.1 h1:ypdvNTXMiITXQxuqIl6t1f8R3V1FrUXPi1CwQ4guClo= -github.com/git-hooks/git-hooks v1.3.1/go.mod h1:RBqjgxUpRSLI4AxVZGV1FsODclMjpsJIaDEyJSoK3ws= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= @@ -139,10 +137,6 @@ github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.2 h1:X2ev0eStA3AbceY54o37/0PQ/UWqKEiiO2dKL5OPaFM= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-github v17.0.0+incompatible h1:N0LgJ1j65A7kfXrZnUDaYCs/Sf4rEjNlfyDHW9dolSY= -github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ= -github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk= -github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= @@ -350,19 +344,18 @@ github.com/prometheus/procfs v0.1.3 h1:F0+tqvhOksq22sc6iCHF5WGlWjdwj92p0udFh1VFB github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/protocolbuffers/protobuf v3.14.0+incompatible h1:8r0H76h/Q/lEnFFY60AuM23NOnaDMi6bd7zuboSYM+o= +github.com/quasilyte/go-ruleguard v0.2.1 h1:56eRm0daAyny9UhJnmtJW/UyLZQusukBAB8oT8AHKHo= +github.com/quasilyte/go-ruleguard v0.2.1/go.mod h1:hN2rVc/uS4bQhQKTio2XaSJSafJwqBUWWwtssT3cQmc= +github.com/quasilyte/go-ruleguard/dsl v0.0.0-20210108021830-2a284c158646 h1:ShKMENtS4KYekn92z5vdicyR7wBOgpxFNj5h0M2rNKg= github.com/remyoudompheng/bigfft v0.0.0-20170806203942-52369c62f446 h1:/NRJ5vAYoqz+7sG51ubIDHXeWO8DlTSrToPu6q11ziA= github.com/remyoudompheng/bigfft v0.0.0-20170806203942-52369c62f446/go.mod h1:uYEyJGbgTkfkS4+E/PavXkNJcbFIpEtjt2B0KDQ5+9M= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rs/xid v1.2.1 h1:mhH9Nq+C1fY2l1XIpgxIiUOfNpRBYH1kKcr+qfKgjRc= github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= -github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= -github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= -github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.3.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= @@ -396,7 +389,6 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.2.0 h1:Hbg2NidpLE8veEBkEZTL3CvlkUIVzuU9jDplZO54c48= github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= -github.com/stretchr/testify v0.0.0-20141015234014-d6577e08ec30/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= @@ -405,6 +397,8 @@ github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= +github.com/tecbot/gorocksdb v0.0.0-20191217155057-f0fad39f321c h1:g+WoO5jjkqGAzHWCjJB1zZfXPIAaDpzXIEJ0eS6B5Ok= +github.com/tecbot/gorocksdb v0.0.0-20191217155057-f0fad39f321c/go.mod h1:ahpPrc7HpcfEWDQRZEmnXMzHY03mLDYMCxeDzy46i+8= github.com/tikv/client-go v0.0.0-20200824032810-95774393107b h1:VOG2GkM7RpRrT0St7HIIwCWrc3mVdf+DjcT8r2ucusI= github.com/tikv/client-go v0.0.0-20200824032810-95774393107b/go.mod h1:K0NcdVNrXDq92YPLytsrAwRMyuXi7GZCO6dXNH7OzQc= github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= @@ -421,14 +415,9 @@ github.com/ugorji/go/codec v0.0.0-20190204201341-e444a5086c43/go.mod h1:iT03XoTw github.com/unrolled/render v1.0.0 h1:XYtvhA3UkpB7PqkvhUFYmpKD55OudoIeygcfus4vcd4= github.com/unrolled/render v1.0.0/go.mod h1:tu82oB5W2ykJRVioYsB+IQKcft7ryBr7w12qMBUPyXg= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= -github.com/urfave/cli v1.22.1 h1:+mkCCcOFKPnCmVYVcURKps1Xe+3zP90gSYGNfRkjoIY= -github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= -github.com/urfave/cli v1.22.5 h1:lNq9sAHXK2qfdI8W+GRItjCEkI+2oR4d+MEHy1CKXoU= github.com/urfave/cli v1.22.5/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/valyala/gozstd v1.7.0 h1:Ljh5c9zboqLhwTI33al32R72iCZfn0mCbVGcFWbGwRQ= github.com/valyala/gozstd v1.7.0/go.mod h1:y5Ew47GLlP37EkTB+B4s7r6A5rdaeB7ftbl9zoYiIPQ= -github.com/wsxiaoys/terminal v0.0.0-20160513160801-0940f3fc43a0 h1:3UeQBvD0TFrlVjOeLOBz+CPAI8dnbqNSVwUwRrkp7vQ= -github.com/wsxiaoys/terminal v0.0.0-20160513160801-0940f3fc43a0/go.mod h1:IXCdmsXIht47RaVFLEdVnh1t+pgYtTAhQGj73kz+2DM= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/yahoo/athenz v1.8.55/go.mod h1:G7LLFUH7Z/r4QAB7FfudfuA7Am/eCzO1GlzBhDL6Kv0= @@ -582,6 +571,7 @@ golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200812195022-5ae4c3c160a0/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200825202427-b303f430e36d h1:W07d4xkoAUSNOkOzdzXCdFGxT7o2rW4q8M34tB2i//k= golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/internal/kv/rocksdb/cwrapper/CMakeLists.txt b/internal/kv/rocksdb/cwrapper/CMakeLists.txt new file mode 100644 index 0000000000..56602edcda --- /dev/null +++ b/internal/kv/rocksdb/cwrapper/CMakeLists.txt @@ -0,0 +1,55 @@ +cmake_minimum_required(VERSION 3.14...3.17 FATAL_ERROR) +project(wrapper) + +set(CMAKE_CXX_STANDARD 17) +set(CMAKE_EXPORT_COMPILE_COMMANDS ON) + +include( ExternalProject ) +set( ROCKSDB_VERSION "6.15.2" ) +set( ROCKSDB_SOURCE_URL + "https://github.com/facebook/rocksdb/archive/v${ROCKSDB_VERSION}.tar.gz") + +if( CUSTOM_THIRDPARTY_DOWNLOAD_PATH ) + set( THIRDPARTY_DOWNLOAD_PATH ${CUSTOM_THIRDPARTY_DOWNLOAD_PATH} ) +else() + set( THIRDPARTY_DOWNLOAD_PATH ${CMAKE_BINARY_DIR}/3rdparty_download/download ) +endif() +message( STATUS "Thirdparty downloaded file path: ${THIRDPARTY_DOWNLOAD_PATH}" ) + +macro( build_rocksdb ) + message( STATUS "Building ROCKSDB-${ROCKSDB_VERSION} from source" ) + + set( ROCKSDB_CMAKE_ARGS + "-DWITH_GFLAGS=OFF" + "-DROCKSDB_BUILD_SHARED=OFF" +# "-DWITH_SNAPPY=ON" +# "-DWITH_LZ4=ON" +# "-DWITH_ZSTD=ON" +# "-DWITH_BZ2=ON" + "-DCMAKE_INSTALL_PREFIX=${CMAKE_CURRENT_BINARY_DIR}" + ) + + ExternalProject_Add( + rocksdb-ep + PREFIX ${CMAKE_BINARY_DIR}/3rdparty_download/rocksdb-subbuild + BINARY_DIR rocksdb-bin + DOWNLOAD_DIR ${THIRDPARTY_DOWNLOAD_PATH} + INSTALL_DIR ${CMAKE_CURRENT_BINARY_DIR} + URL ${ROCKSDB_SOURCE_URL} + URL_MD5 "67f9e04fda62af551dd039c37b1808ac" + CMAKE_ARGS ${ROCKSDB_CMAKE_ARGS} + ${EP_LOG_OPTIONS} + ) + + ExternalProject_Get_Property( rocksdb-ep INSTALL_DIR ) + ExternalProject_Get_Property( rocksdb-ep BINARY_DIR ) + + if( NOT IS_DIRECTORY ${INSTALL_DIR}/include ) + file( MAKE_DIRECTORY "${INSTALL_DIR}/include" ) + endif() + +endmacro() + +build_rocksdb() + +#endif() diff --git a/internal/kv/rocksdb/cwrapper/build.sh b/internal/kv/rocksdb/cwrapper/build.sh new file mode 100755 index 0000000000..6ed4a2bd2a --- /dev/null +++ b/internal/kv/rocksdb/cwrapper/build.sh @@ -0,0 +1,53 @@ +#!/bin/bash + +SOURCE=${BASH_SOURCE[0]} +while [ -h $SOURCE ]; do # resolve $SOURCE until the file is no longer a symlink + DIR=$( cd -P $( dirname $SOURCE ) && pwd ) + SOURCE=$(readlink $SOURCE) + [[ $SOURCE != /* ]] && SOURCE=$DIR/$SOURCE # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located +done +DIR=$( cd -P $( dirname $SOURCE ) && pwd ) +# echo $DIR + +CGO_CFLAGS="-I$(pwd)/output/include" +CGO_LDFLAGS="-L$(pwd)/output/lib -l:librocksdb.a -lstdc++ -lm -lz" + +OUTPUT_LIB=${DIR}/output + +if [ -d ${OUTPUT_LIB} ];then + rm -rf ${OUTPUT_LIB} +fi +mkdir ${OUTPUT_LIB} + +BUILD_TYPE="Debug" + +while getopts "t:h:" arg; do + case $arg in + t) + BUILD_TYPE=$OPTARG # BUILD_TYPE + ;; + h) # help + echo "-t: build type(default: Debug) +-h: help + " + exit 0 + ;; + ?) + echo "ERROR! unknown argument" + exit 1 + ;; + esac +done +echo "BUILD_TYPE: " $BUILD_TYPE + +pushd ${OUTPUT_LIB} +CMAKE_CMD="cmake \ +-DCMAKE_BUILD_TYPE=${BUILD_TYPE} .." + +${CMAKE_CMD} +echo ${CMAKE_CMD} + +if [[ ! ${jobs+1} ]]; then + jobs=$(nproc) +fi +make -j ${jobs} diff --git a/internal/kv/rocksdb/rocksdb_kv.go b/internal/kv/rocksdb/rocksdb_kv.go new file mode 100644 index 0000000000..318f3078e8 --- /dev/null +++ b/internal/kv/rocksdb/rocksdb_kv.go @@ -0,0 +1,128 @@ +package rocksdbkv + +import ( + "github.com/tecbot/gorocksdb" +) + +type RocksdbKV struct { + opts *gorocksdb.Options + db *gorocksdb.DB + writeOptions *gorocksdb.WriteOptions + readOptions *gorocksdb.ReadOptions + name string +} + +func NewRocksdbKV(name string) (*RocksdbKV, error) { + bbto := gorocksdb.NewDefaultBlockBasedTableOptions() + bbto.SetBlockCache(gorocksdb.NewLRUCache(3 << 30)) + opts := gorocksdb.NewDefaultOptions() + opts.SetBlockBasedTableFactory(bbto) + opts.SetCreateIfMissing(true) + + ro := gorocksdb.NewDefaultReadOptions() + ro.SetFillCache(false) + + wo := gorocksdb.NewDefaultWriteOptions() + db, err := gorocksdb.OpenDb(opts, name) + if err != nil { + return nil, err + } + return &RocksdbKV{ + opts: opts, + db: db, + writeOptions: wo, + readOptions: ro, + name: name, + }, nil +} + +func (kv *RocksdbKV) Close() { + kv.db.Close() +} + +func (kv *RocksdbKV) GetName() string { + return kv.name +} + +func (kv *RocksdbKV) Load(key string) (string, error) { + value, err := kv.db.Get(kv.readOptions, []byte(key)) + defer value.Free() + return string(value.Data()), err +} + +func (kv *RocksdbKV) LoadWithPrefix(key string) ([]string, []string, error) { + iter := kv.db.NewIterator(kv.readOptions) + keys := make([]string, 0) + values := make([]string, 0) + iter.Seek([]byte(key)) + for ; iter.Valid(); iter.Next() { + key := iter.Key() + value := iter.Value() + keys = append(keys, string(key.Data())) + values = append(values, string(value.Data())) + key.Free() + value.Free() + } + if err := iter.Err(); err != nil { + return nil, nil, err + } + return keys, values, nil +} + +func (kv *RocksdbKV) MultiLoad(keys []string) ([]string, error) { + values := make([]string, 0, len(keys)) + for _, key := range keys { + value, err := kv.db.Get(kv.readOptions, []byte(key)) + if err != nil { + return []string{}, err + } + values = append(values, string(value.Data())) + } + return values, nil +} + +func (kv *RocksdbKV) Save(key, value string) error { + err := kv.db.Put(kv.writeOptions, []byte(key), []byte(value)) + return err +} + +func (kv *RocksdbKV) MultiSave(kvs map[string]string) error { + writeBatch := gorocksdb.NewWriteBatch() + defer writeBatch.Clear() + for k, v := range kvs { + writeBatch.Put([]byte(k), []byte(v)) + } + err := kv.db.Write(kv.writeOptions, writeBatch) + return err +} + +func (kv *RocksdbKV) RemoveWithPrefix(prefix string) error { + iter := kv.db.NewIterator(kv.readOptions) + iter.Seek([]byte(prefix)) + for ; iter.Valid(); iter.Next() { + key := iter.Key() + err := kv.db.Delete(kv.writeOptions, key.Data()) + if err != nil { + return nil + } + } + if err := iter.Err(); err != nil { + return err + } + return nil +} + +func (kv *RocksdbKV) Remove(key string) error { + err := kv.db.Delete(kv.writeOptions, []byte(key)) + return err +} + +func (kv *RocksdbKV) MultiRemove(keys []string) error { + writeBatch := gorocksdb.NewWriteBatch() + defer writeBatch.Clear() + for _, key := range keys { + writeBatch.Delete([]byte(key)) + } + err := kv.db.Write(kv.writeOptions, writeBatch) + return err +} diff --git a/internal/kv/rocksdb/rocksdb_kv_test.go b/internal/kv/rocksdb/rocksdb_kv_test.go new file mode 100644 index 0000000000..8898415a05 --- /dev/null +++ b/internal/kv/rocksdb/rocksdb_kv_test.go @@ -0,0 +1,54 @@ +package rocksdbkv_test + +import ( + "testing" + + "github.com/stretchr/testify/assert" + rocksdbkv "github.com/zilliztech/milvus-distributed/internal/kv/rocksdb" +) + +func TestRocksdbKV(t *testing.T) { + name := "/tmp/rocksdb" + rocksdbKV, err := rocksdbkv.NewRocksdbKV(name) + if err != nil { + panic(err) + } + + defer rocksdbKV.Close() + // Need to call RemoveWithPrefix + defer rocksdbKV.RemoveWithPrefix("") + + err = rocksdbKV.Save("abc", "123") + assert.Nil(t, err) + + err = rocksdbKV.Save("abcd", "1234") + assert.Nil(t, err) + + val, err := rocksdbKV.Load("abc") + assert.Nil(t, err) + assert.Equal(t, val, "123") + + keys, vals, err := rocksdbKV.LoadWithPrefix("abc") + assert.Nil(t, err) + assert.Equal(t, len(keys), len(vals)) + assert.Equal(t, len(keys), 2) + + assert.Equal(t, keys[0], "abc") + assert.Equal(t, keys[1], "abcd") + assert.Equal(t, vals[0], "123") + assert.Equal(t, vals[1], "1234") + + err = rocksdbKV.Save("key_1", "123") + assert.Nil(t, err) + err = rocksdbKV.Save("key_2", "456") + assert.Nil(t, err) + err = rocksdbKV.Save("key_3", "789") + assert.Nil(t, err) + + keys = []string{"key_1", "key_2"} + vals, err = rocksdbKV.MultiLoad(keys) + assert.Nil(t, err) + assert.Equal(t, len(vals), len(keys)) + assert.Equal(t, vals[0], "123") + assert.Equal(t, vals[1], "456") +} diff --git a/scripts/cwrapper_rocksdb_build.sh b/scripts/cwrapper_rocksdb_build.sh new file mode 100644 index 0000000000..85a9e3615a --- /dev/null +++ b/scripts/cwrapper_rocksdb_build.sh @@ -0,0 +1,54 @@ +#!/bin/bash + +SOURCE=${BASH_SOURCE[0]} +while [ -h $SOURCE ]; do # resolve $SOURCE until the file is no longer a symlink + DIR=$( cd -P $( dirname $SOURCE ) && pwd ) + SOURCE=$(readlink $SOURCE) + [[ $SOURCE != /* ]] && SOURCE=$DIR/$SOURCE # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located +done +DIR=$( cd -P $( dirname $SOURCE ) && pwd ) +# echo $DIR + +SRC_DIR=${DIR}/../internal/kv/rocksdb/cwrapper +CGO_CFLAGS="-I$(SRC_DIR)/output/include" +CGO_LDFLAGS="-L$(SRC_DIR)/output/lib -l:librocksdb.a -lstdc++ -lm -lz" + +OUTPUT_LIB=${SRC_DIR}/output + +if [ -d ${OUTPUT_LIB} ];then + rm -rf ${OUTPUT_LIB} +fi +mkdir ${OUTPUT_LIB} + +BUILD_TYPE="Debug" + +while getopts "t:h:" arg; do + case $arg in + t) + BUILD_TYPE=$OPTARG # BUILD_TYPE + ;; + h) # help + echo "-t: build type(default: Debug) +-h: help + " + exit 0 + ;; + ?) + echo "ERROR! unknown argument" + exit 1 + ;; + esac +done +echo "BUILD_TYPE: " $BUILD_TYPE + +pushd ${OUTPUT_LIB} +CMAKE_CMD="cmake \ +-DCMAKE_BUILD_TYPE=${BUILD_TYPE} ${SRC_DIR}" + +${CMAKE_CMD} +echo ${CMAKE_CMD} + +if [[ ! ${jobs+1} ]]; then + jobs=$(nproc) +fi +make -j ${jobs}