mirror of
https://gitee.com/milvus-io/milvus.git
synced 2026-01-07 19:31:51 +08:00
Add rocksdb in kv
Signed-off-by: yukun <kun.yu@zilliz.com>
This commit is contained in:
yukun
yefu.chen
parent
82ad488555
commit
d11f00cd9f
1
.gitignore
vendored
1
.gitignore
vendored
@ -4,6 +4,7 @@
|
||||
**/cmake-build-release/*
|
||||
internal/core/output/*
|
||||
internal/core/build/*
|
||||
internal/kv/rocksdb/cwrapper/output/*
|
||||
**/.idea/*
|
||||
pulsar/client-cpp/build/
|
||||
pulsar/client-cpp/build/*
|
||||
|
||||
12
Makefile
12
Makefile
@ -57,8 +57,14 @@ lint:tools/bin/revive
|
||||
@echo "Running $@ check"
|
||||
@tools/bin/revive -formatter friendly -config tools/check/revive.toml ./...
|
||||
|
||||
get-rocksdb:
|
||||
@go env -w CGO_CFLAGS="-I$(PWD)/internal/kv/rocksdb/cwrapper/output/include"
|
||||
@go env -w CGO_LDFLAGS="-L$(PWD)/internal/kv/rocksdb/cwrapper/output/lib -l:librocksdb.a -lstdc++ -lm -lz"
|
||||
@(env bash $(PWD)/internal/kv/rocksdb/cwrapper/build.sh)
|
||||
@go get github.com/tecbot/gorocksdb
|
||||
|
||||
#TODO: Check code specifications by golangci-lint
|
||||
static-check:
|
||||
static-check:get-rocksdb
|
||||
@echo "Running $@ check"
|
||||
@GO111MODULE=on ${GOPATH}/bin/golangci-lint cache clean
|
||||
@GO111MODULE=on ${GOPATH}/bin/golangci-lint run --timeout=30m --config ./.golangci.yml ./internal/...
|
||||
@ -76,7 +82,7 @@ else
|
||||
@${GOPATH}/bin/ruleguard -rules ruleguard.rules.go ./tests/go/...
|
||||
endif
|
||||
|
||||
verifiers: getdeps cppcheck fmt static-check ruleguard
|
||||
verifiers: getdeps cppcheck fmt static-check
|
||||
|
||||
# Builds various components locally.
|
||||
build-go: build-cpp
|
||||
@ -106,7 +112,7 @@ build-cpp-with-unittest:
|
||||
unittest: test-cpp test-go
|
||||
|
||||
#TODO: proxy master query node writer's unittest
|
||||
test-go:
|
||||
test-go:get-rocksdb
|
||||
@echo "Running go unittests..."
|
||||
@(env bash $(PWD)/scripts/run_go_unittest.sh)
|
||||
|
||||
|
||||
@ -1,3 +1,5 @@
|
||||
|
||||
|
||||
## 8. Message Stream Service
|
||||
|
||||
|
||||
@ -48,6 +50,8 @@ type ChannelDescriptions struct {
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
#### A.3 Message Stream
|
||||
|
||||
``` go
|
||||
@ -150,55 +154,3 @@ func NewUnmarshalDispatcher() *UnmarshalDispatcher
|
||||
```
|
||||
|
||||
|
||||
|
||||
#### A.4 RocksMQ
|
||||
|
||||
RocksMQ is a RocksDB-based messaging/streaming library.
|
||||
|
||||
```go
|
||||
type ProducerMessage struct {
|
||||
Key string
|
||||
Payload []byte
|
||||
}
|
||||
```
|
||||
|
||||
```go
|
||||
type ConsumerMessage struct {
|
||||
MsgID MessageID
|
||||
Key string
|
||||
Payload []byte
|
||||
}
|
||||
```
|
||||
|
||||
|
||||
|
||||
```GO
|
||||
type RocksMQ struct {
|
||||
CreateChannel(channelName string) error
|
||||
DestroyChannel(channelName string) error
|
||||
CreateConsumerGroup(groupName string) error
|
||||
DestroyConsumerGroup(groupName string) error
|
||||
|
||||
Produce(channelName string, messages []ProducerMessage) error
|
||||
Consume(groupName string, channelName string, n int) ([]ConsumerMessage, error)
|
||||
Seek(groupName string, channelName string, msgID MessageID) error
|
||||
}
|
||||
```
|
||||
|
||||
|
||||
|
||||
##### A.4.1 Meta
|
||||
|
||||
* channel meta
|
||||
|
||||
```go
|
||||
"$(channel_name)/start_id", MessageID
|
||||
"$(channel_name)/end_id", MessageID
|
||||
```
|
||||
|
||||
* consumer group meta
|
||||
|
||||
```go
|
||||
"$(group_name)/$(channel_name)/current_id", MessageID
|
||||
```
|
||||
|
||||
|
||||
9
go.mod
9
go.mod
@ -10,9 +10,11 @@ require (
|
||||
github.com/aws/aws-sdk-go v1.30.8 // indirect
|
||||
github.com/coreos/etcd v3.3.25+incompatible // indirect
|
||||
github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 // indirect
|
||||
github.com/facebookgo/ensure v0.0.0-20200202191622-63f1cf65ac4c // indirect
|
||||
github.com/facebookgo/stack v0.0.0-20160209184415-751773369052 // indirect
|
||||
github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4 // indirect
|
||||
github.com/frankban/quicktest v1.10.2 // indirect
|
||||
github.com/fsnotify/fsnotify v1.4.9 // indirect
|
||||
github.com/git-hooks/git-hooks v1.3.1 // indirect
|
||||
github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect
|
||||
github.com/golang/mock v1.3.1
|
||||
github.com/golang/protobuf v1.3.2
|
||||
@ -32,13 +34,14 @@ require (
|
||||
github.com/prometheus/client_golang v1.5.1 // indirect
|
||||
github.com/prometheus/common v0.10.0 // indirect
|
||||
github.com/prometheus/procfs v0.1.3 // indirect
|
||||
github.com/russross/blackfriday/v2 v2.1.0 // indirect
|
||||
github.com/quasilyte/go-ruleguard v0.2.1 // indirect
|
||||
github.com/sirupsen/logrus v1.6.0 // indirect
|
||||
github.com/spaolacci/murmur3 v1.1.0
|
||||
github.com/spf13/cast v1.3.0
|
||||
github.com/spf13/viper v1.7.1
|
||||
github.com/stretchr/testify v1.6.1
|
||||
github.com/tikv/client-go v0.0.0-20200824032810-95774393107b // indirect
|
||||
github.com/tecbot/gorocksdb v0.0.0-20191217155057-f0fad39f321c
|
||||
github.com/tikv/client-go v0.0.0-20200824032810-95774393107b
|
||||
github.com/uber/jaeger-client-go v2.25.0+incompatible
|
||||
github.com/uber/jaeger-lib v2.4.0+incompatible // indirect
|
||||
github.com/urfave/cli v1.22.5 // indirect
|
||||
|
||||
34
go.sum
34
go.sum
@ -44,12 +44,8 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
|
||||
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
|
||||
github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
|
||||
github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84=
|
||||
github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ=
|
||||
github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
|
||||
github.com/bmizerany/perks v0.0.0-20141205001514-d9a9656a3a4b/go.mod h1:ac9efd0D1fsDb3EJvhqgXRbFx7bs2wqZ10HQPeU8U/Q=
|
||||
github.com/boynton/repl v0.0.0-20170116235056-348863958e3e/go.mod h1:Crc/GCZ3NXDVCio7Yr0o+SSrytpcFhLmVCIzi0s49t4=
|
||||
github.com/cattail/go-exclude v0.0.0-20141118090525-7e63167c2dab h1:1WOH7EEbhb6OZWcIU5RpQx5rmHm1xEUda8Qiw4UzNlU=
|
||||
github.com/cattail/go-exclude v0.0.0-20141118090525-7e63167c2dab/go.mod h1:5MSsYMW59C/HfIUsthTRDxRoMQctcmAVb1JnNSQXERA=
|
||||
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
|
||||
github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
|
||||
github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
|
||||
@ -72,8 +68,6 @@ github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfc
|
||||
github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f h1:lBNOc5arjvs8E5mO2tbpBpLoyyu8B6e44T7hJy6potg=
|
||||
github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
|
||||
github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
|
||||
github.com/cpuguy83/go-md2man/v2 v2.0.0 h1:EoUDS0afbrsXAZ9YQ9jdu/mZ2sXgT1/2yyNng4PGlyM=
|
||||
github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
|
||||
github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY=
|
||||
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
|
||||
github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 h1:iwZdTE0PVqJCos1vaoKsclOGD3ADKpshg3SRtYBbwso=
|
||||
@ -93,6 +87,12 @@ github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385 h1:clC1lXBpe2kTj2VHdaI
|
||||
github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385/go.mod h1:0vRUJqYpeSZifjYj7uP3BG/gKcuzL9xWVV/Y+cK33KM=
|
||||
github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
|
||||
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
|
||||
github.com/facebookgo/ensure v0.0.0-20200202191622-63f1cf65ac4c h1:8ISkoahWXwZR41ois5lSJBSVw4D0OV19Ht/JSTzvSv0=
|
||||
github.com/facebookgo/ensure v0.0.0-20200202191622-63f1cf65ac4c/go.mod h1:Yg+htXGokKKdzcwhuNDwVvN+uBxDGXJ7G/VN1d8fa64=
|
||||
github.com/facebookgo/stack v0.0.0-20160209184415-751773369052 h1:JWuenKqqX8nojtoVVWjGfOF9635RETekkoH6Cc9SX0A=
|
||||
github.com/facebookgo/stack v0.0.0-20160209184415-751773369052/go.mod h1:UbMTZqLaRiH3MsBH8va0n7s1pQYcu3uTb8G4tygF4Zg=
|
||||
github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4 h1:7HZCaLC5+BZpmbhCOZJ293Lz68O7PYrF2EzeiFMwCLk=
|
||||
github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4/go.mod h1:5tD+neXqOorC30/tWg0LCSkrqj/AR6gu8yY8/fpw1q0=
|
||||
github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
|
||||
github.com/frankban/quicktest v1.10.2 h1:19ARM85nVi4xH7xPXuc5eM/udya5ieh7b/Sv+d844Tk=
|
||||
github.com/frankban/quicktest v1.10.2/go.mod h1:K+q6oSqb0W0Ininfk863uOk1lMy69l/P6txr3mVT54s=
|
||||
@ -100,8 +100,6 @@ github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMo
|
||||
github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
|
||||
github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
|
||||
github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
|
||||
github.com/git-hooks/git-hooks v1.3.1 h1:ypdvNTXMiITXQxuqIl6t1f8R3V1FrUXPi1CwQ4guClo=
|
||||
github.com/git-hooks/git-hooks v1.3.1/go.mod h1:RBqjgxUpRSLI4AxVZGV1FsODclMjpsJIaDEyJSoK3ws=
|
||||
github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
|
||||
github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
|
||||
github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
|
||||
@ -139,10 +137,6 @@ github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw
|
||||
github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/go-cmp v0.5.2 h1:X2ev0eStA3AbceY54o37/0PQ/UWqKEiiO2dKL5OPaFM=
|
||||
github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/go-github v17.0.0+incompatible h1:N0LgJ1j65A7kfXrZnUDaYCs/Sf4rEjNlfyDHW9dolSY=
|
||||
github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ=
|
||||
github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk=
|
||||
github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
|
||||
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
|
||||
github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
|
||||
github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
|
||||
@ -350,19 +344,18 @@ github.com/prometheus/procfs v0.1.3 h1:F0+tqvhOksq22sc6iCHF5WGlWjdwj92p0udFh1VFB
|
||||
github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
|
||||
github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
|
||||
github.com/protocolbuffers/protobuf v3.14.0+incompatible h1:8r0H76h/Q/lEnFFY60AuM23NOnaDMi6bd7zuboSYM+o=
|
||||
github.com/quasilyte/go-ruleguard v0.2.1 h1:56eRm0daAyny9UhJnmtJW/UyLZQusukBAB8oT8AHKHo=
|
||||
github.com/quasilyte/go-ruleguard v0.2.1/go.mod h1:hN2rVc/uS4bQhQKTio2XaSJSafJwqBUWWwtssT3cQmc=
|
||||
github.com/quasilyte/go-ruleguard/dsl v0.0.0-20210108021830-2a284c158646 h1:ShKMENtS4KYekn92z5vdicyR7wBOgpxFNj5h0M2rNKg=
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20170806203942-52369c62f446 h1:/NRJ5vAYoqz+7sG51ubIDHXeWO8DlTSrToPu6q11ziA=
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20170806203942-52369c62f446/go.mod h1:uYEyJGbgTkfkS4+E/PavXkNJcbFIpEtjt2B0KDQ5+9M=
|
||||
github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
|
||||
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
|
||||
github.com/rs/xid v1.2.1 h1:mhH9Nq+C1fY2l1XIpgxIiUOfNpRBYH1kKcr+qfKgjRc=
|
||||
github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ=
|
||||
github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q=
|
||||
github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
|
||||
github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
|
||||
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
|
||||
github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
|
||||
github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
|
||||
github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo=
|
||||
github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
|
||||
github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
|
||||
github.com/sirupsen/logrus v1.3.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
|
||||
@ -396,7 +389,6 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+
|
||||
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/objx v0.2.0 h1:Hbg2NidpLE8veEBkEZTL3CvlkUIVzuU9jDplZO54c48=
|
||||
github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=
|
||||
github.com/stretchr/testify v0.0.0-20141015234014-d6577e08ec30/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
|
||||
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
|
||||
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
||||
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
|
||||
@ -405,6 +397,8 @@ github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd
|
||||
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s=
|
||||
github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
|
||||
github.com/tecbot/gorocksdb v0.0.0-20191217155057-f0fad39f321c h1:g+WoO5jjkqGAzHWCjJB1zZfXPIAaDpzXIEJ0eS6B5Ok=
|
||||
github.com/tecbot/gorocksdb v0.0.0-20191217155057-f0fad39f321c/go.mod h1:ahpPrc7HpcfEWDQRZEmnXMzHY03mLDYMCxeDzy46i+8=
|
||||
github.com/tikv/client-go v0.0.0-20200824032810-95774393107b h1:VOG2GkM7RpRrT0St7HIIwCWrc3mVdf+DjcT8r2ucusI=
|
||||
github.com/tikv/client-go v0.0.0-20200824032810-95774393107b/go.mod h1:K0NcdVNrXDq92YPLytsrAwRMyuXi7GZCO6dXNH7OzQc=
|
||||
github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
|
||||
@ -421,14 +415,9 @@ github.com/ugorji/go/codec v0.0.0-20190204201341-e444a5086c43/go.mod h1:iT03XoTw
|
||||
github.com/unrolled/render v1.0.0 h1:XYtvhA3UkpB7PqkvhUFYmpKD55OudoIeygcfus4vcd4=
|
||||
github.com/unrolled/render v1.0.0/go.mod h1:tu82oB5W2ykJRVioYsB+IQKcft7ryBr7w12qMBUPyXg=
|
||||
github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
|
||||
github.com/urfave/cli v1.22.1 h1:+mkCCcOFKPnCmVYVcURKps1Xe+3zP90gSYGNfRkjoIY=
|
||||
github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
|
||||
github.com/urfave/cli v1.22.5 h1:lNq9sAHXK2qfdI8W+GRItjCEkI+2oR4d+MEHy1CKXoU=
|
||||
github.com/urfave/cli v1.22.5/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
|
||||
github.com/valyala/gozstd v1.7.0 h1:Ljh5c9zboqLhwTI33al32R72iCZfn0mCbVGcFWbGwRQ=
|
||||
github.com/valyala/gozstd v1.7.0/go.mod h1:y5Ew47GLlP37EkTB+B4s7r6A5rdaeB7ftbl9zoYiIPQ=
|
||||
github.com/wsxiaoys/terminal v0.0.0-20160513160801-0940f3fc43a0 h1:3UeQBvD0TFrlVjOeLOBz+CPAI8dnbqNSVwUwRrkp7vQ=
|
||||
github.com/wsxiaoys/terminal v0.0.0-20160513160801-0940f3fc43a0/go.mod h1:IXCdmsXIht47RaVFLEdVnh1t+pgYtTAhQGj73kz+2DM=
|
||||
github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8=
|
||||
github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
|
||||
github.com/yahoo/athenz v1.8.55/go.mod h1:G7LLFUH7Z/r4QAB7FfudfuA7Am/eCzO1GlzBhDL6Kv0=
|
||||
@ -582,6 +571,7 @@ golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtn
|
||||
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
|
||||
golang.org/x/tools v0.0.0-20200812195022-5ae4c3c160a0/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
|
||||
golang.org/x/tools v0.0.0-20200825202427-b303f430e36d h1:W07d4xkoAUSNOkOzdzXCdFGxT7o2rW4q8M34tB2i//k=
|
||||
golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
|
||||
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
|
||||
55
internal/kv/rocksdb/cwrapper/CMakeLists.txt
Normal file
55
internal/kv/rocksdb/cwrapper/CMakeLists.txt
Normal file
@ -0,0 +1,55 @@
|
||||
cmake_minimum_required(VERSION 3.14...3.17 FATAL_ERROR)
|
||||
project(wrapper)
|
||||
|
||||
set(CMAKE_CXX_STANDARD 17)
|
||||
set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
|
||||
|
||||
include( ExternalProject )
|
||||
set( ROCKSDB_VERSION "6.15.2" )
|
||||
set( ROCKSDB_SOURCE_URL
|
||||
"https://github.com/facebook/rocksdb/archive/v${ROCKSDB_VERSION}.tar.gz")
|
||||
|
||||
if( CUSTOM_THIRDPARTY_DOWNLOAD_PATH )
|
||||
set( THIRDPARTY_DOWNLOAD_PATH ${CUSTOM_THIRDPARTY_DOWNLOAD_PATH} )
|
||||
else()
|
||||
set( THIRDPARTY_DOWNLOAD_PATH ${CMAKE_BINARY_DIR}/3rdparty_download/download )
|
||||
endif()
|
||||
message( STATUS "Thirdparty downloaded file path: ${THIRDPARTY_DOWNLOAD_PATH}" )
|
||||
|
||||
macro( build_rocksdb )
|
||||
message( STATUS "Building ROCKSDB-${ROCKSDB_VERSION} from source" )
|
||||
|
||||
set( ROCKSDB_CMAKE_ARGS
|
||||
"-DWITH_GFLAGS=OFF"
|
||||
"-DROCKSDB_BUILD_SHARED=OFF"
|
||||
# "-DWITH_SNAPPY=ON"
|
||||
# "-DWITH_LZ4=ON"
|
||||
# "-DWITH_ZSTD=ON"
|
||||
# "-DWITH_BZ2=ON"
|
||||
"-DCMAKE_INSTALL_PREFIX=${CMAKE_CURRENT_BINARY_DIR}"
|
||||
)
|
||||
|
||||
ExternalProject_Add(
|
||||
rocksdb-ep
|
||||
PREFIX ${CMAKE_BINARY_DIR}/3rdparty_download/rocksdb-subbuild
|
||||
BINARY_DIR rocksdb-bin
|
||||
DOWNLOAD_DIR ${THIRDPARTY_DOWNLOAD_PATH}
|
||||
INSTALL_DIR ${CMAKE_CURRENT_BINARY_DIR}
|
||||
URL ${ROCKSDB_SOURCE_URL}
|
||||
URL_MD5 "67f9e04fda62af551dd039c37b1808ac"
|
||||
CMAKE_ARGS ${ROCKSDB_CMAKE_ARGS}
|
||||
${EP_LOG_OPTIONS}
|
||||
)
|
||||
|
||||
ExternalProject_Get_Property( rocksdb-ep INSTALL_DIR )
|
||||
ExternalProject_Get_Property( rocksdb-ep BINARY_DIR )
|
||||
|
||||
if( NOT IS_DIRECTORY ${INSTALL_DIR}/include )
|
||||
file( MAKE_DIRECTORY "${INSTALL_DIR}/include" )
|
||||
endif()
|
||||
|
||||
endmacro()
|
||||
|
||||
build_rocksdb()
|
||||
|
||||
#endif()
|
||||
53
internal/kv/rocksdb/cwrapper/build.sh
Executable file
53
internal/kv/rocksdb/cwrapper/build.sh
Executable file
@ -0,0 +1,53 @@
|
||||
#!/bin/bash
|
||||
|
||||
SOURCE=${BASH_SOURCE[0]}
|
||||
while [ -h $SOURCE ]; do # resolve $SOURCE until the file is no longer a symlink
|
||||
DIR=$( cd -P $( dirname $SOURCE ) && pwd )
|
||||
SOURCE=$(readlink $SOURCE)
|
||||
[[ $SOURCE != /* ]] && SOURCE=$DIR/$SOURCE # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located
|
||||
done
|
||||
DIR=$( cd -P $( dirname $SOURCE ) && pwd )
|
||||
# echo $DIR
|
||||
|
||||
CGO_CFLAGS="-I$(pwd)/output/include"
|
||||
CGO_LDFLAGS="-L$(pwd)/output/lib -l:librocksdb.a -lstdc++ -lm -lz"
|
||||
|
||||
OUTPUT_LIB=${DIR}/output
|
||||
|
||||
if [ -d ${OUTPUT_LIB} ];then
|
||||
rm -rf ${OUTPUT_LIB}
|
||||
fi
|
||||
mkdir ${OUTPUT_LIB}
|
||||
|
||||
BUILD_TYPE="Debug"
|
||||
|
||||
while getopts "t:h:" arg; do
|
||||
case $arg in
|
||||
t)
|
||||
BUILD_TYPE=$OPTARG # BUILD_TYPE
|
||||
;;
|
||||
h) # help
|
||||
echo "-t: build type(default: Debug)
|
||||
-h: help
|
||||
"
|
||||
exit 0
|
||||
;;
|
||||
?)
|
||||
echo "ERROR! unknown argument"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
done
|
||||
echo "BUILD_TYPE: " $BUILD_TYPE
|
||||
|
||||
pushd ${OUTPUT_LIB}
|
||||
CMAKE_CMD="cmake \
|
||||
-DCMAKE_BUILD_TYPE=${BUILD_TYPE} .."
|
||||
|
||||
${CMAKE_CMD}
|
||||
echo ${CMAKE_CMD}
|
||||
|
||||
if [[ ! ${jobs+1} ]]; then
|
||||
jobs=$(nproc)
|
||||
fi
|
||||
make -j ${jobs}
|
||||
128
internal/kv/rocksdb/rocksdb_kv.go
Normal file
128
internal/kv/rocksdb/rocksdb_kv.go
Normal file
@ -0,0 +1,128 @@
|
||||
package rocksdbkv
|
||||
|
||||
import (
|
||||
"github.com/tecbot/gorocksdb"
|
||||
)
|
||||
|
||||
type RocksdbKV struct {
|
||||
opts *gorocksdb.Options
|
||||
db *gorocksdb.DB
|
||||
writeOptions *gorocksdb.WriteOptions
|
||||
readOptions *gorocksdb.ReadOptions
|
||||
name string
|
||||
}
|
||||
|
||||
func NewRocksdbKV(name string) (*RocksdbKV, error) {
|
||||
bbto := gorocksdb.NewDefaultBlockBasedTableOptions()
|
||||
bbto.SetBlockCache(gorocksdb.NewLRUCache(3 << 30))
|
||||
opts := gorocksdb.NewDefaultOptions()
|
||||
opts.SetBlockBasedTableFactory(bbto)
|
||||
opts.SetCreateIfMissing(true)
|
||||
|
||||
ro := gorocksdb.NewDefaultReadOptions()
|
||||
ro.SetFillCache(false)
|
||||
|
||||
wo := gorocksdb.NewDefaultWriteOptions()
|
||||
db, err := gorocksdb.OpenDb(opts, name)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &RocksdbKV{
|
||||
opts: opts,
|
||||
db: db,
|
||||
writeOptions: wo,
|
||||
readOptions: ro,
|
||||
name: name,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (kv *RocksdbKV) Close() {
|
||||
kv.db.Close()
|
||||
}
|
||||
|
||||
func (kv *RocksdbKV) GetName() string {
|
||||
return kv.name
|
||||
}
|
||||
|
||||
func (kv *RocksdbKV) Load(key string) (string, error) {
|
||||
value, err := kv.db.Get(kv.readOptions, []byte(key))
|
||||
defer value.Free()
|
||||
return string(value.Data()), err
|
||||
}
|
||||
|
||||
func (kv *RocksdbKV) LoadWithPrefix(key string) ([]string, []string, error) {
|
||||
iter := kv.db.NewIterator(kv.readOptions)
|
||||
keys := make([]string, 0)
|
||||
values := make([]string, 0)
|
||||
iter.Seek([]byte(key))
|
||||
for ; iter.Valid(); iter.Next() {
|
||||
key := iter.Key()
|
||||
value := iter.Value()
|
||||
keys = append(keys, string(key.Data()))
|
||||
values = append(values, string(value.Data()))
|
||||
key.Free()
|
||||
value.Free()
|
||||
}
|
||||
if err := iter.Err(); err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
return keys, values, nil
|
||||
}
|
||||
|
||||
func (kv *RocksdbKV) MultiLoad(keys []string) ([]string, error) {
|
||||
values := make([]string, 0, len(keys))
|
||||
for _, key := range keys {
|
||||
value, err := kv.db.Get(kv.readOptions, []byte(key))
|
||||
if err != nil {
|
||||
return []string{}, err
|
||||
}
|
||||
values = append(values, string(value.Data()))
|
||||
}
|
||||
return values, nil
|
||||
}
|
||||
|
||||
func (kv *RocksdbKV) Save(key, value string) error {
|
||||
err := kv.db.Put(kv.writeOptions, []byte(key), []byte(value))
|
||||
return err
|
||||
}
|
||||
|
||||
func (kv *RocksdbKV) MultiSave(kvs map[string]string) error {
|
||||
writeBatch := gorocksdb.NewWriteBatch()
|
||||
defer writeBatch.Clear()
|
||||
for k, v := range kvs {
|
||||
writeBatch.Put([]byte(k), []byte(v))
|
||||
}
|
||||
err := kv.db.Write(kv.writeOptions, writeBatch)
|
||||
return err
|
||||
}
|
||||
|
||||
func (kv *RocksdbKV) RemoveWithPrefix(prefix string) error {
|
||||
iter := kv.db.NewIterator(kv.readOptions)
|
||||
iter.Seek([]byte(prefix))
|
||||
for ; iter.Valid(); iter.Next() {
|
||||
key := iter.Key()
|
||||
err := kv.db.Delete(kv.writeOptions, key.Data())
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
if err := iter.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (kv *RocksdbKV) Remove(key string) error {
|
||||
err := kv.db.Delete(kv.writeOptions, []byte(key))
|
||||
return err
|
||||
}
|
||||
|
||||
func (kv *RocksdbKV) MultiRemove(keys []string) error {
|
||||
writeBatch := gorocksdb.NewWriteBatch()
|
||||
defer writeBatch.Clear()
|
||||
for _, key := range keys {
|
||||
writeBatch.Delete([]byte(key))
|
||||
}
|
||||
err := kv.db.Write(kv.writeOptions, writeBatch)
|
||||
return err
|
||||
}
|
||||
54
internal/kv/rocksdb/rocksdb_kv_test.go
Normal file
54
internal/kv/rocksdb/rocksdb_kv_test.go
Normal file
@ -0,0 +1,54 @@
|
||||
package rocksdbkv_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
rocksdbkv "github.com/zilliztech/milvus-distributed/internal/kv/rocksdb"
|
||||
)
|
||||
|
||||
func TestRocksdbKV(t *testing.T) {
|
||||
name := "/tmp/rocksdb"
|
||||
rocksdbKV, err := rocksdbkv.NewRocksdbKV(name)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
defer rocksdbKV.Close()
|
||||
// Need to call RemoveWithPrefix
|
||||
defer rocksdbKV.RemoveWithPrefix("")
|
||||
|
||||
err = rocksdbKV.Save("abc", "123")
|
||||
assert.Nil(t, err)
|
||||
|
||||
err = rocksdbKV.Save("abcd", "1234")
|
||||
assert.Nil(t, err)
|
||||
|
||||
val, err := rocksdbKV.Load("abc")
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, val, "123")
|
||||
|
||||
keys, vals, err := rocksdbKV.LoadWithPrefix("abc")
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, len(keys), len(vals))
|
||||
assert.Equal(t, len(keys), 2)
|
||||
|
||||
assert.Equal(t, keys[0], "abc")
|
||||
assert.Equal(t, keys[1], "abcd")
|
||||
assert.Equal(t, vals[0], "123")
|
||||
assert.Equal(t, vals[1], "1234")
|
||||
|
||||
err = rocksdbKV.Save("key_1", "123")
|
||||
assert.Nil(t, err)
|
||||
err = rocksdbKV.Save("key_2", "456")
|
||||
assert.Nil(t, err)
|
||||
err = rocksdbKV.Save("key_3", "789")
|
||||
assert.Nil(t, err)
|
||||
|
||||
keys = []string{"key_1", "key_2"}
|
||||
vals, err = rocksdbKV.MultiLoad(keys)
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, len(vals), len(keys))
|
||||
assert.Equal(t, vals[0], "123")
|
||||
assert.Equal(t, vals[1], "456")
|
||||
}
|
||||
54
scripts/cwrapper_rocksdb_build.sh
Normal file
54
scripts/cwrapper_rocksdb_build.sh
Normal file
@ -0,0 +1,54 @@
|
||||
#!/bin/bash
|
||||
|
||||
SOURCE=${BASH_SOURCE[0]}
|
||||
while [ -h $SOURCE ]; do # resolve $SOURCE until the file is no longer a symlink
|
||||
DIR=$( cd -P $( dirname $SOURCE ) && pwd )
|
||||
SOURCE=$(readlink $SOURCE)
|
||||
[[ $SOURCE != /* ]] && SOURCE=$DIR/$SOURCE # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located
|
||||
done
|
||||
DIR=$( cd -P $( dirname $SOURCE ) && pwd )
|
||||
# echo $DIR
|
||||
|
||||
SRC_DIR=${DIR}/../internal/kv/rocksdb/cwrapper
|
||||
CGO_CFLAGS="-I$(SRC_DIR)/output/include"
|
||||
CGO_LDFLAGS="-L$(SRC_DIR)/output/lib -l:librocksdb.a -lstdc++ -lm -lz"
|
||||
|
||||
OUTPUT_LIB=${SRC_DIR}/output
|
||||
|
||||
if [ -d ${OUTPUT_LIB} ];then
|
||||
rm -rf ${OUTPUT_LIB}
|
||||
fi
|
||||
mkdir ${OUTPUT_LIB}
|
||||
|
||||
BUILD_TYPE="Debug"
|
||||
|
||||
while getopts "t:h:" arg; do
|
||||
case $arg in
|
||||
t)
|
||||
BUILD_TYPE=$OPTARG # BUILD_TYPE
|
||||
;;
|
||||
h) # help
|
||||
echo "-t: build type(default: Debug)
|
||||
-h: help
|
||||
"
|
||||
exit 0
|
||||
;;
|
||||
?)
|
||||
echo "ERROR! unknown argument"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
done
|
||||
echo "BUILD_TYPE: " $BUILD_TYPE
|
||||
|
||||
pushd ${OUTPUT_LIB}
|
||||
CMAKE_CMD="cmake \
|
||||
-DCMAKE_BUILD_TYPE=${BUILD_TYPE} ${SRC_DIR}"
|
||||
|
||||
${CMAKE_CMD}
|
||||
echo ${CMAKE_CMD}
|
||||
|
||||
if [[ ! ${jobs+1} ]]; then
|
||||
jobs=$(nproc)
|
||||
fi
|
||||
make -j ${jobs}
|
||||
Loading…
x
Reference in New Issue
Block a user