diff --git a/core/include/segment_c.h b/core/include/segment_c.h index f022c3321c..3dcee14ae6 100644 --- a/core/include/segment_c.h +++ b/core/include/segment_c.h @@ -2,6 +2,7 @@ extern "C" { #endif +#include #include "partition_c.h" typedef void* CSegmentBase; @@ -12,6 +13,8 @@ NewSegment(CPartition partition, unsigned long segment_id); void DeleteSegment(CSegmentBase segment); +////////////////////////////////////////////////////////////////// + int Insert(CSegmentBase c_segment, signed long int size, @@ -34,6 +37,40 @@ Search(CSegmentBase c_segment, long int* result_ids, float* result_distances); +////////////////////////////////////////////////////////////////// + +int +Close(CSegmentBase c_segment); + +bool +IsOpened(CSegmentBase c_segment); + +////////////////////////////////////////////////////////////////// + +long int +GetRowCount(CSegmentBase c_segment); + +long int +GetDeletedCount(CSegmentBase c_segment); + +unsigned long +GetTimeBegin(CSegmentBase c_segment); + +void +SetTimeBegin(CSegmentBase c_segment, unsigned long time_begin); + +unsigned long +GetTimeEnd(CSegmentBase c_segment); + +void +SetTimeEnd(CSegmentBase c_segment, unsigned long time_end); + +unsigned long +GetSegmentId(CSegmentBase c_segment); + +void +SetSegmentId(CSegmentBase c_segment, unsigned long segment_id); + #ifdef __cplusplus } #endif \ No newline at end of file diff --git a/core/src/dog_segment/SegmentBase.h b/core/src/dog_segment/SegmentBase.h index 828eb249c0..8bbdc71684 100644 --- a/core/src/dog_segment/SegmentBase.h +++ b/core/src/dog_segment/SegmentBase.h @@ -92,7 +92,7 @@ class SegmentBase { uint64_t get_segment_id() { return segment_id_; } - uint64_t set_segment_id(uint64_t segment_id) { + void set_segment_id(uint64_t segment_id) { this->segment_id_ = segment_id; } diff --git a/core/src/dog_segment/segment_c.cpp b/core/src/dog_segment/segment_c.cpp index 6c54e5ed84..2ebab0cf0f 100644 --- a/core/src/dog_segment/segment_c.cpp +++ b/core/src/dog_segment/segment_c.cpp @@ -28,6 +28,7 @@ DeleteSegment(CSegmentBase segment) { delete s; } +////////////////////////////////////////////////////////////////// int Insert(CSegmentBase c_segment, @@ -79,3 +80,80 @@ Search(CSegmentBase c_segment, return res.code(); } + +////////////////////////////////////////////////////////////////// + +int +Close(CSegmentBase c_segment) { + auto segment = (milvus::dog_segment::SegmentBase*)c_segment; + auto status = segment->Close(); + return status.code(); +} + + +bool +IsOpened(CSegmentBase c_segment) { + auto segment = (milvus::dog_segment::SegmentBase*)c_segment; + auto status = segment->get_state(); + return status == milvus::dog_segment::SegmentBase::SegmentState::Open; +} + +////////////////////////////////////////////////////////////////// + +long int +GetRowCount(CSegmentBase c_segment) { + auto segment = (milvus::dog_segment::SegmentBase*)c_segment; + auto row_count = segment->get_row_count(); + return row_count; +} + + +long int +GetDeletedCount(CSegmentBase c_segment) { + auto segment = (milvus::dog_segment::SegmentBase*)c_segment; + auto deleted_count = segment->get_deleted_count(); + return deleted_count; +} + + +unsigned long +GetTimeBegin(CSegmentBase c_segment) { + auto segment = (milvus::dog_segment::SegmentBase*)c_segment; + auto time_begin = segment->get_time_begin(); + return time_begin; +} + + +void +SetTimeBegin(CSegmentBase c_segment, unsigned long time_begin) { + auto segment = (milvus::dog_segment::SegmentBase*)c_segment; + segment->set_time_begin(time_begin); +} + + +unsigned long +GetTimeEnd(CSegmentBase c_segment) { + auto segment = (milvus::dog_segment::SegmentBase*)c_segment; + auto time_end = segment->get_time_end(); + return time_end; +} + +void +SetTimeEnd(CSegmentBase c_segment, unsigned long time_end) { + auto segment = (milvus::dog_segment::SegmentBase*)c_segment; + segment->set_time_end(time_end); +} + + +unsigned long +GetSegmentId(CSegmentBase c_segment) { + auto segment = (milvus::dog_segment::SegmentBase*)c_segment; + auto segment_id = segment->get_segment_id(); + return segment_id; +} + +void +SetSegmentId(CSegmentBase c_segment, unsigned long segment_id) { + auto segment = (milvus::dog_segment::SegmentBase*)c_segment; + segment->set_segment_id(segment_id); +} diff --git a/core/src/dog_segment/segment_c.h b/core/src/dog_segment/segment_c.h index f022c3321c..3dcee14ae6 100644 --- a/core/src/dog_segment/segment_c.h +++ b/core/src/dog_segment/segment_c.h @@ -2,6 +2,7 @@ extern "C" { #endif +#include #include "partition_c.h" typedef void* CSegmentBase; @@ -12,6 +13,8 @@ NewSegment(CPartition partition, unsigned long segment_id); void DeleteSegment(CSegmentBase segment); +////////////////////////////////////////////////////////////////// + int Insert(CSegmentBase c_segment, signed long int size, @@ -34,6 +37,40 @@ Search(CSegmentBase c_segment, long int* result_ids, float* result_distances); +////////////////////////////////////////////////////////////////// + +int +Close(CSegmentBase c_segment); + +bool +IsOpened(CSegmentBase c_segment); + +////////////////////////////////////////////////////////////////// + +long int +GetRowCount(CSegmentBase c_segment); + +long int +GetDeletedCount(CSegmentBase c_segment); + +unsigned long +GetTimeBegin(CSegmentBase c_segment); + +void +SetTimeBegin(CSegmentBase c_segment, unsigned long time_begin); + +unsigned long +GetTimeEnd(CSegmentBase c_segment); + +void +SetTimeEnd(CSegmentBase c_segment, unsigned long time_end); + +unsigned long +GetSegmentId(CSegmentBase c_segment); + +void +SetSegmentId(CSegmentBase c_segment, unsigned long segment_id); + #ifdef __cplusplus } #endif \ No newline at end of file diff --git a/core/unittest/test_c_api.cpp b/core/unittest/test_c_api.cpp index 6907e7b37e..55b95285fc 100644 --- a/core/unittest/test_c_api.cpp +++ b/core/unittest/test_c_api.cpp @@ -138,3 +138,143 @@ TEST(CApiTest, SearchTest) { DeletePartition(partition); DeleteSegment(segment); } + + +TEST(CApiTest, IsOpenedTest) { + auto collection_name = "collection0"; + auto schema_tmp_conf = "null_schema"; + auto collection = NewCollection(collection_name, schema_tmp_conf); + auto partition_name = "partition0"; + auto partition = NewPartition(collection, partition_name); + auto segment = NewSegment(partition, 0); + + auto is_opened = IsOpened(segment); + assert(is_opened); + + DeleteCollection(collection); + DeletePartition(partition); + DeleteSegment(segment); +} + + +TEST(CApiTest, CloseTest) { + auto collection_name = "collection0"; + auto schema_tmp_conf = "null_schema"; + auto collection = NewCollection(collection_name, schema_tmp_conf); + auto partition_name = "partition0"; + auto partition = NewPartition(collection, partition_name); + auto segment = NewSegment(partition, 0); + + auto status = Close(segment); + assert(status == 0); + + DeleteCollection(collection); + DeletePartition(partition); + DeleteSegment(segment); +} + + +TEST(CApiTest, GetRowCountTest) { + auto collection_name = "collection0"; + auto schema_tmp_conf = "null_schema"; + auto collection = NewCollection(collection_name, schema_tmp_conf); + auto partition_name = "partition0"; + auto partition = NewPartition(collection, partition_name); + auto segment = NewSegment(partition, 0); + + std::vector raw_data; + std::vector timestamps; + std::vector uids; + int N = 10000; + std::default_random_engine e(67); + for(int i = 0; i < N; ++i) { + uids.push_back(100000 + i); + timestamps.push_back(0); + // append vec + float vec[16]; + for(auto &x: vec) { + x = e() % 2000 * 0.001 - 1.0; + } + raw_data.insert(raw_data.end(), (const char*)std::begin(vec), (const char*)std::end(vec)); + int age = e() % 100; + raw_data.insert(raw_data.end(), (const char*)&age, ((const char*)&age) + sizeof(age)); + } + + auto line_sizeof = (sizeof(int) + sizeof(float) * 16); + + auto res = Insert(segment, N, uids.data(), timestamps.data(), raw_data.data(), (int)line_sizeof, N); + assert(res == 0); + + auto row_count = GetRowCount(segment); + assert(row_count == N); + + DeleteCollection(collection); + DeletePartition(partition); + DeleteSegment(segment); +} + +TEST(CApiTest, GetDeletedCountTest) { + auto collection_name = "collection0"; + auto schema_tmp_conf = "null_schema"; + auto collection = NewCollection(collection_name, schema_tmp_conf); + auto partition_name = "partition0"; + auto partition = NewPartition(collection, partition_name); + auto segment = NewSegment(partition, 0); + + unsigned long delete_primary_keys[] = {100000, 100001, 100002}; + unsigned long delete_timestamps[] = {0, 0, 0}; + + auto del_res = Delete(segment, 1, delete_primary_keys, delete_timestamps); + assert(del_res == 0); + + // TODO: assert(deleted_count == len(delete_primary_keys)) + auto deleted_count = GetDeletedCount(segment); + assert(deleted_count == 0); + + DeleteCollection(collection); + DeletePartition(partition); + DeleteSegment(segment); +} + +TEST(CApiTest, TimeGetterAndSetterTest) { + auto collection_name = "collection0"; + auto schema_tmp_conf = "null_schema"; + auto collection = NewCollection(collection_name, schema_tmp_conf); + auto partition_name = "partition0"; + auto partition = NewPartition(collection, partition_name); + auto segment = NewSegment(partition, 0); + + uint64_t TIME_BEGIN = 100; + uint64_t TIME_END = 200; + + SetTimeBegin(segment, TIME_BEGIN); + auto time_begin = GetTimeBegin(segment); + assert(time_begin == TIME_BEGIN); + + SetTimeEnd(segment, TIME_END); + auto time_end = GetTimeEnd(segment); + assert(time_end == TIME_END); + + DeleteCollection(collection); + DeletePartition(partition); + DeleteSegment(segment); +} + + +TEST(CApiTest, SegmentIDTest) { + auto collection_name = "collection0"; + auto schema_tmp_conf = "null_schema"; + auto collection = NewCollection(collection_name, schema_tmp_conf); + auto partition_name = "partition0"; + auto partition = NewPartition(collection, partition_name); + auto segment = NewSegment(partition, 0); + + uint64_t SEGMENT_ID = 1; + SetSegmentId(segment, SEGMENT_ID); + auto segment_id = GetSegmentId(segment); + assert(segment_id == SEGMENT_ID); + + DeleteCollection(collection); + DeletePartition(partition); + DeleteSegment(segment); +} diff --git a/go.sum b/go.sum index f08f5b1171..1002365542 100644 --- a/go.sum +++ b/go.sum @@ -44,6 +44,8 @@ github.com/apache/pulsar/pulsar-client-go v0.0.0-20200901051823-800681aaa9af h1: github.com/apache/pulsar/pulsar-client-go v0.0.0-20200901051823-800681aaa9af/go.mod h1:QdYxU2iG99VVU6cvoBRkCgkazfJSL9WwPZ20PZR6aUk= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLMYoU8P317H5OQ+Via4RmuPwCS0= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= @@ -51,6 +53,7 @@ github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMn github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa v0.0.1 h1:r3ncXbtIiad9owWu22r8ryYogBEV9NbJykk8k6K+u0w= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= github.com/coreos/bbolt v1.3.2 h1:wZwiHHUieZCquLkDL0B8UhzreNWsPHooDAG3q34zk0s= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.12+incompatible h1:pAWNwdf7QiT1zfaWyqCtNZQWCLByQyA3JrSQyuYAqnQ= @@ -59,10 +62,13 @@ github.com/coreos/etcd v3.3.25+incompatible h1:0GQEw6h3YnuOVdtwygkIfJ+Omx0tZ8/Qk github.com/coreos/etcd v3.3.25+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-semver v0.2.0 h1:3Jm3tLmsgAYcjC+4Up7hJrFBPr+n7rAqYeSw/SZazuY= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20190212144455-93d5ec2c7f76 h1:FE783w8WFh+Rvg+7bZ5g8p7gP4SeVS4AoNwkvazlsBg= github.com/coreos/go-systemd v0.0.0-20190212144455-93d5ec2c7f76/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f h1:lBNOc5arjvs8E5mO2tbpBpLoyyu8B6e44T7hJy6potg= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548/go.mod h1:e6NPNENfs9mPDVNRekM7lKScauxd5kXTr1Mfyig6TDM= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= @@ -71,6 +77,7 @@ github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumC github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw= github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385 h1:clC1lXBpe2kTj2VHdaIu9ajZQe4kcEY9j0NsnDDBZ3o= @@ -80,6 +87,7 @@ github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymF github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= @@ -87,11 +95,16 @@ github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeME github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= +github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/gogo/protobuf v0.0.0-20180717141946-636bf0302bc9/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls= github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef h1:veQD95Isof8w9/WXiA+pa3tz3fJXkt5B7QaRBrM62gk= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -145,6 +158,7 @@ github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hf github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= @@ -153,27 +167,34 @@ github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGa github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/mux v1.7.0 h1:tOSd0UKHQd6urX6ApfOn4XdBMY6Sh1MfxV3kmaazO+U= github.com/gorilla/mux v1.7.0/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= +github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.0 h1:WDFjx/TMzVgy9VdMMQi2K2Emtwi2QcUQsztZ/zLaH/Q= github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/grpc-ecosystem/go-grpc-middleware v1.0.0 h1:Iju5GlWwrvL6UBg4zJJt3btmonfrMlCDdsejg4CZE7c= github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= +github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.8.1 h1:VNUuLKyFcJ5IektwBKcZU4J5GJKEt+Odb8dl1d61BGQ= github.com/grpc-ecosystem/grpc-gateway v1.8.1/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= +github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/jonboulle/clockwork v0.1.0 h1:VKV+ZcuP6l3yW9doeqz6ziZGgcynBVQO+obU0+0hcPo= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= +github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= +github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= @@ -183,6 +204,7 @@ github.com/klauspost/cpuid v1.3.1/go.mod h1:bYW4mA6ZgKPob1/Dlai2LviZJO7KGI3uoWLd github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3 h1:CE8S1cTafDpPvMhIxNJKvHsGVBgn1xWYf1NbHQhywc8= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= @@ -190,6 +212,9 @@ github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/linkedin/goavro v2.1.0+incompatible h1:DV2aUlj2xZiuxQyvag8Dy7zjY69ENjS66bWkSfdpddY= github.com/linkedin/goavro v2.1.0+incompatible/go.mod h1:bBCwI2eGYpUI/4820s67MElg9tdeLbINjLjiM2xZFYM= +github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= +github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/minio/md5-simd v1.1.0 h1:QPfiOqlZH+Cj9teu0t9b1nTBfPbyTl16Of5MeuShdK4= @@ -210,6 +235,8 @@ github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9 github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/montanaflynn/stats v0.5.0 h1:2EkzeTSqBB4V4bJwWrt5gIIrZmpJBcoIRGS2kWLgzmk= github.com/montanaflynn/stats v0.5.0/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= +github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.7.0 h1:WSHQ+IS43OoUrWtD1/bbclrwK8TTH5hzp+umCiuxHgs= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= @@ -237,25 +264,33 @@ github.com/pingcap/pd v2.1.5+incompatible h1:vOLV2tSQdRjjmxaTXtJULoC94dYQOd+6fzn github.com/pingcap/pd v2.1.5+incompatible/go.mod h1:nD3+EoYes4+aNNODO99ES59V83MZSI+dFbhyr667a0E= github.com/pivotal-golang/bytefmt v0.0.0-20200131002437-cf55d5288a48 h1:2JCf+JCLBs7IUZzYdIrSDN+GWYacKOdToIAt5zcga54= github.com/pivotal-golang/bytefmt v0.0.0-20200131002437-cf55d5288a48/go.mod h1:43j3yLP9UiXa0z95/W3hN7yTjoxsQoOll5rrGBgBcnE= +github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.2 h1:awm861/B8OKDd2I/6o1dy3ra4BamzKhYOiGItCeZ740= github.com/prometheus/client_golang v0.9.2/go.mod h1:OsXs2jCmiKlQ1lTBmv21f2mNfw4xf/QclQDMrYNZzcM= +github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4 h1:gQz4mCbXsO+nc9n1hCxHcGA3Zx3Eo+UHZoInFGUIXNM= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/common v0.0.0-20181126121408-4724e9255275 h1:PnBWHBf+6L0jOqq0gIVUe6Yk0/QMZ640k6NvkxcBf+8= github.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= +github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a h1:9a8MnZMP0X2nLJdBg+pBmGgkJlSaKC2KaQmTCk1XDtE= github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/protocolbuffers/protobuf v3.13.0+incompatible h1:omZA3Tuq+U2kJ2uMuqMR9c1VO5qLEgZ19m9878fXNtg= github.com/remyoudompheng/bigfft v0.0.0-20170806203942-52369c62f446/go.mod h1:uYEyJGbgTkfkS4+E/PavXkNJcbFIpEtjt2B0KDQ5+9M= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rs/xid v1.2.1 h1:mhH9Nq+C1fY2l1XIpgxIiUOfNpRBYH1kKcr+qfKgjRc= github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= +github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.3.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.6.0 h1:UBcNElsrwanuuMsnGSlYmtmgbb23qDR5dG+6X6Oo89I= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= @@ -266,6 +301,8 @@ github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:s github.com/soheilhy/cmux v0.1.4 h1:0HKaf1o97UwFjHH9o5XsHUOF+tqmdA7KEzXLpiyaw0E= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= +github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= @@ -278,6 +315,7 @@ github.com/tikv/client-go v0.0.0-20200824032810-95774393107b/go.mod h1:K0NcdVNrX github.com/tikv/pd v1.1.0-beta h1:Ke5jqQ7P1zS9ZbbEW1ZPFmMTQPTzrQEMsCumjVkRh1Y= github.com/tikv/pd v2.1.19+incompatible h1:rqjHqO7t/STke/R2Yz6+lQj6NPA8u7G2Otwqup4K+P8= github.com/tikv/pd v2.1.19+incompatible/go.mod h1:v6C/D7ONC49SgjI4jbGnooSizvijaO/bdIm62DVR4tI= +github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5 h1:LnC5Kc/wtumK+WB441p7ynQJzVuNRJiqddSIE3IlSEQ= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/ugorji/go v1.1.2 h1:JON3E2/GPW2iDNGoSAusl1KDf5TRQ8k8q7Tp097pZGs= @@ -286,6 +324,7 @@ github.com/ugorji/go/codec v0.0.0-20190204201341-e444a5086c43 h1:BasDe+IErOQKrMV github.com/ugorji/go/codec v0.0.0-20190204201341-e444a5086c43/go.mod h1:iT03XoTwV7xq/+UGwKO3UbC1nNNlopQiY61beSdrtOA= github.com/unrolled/render v1.0.0 h1:XYtvhA3UkpB7PqkvhUFYmpKD55OudoIeygcfus4vcd4= github.com/unrolled/render v1.0.0/go.mod h1:tu82oB5W2ykJRVioYsB+IQKcft7ryBr7w12qMBUPyXg= +github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -293,7 +332,9 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= go.etcd.io/bbolt v1.3.2 h1:Z/90sZLPOeCy2PwprqkFa25PdkusRzaj9P8zm/KNyvk= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= +go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= go.etcd.io/etcd v0.5.0-alpha.5 h1:VOolFSo3XgsmnYDLozjvZ6JL6AAwIDu1Yx1y+4EYLDo= +go.etcd.io/etcd v0.5.0-alpha.5.0.20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg= go.etcd.io/etcd v3.3.25+incompatible h1:V1RzkZJj9LqsJRy+TUBgpWSbZXITLB819lstuTFoZOY= go.etcd.io/etcd v3.3.25+incompatible/go.mod h1:yaeTdrJi5lOmYerz05bd8+V7KubZs8YSFZfzsF9A6aI= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= @@ -313,6 +354,7 @@ go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKY go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee h1:0mgffUl7nfd+FpvXMVz4IDEaUSmT1ysygQC7qYo7sG4= go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.12.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= go.uber.org/zap v1.15.0 h1:ZZCA22JRF2gQE5FoNmhmrf7jeJJ2uhqDUNRYKm8dvmM= go.uber.org/zap v1.15.0/go.mod h1:Mb2vm2krFEG5DV0W9qcHBYFtp/Wku1cvYaqPsS/WYfc= @@ -361,6 +403,7 @@ golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73r golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181005035420-146acd28ed58/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -373,6 +416,7 @@ golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -405,6 +449,7 @@ golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -414,6 +459,7 @@ golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -439,6 +485,7 @@ golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c h1:fqgJT0MGcGpPgpWU7VRdRjuArfcOvC4AoJmILihzhDg= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -557,6 +604,7 @@ google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZi google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= @@ -585,6 +633,7 @@ gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLks gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= @@ -616,5 +665,6 @@ honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9 rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= +sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q= -sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= \ No newline at end of file +sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= diff --git a/proto/suvlim.proto b/proto/suvlim.proto new file mode 100644 index 0000000000..6dfcf37c65 --- /dev/null +++ b/proto/suvlim.proto @@ -0,0 +1,715 @@ +syntax = "proto3"; + +package milvus.grpc; + +enum ErrorCode { + SUCCESS = 0; + UNEXPECTED_ERROR = 1; + CONNECT_FAILED = 2; + PERMISSION_DENIED = 3; + COLLECTION_NOT_EXISTS = 4; + ILLEGAL_ARGUMENT = 5; + ILLEGAL_DIMENSION = 7; + ILLEGAL_INDEX_TYPE = 8; + ILLEGAL_COLLECTION_NAME = 9; + ILLEGAL_TOPK = 10; + ILLEGAL_ROWRECORD = 11; + ILLEGAL_VECTOR_ID = 12; + ILLEGAL_SEARCH_RESULT = 13; + FILE_NOT_FOUND = 14; + META_FAILED = 15; + CACHE_FAILED = 16; + CANNOT_CREATE_FOLDER = 17; + CANNOT_CREATE_FILE = 18; + CANNOT_DELETE_FOLDER = 19; + CANNOT_DELETE_FILE = 20; + BUILD_INDEX_ERROR = 21; + ILLEGAL_NLIST = 22; + ILLEGAL_METRIC_TYPE = 23; + OUT_OF_MEMORY = 24; +} + +message Status { + ErrorCode error_code = 1; + string reason = 2; +} + +/** + * @brief Field data type + */ +enum DataType { + NONE = 0; + BOOL = 1; + INT8 = 2; + INT16 = 3; + INT32 = 4; + INT64 = 5; + + FLOAT = 10; + DOUBLE = 11; + + STRING = 20; + + VECTOR_BINARY = 100; + VECTOR_FLOAT = 101; +} + +/** + * @brief General usage + */ +message KeyValuePair { + string key = 1; + string value = 2; +} + +/** + * @brief Collection name + */ +message CollectionName { + string collection_name = 1; +} + +/** + * @brief Collection name list + */ +message CollectionNameList { + Status status = 1; + repeated string collection_names = 2; +} + +/** + * @brief Field name + */ +message FieldName { + string collection_name = 1; + string field_name = 2; +} + +/** + * @brief Collection mapping + * @extra_params: key-value pair for extra parameters of the collection + * typically usage: + * extra_params["params"] = {segment_row_count: 1000000, auto_id: true} + * Note: + * the segment_row_count specify segment row count limit for merging + * the auto_id = true means entity id is auto-generated by milvus + */ +message Mapping { + Status status = 1; + string collection_name = 2; + repeated FieldParam fields = 3; + repeated KeyValuePair extra_params = 4; +} + +/** + * @brief Collection mapping list + */ +message MappingList { + Status status = 1; + repeated Mapping mapping_list = 2; +} + +/** + * @brief Parameters of partition + */ +message PartitionParam { + string collection_name = 1; + string tag = 2; +} + +/** + * @brief Partition list + */ +message PartitionList { + Status status = 1; + repeated string partition_tag_array = 2; +} + +/** + * @brief Vector row record + */ +message VectorRowRecord { + repeated float float_data = 1; //float vector data + bytes binary_data = 2; //binary vector data +} + +message EntityIds { + Status status = 1; + repeated int64 entity_id_array = 2; +} + +message VectorRecord { + repeated VectorRowRecord records = 1; +} + +message VectorParam { + string json = 1; + VectorRecord row_record = 2; +} + +//////////////////////////row schema and data/////////////////////////////////// +/** + * @brief schema + */ +message FieldMeta { + string field_name = 1; + DataType type = 2; + int64 dim = 3; +} + +message Schema { + repeated FieldMeta field_metas = 1; +} + +message RowData { + bytes blob = 1; +} + +//////////////////////suvlim-proxy/////////////////////////////////// +message InsertParam { + string collection_name = 1; + Schema schema = 2; + repeated RowData rows_data = 3; + repeated int64 entity_id_array = 4; //optional + string partition_tag = 5; + repeated KeyValuePair extra_params = 6; +} + +message SearchParam { + string collection_name = 1; + repeated VectorParam vector_param = 2; + string dsl = 3; //optional + repeated string partition_tag = 4; //why + repeated KeyValuePair extra_params = 5; +} + +message SearchInSegmentParam { + repeated string file_id_array = 1; + SearchParam search_param = 2; +} + +message Entities { + Status status = 1; + repeated int64 ids = 2; + repeated bool valid_row = 3; + repeated RowData rows_data = 4; +} + + +///////////////////////////milvus-server/////////////////////////// +/** + * @brief Query result + */ +message QueryResult { + Status status = 1; + Entities entities = 2; + int64 row_num = 3; + repeated float scores = 4; + repeated float distances = 5; + repeated KeyValuePair extra_params = 6; +} + +/** + * @brief Server string Reply + */ +message StringReply { + Status status = 1; + string string_reply = 2; +} + +/** + * @brief Server bool Reply + */ +message BoolReply { + Status status = 1; + bool bool_reply = 2; +} + +/** + * @brief Return collection row count + */ +message CollectionRowCount { + Status status = 1; + int64 collection_row_count = 2; +} + +/** + * @brief Server command parameters + */ +message Command { + string cmd = 1; +} + +/** + * @brief Index params + * @collection_name: target collection + * @field_name: target field + * @index_name: a name for index provided by user, unique within this field + * @extra_params: index parameters in json format + * for vector field: + * extra_params["index_type"] = one of the values: FLAT, IVF_LAT, IVF_SQ8, NSGMIX, IVFSQ8H, + * PQ, HNSW, HNSW_SQ8NM, ANNOY + * extra_params["metric_type"] = one of the values: L2, IP, HAMMING, JACCARD, TANIMOTO + * SUBSTRUCTURE, SUPERSTRUCTURE + * extra_params["params"] = extra parameters for index, for example ivflat: {nlist: 2048} + * for structured field: + * extra_params["index_type"] = one of the values: SORTED + */ +message IndexParam { + Status status = 1; + string collection_name = 2; + string field_name = 3; + string index_name = 4; + repeated KeyValuePair extra_params = 5; +} + +/** + * @brief Parameters for flush action + */ +message FlushParam { + repeated string collection_name_array = 1; +} + +/** + * @brief Parameters for flush action + */ +message CompactParam { + string collection_name = 1; + double threshold = 2; +} + +/** + * @brief Parameters for deleting entities by id + */ +message DeleteByIDParam { + string collection_name = 1; + repeated int64 id_array = 2; +} + +/** + * @brief Return collection stats + * @json_info: collection stats in json format, typically, the format is like: + * { + * row_count: xxx, + * data_size: xxx, + * partitions: [ + * { + * tag: xxx, + * id: xxx, + * row_count: xxx, + * data_size: xxx, + * segments: [ + * { + * id: xxx, + * row_count: xxx, + * data_size: xxx, + * files: [ + * { + * field: xxx, + * name: xxx, + * index_type: xxx, + * path: xxx, + * data_size: xxx, + * } + * ] + * } + * ] + * } + * ] + * } + */ +message CollectionInfo { + Status status = 1; + string json_info = 2; +} + +/** + * @brief Parameters for returning entities id of a segment + */ +message GetEntityIDsParam { + string collection_name = 1; + int64 segment_id = 2; +} + +/** + * @brief Entities identity + */ +message EntityIdentity { + string collection_name = 1; + repeated int64 id_array = 2; + repeated string field_names = 3; +} + +/********************************************SearchPB interface***************************************************/ +/** + * @brief Vector field parameters + */ +message VectorFieldParam { + int64 dimension = 1; +} + +/** + * @brief Field type + */ +message FieldType { + oneof value { + DataType data_type = 1; + VectorFieldParam vector_param = 2; + } +} + +/** + * @brief Field parameters + */ +message FieldParam { + uint64 id = 1; + string name = 2; + DataType type = 3; + repeated KeyValuePair index_params = 4; + repeated KeyValuePair extra_params = 5; +} + +/** + * @brief Vector field record + */ +message VectorFieldRecord { + repeated VectorRowRecord value = 1; +} + +/////////////////////////////////////////////////////////////////// + +message TermQuery { + string field_name = 1; + repeated int64 int_value = 2; + repeated double double_value = 3; + int64 value_num = 4; + float boost = 5; + repeated KeyValuePair extra_params = 6; +} + +enum CompareOperator { + LT = 0; + LTE = 1; + EQ = 2; + GT = 3; + GTE = 4; + NE = 5; +} + +message CompareExpr { + CompareOperator operator = 1; + string operand = 2; +} + +message RangeQuery { + string field_name = 1; + repeated CompareExpr operand = 2; + float boost = 3; + repeated KeyValuePair extra_params = 4; +} + +message VectorQuery { + string field_name = 1; + float query_boost = 2; + repeated VectorRowRecord records = 3; + int64 topk = 4; + repeated KeyValuePair extra_params = 5; +} + +enum Occur { + INVALID = 0; + MUST = 1; + SHOULD = 2; + MUST_NOT = 3; +} + +message BooleanQuery { + Occur occur = 1; + repeated GeneralQuery general_query = 2; +} + +message GeneralQuery { + oneof query { + BooleanQuery boolean_query = 1; + TermQuery term_query = 2; + RangeQuery range_query = 3; + VectorQuery vector_query = 4; + } +} + +message SearchParamPB { + string collection_name = 1; + repeated string partition_tag_array = 2; + GeneralQuery general_query = 3; + repeated KeyValuePair extra_params = 4; +} + +service MilvusService { + /** + * @brief This method is used to create collection + * + * @param CollectionSchema, use to provide collection information to be created. + * + * @return Status + */ + rpc CreateCollection(Mapping) returns (Status){} + + /** + * @brief This method is used to test collection existence. + * + * @param CollectionName, collection name is going to be tested. + * + * @return BoolReply + */ + rpc HasCollection(CollectionName) returns (BoolReply) {} + + /** + * @brief This method is used to get collection schema. + * + * @param CollectionName, target collection name. + * + * @return CollectionSchema + */ + rpc DescribeCollection(CollectionName) returns (Mapping) {} + + /** + * @brief This method is used to get collection schema. + * + * @param CollectionName, target collection name. + * + * @return CollectionRowCount + */ + rpc CountCollection(CollectionName) returns (CollectionRowCount) {} + + /** + * @brief This method is used to list all collections. + * + * @param Command, dummy parameter. + * + * @return CollectionNameList + */ + rpc ShowCollections(Command) returns (CollectionNameList) {} + + /** + * @brief This method is used to get collection detail information. + * + * @param CollectionName, target collection name. + * + * @return CollectionInfo + */ + rpc ShowCollectionInfo(CollectionName) returns (CollectionInfo) {} + + /** + * @brief This method is used to delete collection. + * + * @param CollectionName, collection name is going to be deleted. + * + * @return Status + */ + rpc DropCollection(CollectionName) returns (Status) {} + + /** + * @brief This method is used to build index by collection in sync mode. + * + * @param IndexParam, index paramters. + * + * @return Status + */ + rpc CreateIndex(IndexParam) returns (Status) {} + + /** + * @brief This method is used to describe index + * + * @param IndexParam, target index. + * + * @return IndexParam + */ + rpc DescribeIndex(IndexParam) returns (IndexParam) {} + + /** + * @brief This method is used to drop index + * + * @param IndexParam, target field. if the IndexParam.field_name is empty, will drop all index of the collection + * + * @return Status + */ + rpc DropIndex(IndexParam) returns (Status) {} + + /** + * @brief This method is used to create partition + * + * @param PartitionParam, partition parameters. + * + * @return Status + */ + rpc CreatePartition(PartitionParam) returns (Status) {} + + /** + * @brief This method is used to test partition existence. + * + * @param PartitionParam, target partition. + * + * @return BoolReply + */ + rpc HasPartition(PartitionParam) returns (BoolReply) {} + + /** + * @brief This method is used to show partition information + * + * @param CollectionName, target collection name. + * + * @return PartitionList + */ + rpc ShowPartitions(CollectionName) returns (PartitionList) {} + + /** + * @brief This method is used to drop partition + * + * @param PartitionParam, target partition. + * + * @return Status + */ + rpc DropPartition(PartitionParam) returns (Status) {} + + /** + * @brief This method is used to add vector array to collection. + * + * @param InsertParam, insert parameters. + * + * @return VectorIds + */ + rpc Insert(InsertParam) returns (EntityIds) {} + + /** + * @brief This method is used to get entities data by id array. + * + * @param EntitiesIdentity, target entity id array. + * + * @return EntitiesData + */ + rpc GetEntityByID(EntityIdentity) returns (Entities) {} + + /** + * @brief This method is used to get vector ids from a segment + * + * @param GetVectorIDsParam, target collection and segment + * + * @return VectorIds + */ + rpc GetEntityIDs(GetEntityIDsParam) returns (EntityIds) {} + + /** + * @brief This method is used to query vector in collection. + * + * @param SearchParam, search parameters. + * + * @return KQueryResult + */ + rpc Search(SearchParam) returns (QueryResult) {} + + /** + * @brief This method is used to query vector in specified files. + * + * @param SearchInSegmentParam, target segments to search. + * + * @return TopKQueryResult + */ + rpc SearchInSegment(SearchInSegmentParam) returns (QueryResult) {} + + /** + * @brief This method is used to give the server status. + * + * @param Command, command string + * + * @return StringReply + */ + rpc Cmd(Command) returns (StringReply) {} + + /** + * @brief This method is used to delete vector by id + * + * @param DeleteByIDParam, delete parameters. + * + * @return status + */ + rpc DeleteByID(DeleteByIDParam) returns (Status) {} + + /** + * @brief This method is used to preload collection + * + * @param CollectionName, target collection name. + * + * @return Status + */ + rpc PreloadCollection(CollectionName) returns (Status) {} + + /** + * @brief This method is used to flush buffer into storage. + * + * @param FlushParam, flush parameters + * + * @return Status + */ + rpc Flush(FlushParam) returns (Status) {} + + /** + * @brief This method is used to compact collection + * + * @param CompactParam, compact parameters + * + * @return Status + */ + rpc Compact(CompactParam) returns (Status) {} + + /********************************New Interface********************************************/ + + rpc SearchPB(SearchParamPB) returns (QueryResult) {} +} + +////////////////////pulsar////////////////////////////////////// +enum OpType { + INSERT = 0; + DELETE = 1; + SEARCH = 2; + SEARCH_RESULT = 3; +} + +message InsertOrDeleteMsg { + string collection_name = 1; + RowData rows_data = 2; + int64 uid = 3; //optional + string partition_tag = 4; + int64 timestamp =5; + int64 segment_id = 6; + int64 channel_id = 7; + OpType op = 8; + int64 client_id = 9; + repeated KeyValuePair extra_params = 10; +} + +message SearchMsg { + string collection_name = 1; + VectorRowRecord records = 2; + string partition_tag = 3; + int64 uid = 4; + int64 timestamp =5; + int64 client_id = 6; + repeated KeyValuePair extra_params = 7; +} + +message TimeSyncMsg{ + int64 ClientId = 1; + int64 Timestamp = 2; + OpType MsgType = 3; +} + +message SegmentRecord { + int64 uid = 1; + repeated int64 segment_id = 2; +} + +message Key2SegMsg { + int64 client_id = 1; + SegmentRecord records = 2; + OpType msg_type = 3; +} diff --git a/pulsar/client-go/go_client.go b/pulsar/client-go/go_client.go index 19e727b1f2..e49e697202 100644 --- a/pulsar/client-go/go_client.go +++ b/pulsar/client-go/go_client.go @@ -3,8 +3,8 @@ package client_go import ( "context" "github.com/apache/pulsar/pulsar-client-go/pulsar" + "github.com/czs007/suvlim/pulsar/client-go/schema" "log" - "suvlim/pulsar/client-go/schema" ) var ( diff --git a/pulsar/client-go/query_node.go b/pulsar/client-go/query_node.go index 60b4f737fe..39f781f5bd 100644 --- a/pulsar/client-go/query_node.go +++ b/pulsar/client-go/query_node.go @@ -2,7 +2,7 @@ package client_go import ( "fmt" - "suvlim/pulsar/client-go/schema" + "github.com/czs007/suvlim/pulsar/client-go/schema" "sync" "time" ) @@ -34,7 +34,7 @@ func (qn *QueryNode)ReceiveMessage() { qn.mc.ReceiveMessage() } -func main() { +func queryNodeMain() { mc := MessageClient{} topics := []string{"insert", "delete"} diff --git a/pulsar/client-go/storage_node.go b/pulsar/client-go/storage_node.go index b8a37a8098..4c7c589254 100644 --- a/pulsar/client-go/storage_node.go +++ b/pulsar/client-go/storage_node.go @@ -2,7 +2,7 @@ package client_go import ( "fmt" - "suvlim/pulsar/client-go/schema" + "github.com/czs007/suvlim/pulsar/client-go/schema" "sync" "time" ) @@ -27,7 +27,7 @@ func (wn *WriteNode) doWriteNode(wg sync.WaitGroup) { func (wn *WriteNode) PrepareBatchMsg() { wn.mc.PrepareBatchMsg(JobType(1)) } -func main() { +func storageNodeMain() { mc := MessageClient{} topics := []string{"insert", "delete"} diff --git a/reader/partition.go b/reader/partition.go index aae4c1c088..12c801478d 100644 --- a/reader/partition.go +++ b/reader/partition.go @@ -16,14 +16,15 @@ import "C" type Partition struct { PartitionPtr C.CPartition PartitionName string - Segments []*Segment + OpenedSegments []*Segment + ClosedSegments []*Segment } func (p *Partition) NewSegment(segmentId uint64) *Segment { segmentPtr := C.NewSegment(p.PartitionPtr, C.ulong(segmentId)) var newSegment = &Segment{SegmentPtr: segmentPtr, SegmentId: segmentId} - p.Segments = append(p.Segments, newSegment) + p.OpenedSegments = append(p.OpenedSegments, newSegment) return newSegment } diff --git a/reader/query_node.go b/reader/query_node.go index 842df36122..297af962e6 100644 --- a/reader/query_node.go +++ b/reader/query_node.go @@ -125,17 +125,15 @@ func (node *QueryNode) GetTargetSegment(collectionName *string, partitionTag *st return nil, errors.New("cannot found target partition") } - for _, segment := range targetPartition.Segments { - var segmentStatus = segment.GetStatus() - if segmentStatus == 0 { - return segment, nil - } + for _, segment := range targetPartition.OpenedSegments { + // TODO: add other conditions + return segment, nil } return nil, errors.New("cannot found target segment") } -func (node *QueryNode) GetTimeSync() uint64 { +func (node *QueryNode) GetTSOTime() uint64 { // TODO: Add time sync return 0 } @@ -152,17 +150,22 @@ func (node *QueryNode) InitQueryNodeCollection() { } func (node *QueryNode) SegmentsManagement() { - var timeSync = node.GetTimeSync() + var timeNow = node.GetTSOTime() for _, collection := range node.Collections { for _, partition := range collection.Partitions { - for _, segment := range partition.Segments { + for _, oldSegment := range partition.OpenedSegments { // TODO: check segment status - if timeSync >= segment.SegmentCloseTime { - segment.Close() + if timeNow >= oldSegment.SegmentCloseTime { + // start new segment and add it into partition.OpenedSegments // TODO: add atomic segment id var newSegment = partition.NewSegment(0) - newSegment.SegmentCloseTime = timeSync + SegmentLifetime - partition.Segments = append(partition.Segments, newSegment) + newSegment.SegmentCloseTime = timeNow + SegmentLifetime + partition.OpenedSegments = append(partition.OpenedSegments, newSegment) + + // close old segment and move it into partition.ClosedSegments + // TODO: check status + var _ = oldSegment.Close() + partition.ClosedSegments = append(partition.ClosedSegments, oldSegment) } } } @@ -179,7 +182,7 @@ func (node *QueryNode) SegmentService() { /////////////////////////////////////////////////////////////////////////////////////////////////// func (node *QueryNode) Insert(insertMessages []*schema.InsertMsg, wg *sync.WaitGroup) schema.Status { - var timeSync = node.GetTimeSync() + var timeNow = node.GetTSOTime() var collectionName = insertMessages[0].CollectionName var partitionTag = insertMessages[0].PartitionTag var clientId = insertMessages[0].ClientId @@ -190,7 +193,7 @@ func (node *QueryNode) Insert(insertMessages []*schema.InsertMsg, wg *sync.WaitG var vectorRecords [][]*schema.FieldValue for i, msg := range node.buffer.InsertBuffer { - if msg.Timestamp <= timeSync { + if msg.Timestamp <= timeNow { entityIds = append(entityIds, msg.EntityId) timestamps = append(timestamps, msg.Timestamp) vectorRecords = append(vectorRecords, msg.Fields) @@ -207,7 +210,7 @@ func (node *QueryNode) Insert(insertMessages []*schema.InsertMsg, wg *sync.WaitG } for _, msg := range insertMessages { - if msg.Timestamp <= timeSync { + if msg.Timestamp <= timeNow { entityIds = append(entityIds, msg.EntityId) timestamps = append(timestamps, msg.Timestamp) vectorRecords = append(vectorRecords, msg.Fields) @@ -232,7 +235,7 @@ func (node *QueryNode) Insert(insertMessages []*schema.InsertMsg, wg *sync.WaitG } func (node *QueryNode) Delete(deleteMessages []*schema.DeleteMsg, wg *sync.WaitGroup) schema.Status { - var timeSync = node.GetTimeSync() + var timeNow = node.GetTSOTime() var clientId = deleteMessages[0].ClientId // TODO: prevent Memory copy @@ -240,7 +243,7 @@ func (node *QueryNode) Delete(deleteMessages []*schema.DeleteMsg, wg *sync.WaitG var timestamps []uint64 for i, msg := range node.buffer.DeleteBuffer { - if msg.Timestamp <= timeSync { + if msg.Timestamp <= timeNow { entityIds = append(entityIds, msg.EntityId) timestamps = append(timestamps, msg.Timestamp) node.buffer.validDeleteBuffer[i] = false @@ -256,7 +259,7 @@ func (node *QueryNode) Delete(deleteMessages []*schema.DeleteMsg, wg *sync.WaitG } for _, msg := range deleteMessages { - if msg.Timestamp <= timeSync { + if msg.Timestamp <= timeNow { entityIds = append(entityIds, msg.EntityId) timestamps = append(timestamps, msg.Timestamp) } else { @@ -281,7 +284,7 @@ func (node *QueryNode) Delete(deleteMessages []*schema.DeleteMsg, wg *sync.WaitG } func (node *QueryNode) Search(searchMessages []*schema.SearchMsg, wg *sync.WaitGroup) schema.Status { - var timeSync = node.GetTimeSync() + var timeNow = node.GetTSOTime() var collectionName = searchMessages[0].CollectionName var partitionTag = searchMessages[0].PartitionTag var clientId = searchMessages[0].ClientId @@ -292,7 +295,7 @@ func (node *QueryNode) Search(searchMessages []*schema.SearchMsg, wg *sync.WaitG var timestamps []uint64 for i, msg := range node.buffer.SearchBuffer { - if msg.Timestamp <= timeSync { + if msg.Timestamp <= timeNow { records = append(records, *msg.VectorParam.RowRecord) timestamps = append(timestamps, msg.Timestamp) node.buffer.validSearchBuffer[i] = false @@ -308,7 +311,7 @@ func (node *QueryNode) Search(searchMessages []*schema.SearchMsg, wg *sync.WaitG } for _, msg := range searchMessages { - if msg.Timestamp <= timeSync { + if msg.Timestamp <= timeNow { records = append(records, *msg.VectorParam.RowRecord) timestamps = append(timestamps, msg.Timestamp) } else { diff --git a/reader/segment.go b/reader/segment.go index 5c94919d65..756e5b3342 100644 --- a/reader/segment.go +++ b/reader/segment.go @@ -21,54 +21,113 @@ import ( const SegmentLifetime = 20000 +const ( + SegmentOpened = 0 + SegmentClosed = 1 +) + type Segment struct { SegmentPtr C.CSegmentBase SegmentId uint64 SegmentCloseTime uint64 } -func (s *Segment) GetRowCount() int64 { - // TODO: C type to go type - //return C.GetRowCount(s) - return 0 +func (s *Segment) GetStatus() int { + /*C.IsOpened + bool + IsOpened(CSegmentBase c_segment); + */ + var isOpened = C.IsOpened(s.SegmentPtr) + if isOpened { + return SegmentOpened + } else { + return SegmentClosed + } } -func (s *Segment) GetStatus() int { - // TODO: C type to go type - //return C.GetStatus(s) - return 0 +func (s *Segment) GetSegmentID() uint64 { + /*C.GetSegmentId + unsigned long + GetSegmentId(CSegmentBase c_segment); + */ + var segmentID = C.GetSegmentId(s.SegmentPtr) + return uint64(segmentID) +} + +func (s *Segment) SetSegmentID(segmentID uint64) { + /*C.SetSegmentId + void + SetSegmentId(CSegmentBase c_segment, unsigned long segment_id); + */ + C.SetSegmentId(s.SegmentPtr, C.ulong(segmentID)) } func (s *Segment) GetMaxTimestamp() uint64 { - // TODO: C type to go type - //return C.GetMaxTimestamp(s) - return 0 + /*C.GetTimeEnd + unsigned long + GetTimeEnd(CSegmentBase c_segment); + */ + var maxTimestamp = C.GetTimeEnd(s.SegmentPtr) + return uint64(maxTimestamp) +} + +func (s *Segment) SetMaxTimestamp(maxTimestamp uint64) { + /*C.SetTimeEnd + void + SetTimeEnd(CSegmentBase c_segment, unsigned long time_end); + */ + C.SetTimeEnd(s.SegmentPtr, C.ulong(maxTimestamp)) } func (s *Segment) GetMinTimestamp() uint64 { - // TODO: C type to go type - //return C.GetMinTimestamp(s) - return 0 + /*C.GetTimeBegin + unsigned long + GetTimeBegin(CSegmentBase c_segment); + */ + var minTimestamp = C.GetTimeBegin(s.SegmentPtr) + return uint64(minTimestamp) } -func (s *Segment) GetDeletedCount() uint64 { - // TODO: C type to go type - //return C.GetDeletedCount(s) - return 0 +func (s *Segment) SetMinTimestamp(minTimestamp uint64) { + /*C.SetTimeBegin + void + SetTimeBegin(CSegmentBase c_segment, unsigned long time_begin); + */ + C.SetTimeBegin(s.SegmentPtr, C.ulong(minTimestamp)) } -func (s *Segment) Close() { - // TODO: C type to go type - //C.CloseSegment(s) +func (s *Segment) GetRowCount() int64 { + /*C.GetRowCount + long int + GetRowCount(CSegmentBase c_segment); + */ + var rowCount = C.GetRowCount(s.SegmentPtr) + return int64(rowCount) +} + +func (s *Segment) GetDeletedCount() int64 { + /*C.GetDeletedCount + long int + GetDeletedCount(CSegmentBase c_segment); + */ + var deletedCount = C.GetDeletedCount(s.SegmentPtr) + return int64(deletedCount) +} + +func (s *Segment) Close() error { + /*C.Close + int + Close(CSegmentBase c_segment); + */ + var status = C.Close(s.SegmentPtr) + if status != 0 { + return errors.New("Close segment failed, error code = " + strconv.Itoa(int(status))) + } + return nil } //////////////////////////////////////////////////////////////////////////// func SegmentInsert(segment *Segment, entityIds *[]uint64, timestamps *[]uint64, dataChunk [][]*schema.FieldValue) (ResultEntityIds, error) { - // TODO: remove hard code schema - // auto schema_tmp = std::make_shared(); - // schema_tmp->AddField("fakeVec", DataType::VECTOR_FLOAT, 16); - // schema_tmp->AddField("age", DataType::INT32); - /*C.Insert int Insert(CSegmentBase c_segment, @@ -80,6 +139,10 @@ func SegmentInsert(segment *Segment, entityIds *[]uint64, timestamps *[]uint64, signed long int count); */ + // TODO: remove hard code schema + // auto schema_tmp = std::make_shared(); + // schema_tmp->AddField("fakeVec", DataType::VECTOR_FLOAT, 16); + // schema_tmp->AddField("age", DataType::INT32); // TODO: remove hard code & fake dataChunk const DIM = 4 const N = 3 diff --git a/reader/segment_test.go b/reader/segment_test.go index 11f5f00d96..d4db565420 100644 --- a/reader/segment_test.go +++ b/reader/segment_test.go @@ -71,3 +71,111 @@ func TestSegmentSearch(t *testing.T) { collection.DeletePartition(partition) node.DeleteCollection(collection) } + +func TestSegment_GetStatus(t *testing.T) { + node := NewQueryNode(0, 0) + var collection = node.NewCollection("collection0", "fake schema") + var partition = collection.NewPartition("partition0") + var segment = partition.NewSegment(0) + + var status = segment.GetStatus() + assert.Equal(t, status, SegmentOpened) + + partition.DeleteSegment(segment) + collection.DeletePartition(partition) + node.DeleteCollection(collection) +} + +func TestSegment_Close(t *testing.T) { + node := NewQueryNode(0, 0) + var collection = node.NewCollection("collection0", "fake schema") + var partition = collection.NewPartition("partition0") + var segment = partition.NewSegment(0) + + var err = segment.Close() + assert.NoError(t, err) + + partition.DeleteSegment(segment) + collection.DeletePartition(partition) + node.DeleteCollection(collection) +} + +func TestSegment_GetRowCount(t *testing.T) { + node := NewQueryNode(0, 0) + var collection = node.NewCollection("collection0", "fake schema") + var partition = collection.NewPartition("partition0") + var segment = partition.NewSegment(0) + + ids :=[] uint64{1, 2, 3} + timestamps :=[] uint64 {0, 0, 0} + + var _, err = SegmentInsert(segment, &ids, ×tamps, nil) + assert.NoError(t, err) + + var rowCount = segment.GetRowCount() + assert.Equal(t, rowCount, int64(len(ids))) + + partition.DeleteSegment(segment) + collection.DeletePartition(partition) + node.DeleteCollection(collection) +} + +func TestSegment_GetDeletedCount(t *testing.T) { + node := NewQueryNode(0, 0) + var collection = node.NewCollection("collection0", "fake schema") + var partition = collection.NewPartition("partition0") + var segment = partition.NewSegment(0) + + ids :=[] uint64{1, 2, 3} + timestamps :=[] uint64 {0, 0, 0} + + var _, err = SegmentDelete(segment, &ids, ×tamps) + assert.NoError(t, err) + + var deletedCount = segment.GetDeletedCount() + // TODO: assert.Equal(t, deletedCount, len(ids)) + assert.Equal(t, deletedCount, int64(0)) + + partition.DeleteSegment(segment) + collection.DeletePartition(partition) + node.DeleteCollection(collection) +} + +func TestSegment_TimestampGetterAndSetter(t *testing.T) { + node := NewQueryNode(0, 0) + var collection = node.NewCollection("collection0", "fake schema") + var partition = collection.NewPartition("partition0") + var segment = partition.NewSegment(0) + + const MinTimestamp = 100 + const MaxTimestamp = 200 + + segment.SetMinTimestamp(MinTimestamp) + var minTimestamp = segment.GetMinTimestamp() + assert.Equal(t, minTimestamp, uint64(MinTimestamp)) + + segment.SetMaxTimestamp(MaxTimestamp) + var maxTimestamp = segment.GetMaxTimestamp() + assert.Equal(t, maxTimestamp, uint64(MaxTimestamp)) + + partition.DeleteSegment(segment) + collection.DeletePartition(partition) + node.DeleteCollection(collection) +} + +func TestSegment_SegmentIDGetterAndSetter(t *testing.T) { + node := NewQueryNode(0, 0) + var collection = node.NewCollection("collection0", "fake schema") + var partition = collection.NewPartition("partition0") + var segment = partition.NewSegment(0) + + const SegmentID = 1 + + segment.SetSegmentID(SegmentID) + var segmentID = segment.GetSegmentID() + assert.Equal(t, segmentID, uint64(SegmentID)) + + partition.DeleteSegment(segment) + collection.DeletePartition(partition) + node.DeleteCollection(collection) +}