admin/milvus
1
0
Fork 0
mirror of https://gitee.com/milvus-io/milvus.git synced 2025-12-29 06:55:27 +08:00
Code Issues Packages Projects Releases Wiki Activity

Add pulsar message

Browse Source 
Signed-off-by: shengjh <1572099106@qq.com>
This commit is contained in:
shengjh 2020-09-04 15:17:11 +08:00 committed by yefu.chen
parent 5e89722d88
commit e69c19db5f
33 changed files with 18842 additions and 89835 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
proxy/src/CMakeLists.txt
View File

@ -62,6 +62,7 @@ target_link_libraries( server
PUBLIC ${link_lib}
tracing
pulsarStatic
message_client_cpp
${THIRD_PARTY_LIBS}
${BOOST_LIB}
)

37
proxy/src/grpc/cpp_gen.sh
View File

@ -1,18 +1,23 @@
#!/bin/bash
#../../cmake-build-debug/thirdparty/grpc/grpc-build/third_party/protobuf/protoc -I . --grpc_out=./gen-status --plugin=protoc-gen-grpc="../../cmake-build-debug/thirdparty/grpc/grpc-build/grpc_cpp_plugin" status.proto
#
#../../cmake-build-debug/thirdparty/grpc/grpc-build/third_party/protobuf/protoc -I . --cpp_out=./gen-status status.proto
#
#../../cmake-build-debug/thirdparty/grpc/grpc-build/third_party/protobuf/protoc -I . --grpc_out=./gen-milvus --plugin=protoc-gen-grpc="../../cmake-build-debug/thirdparty/grpc/grpc-build/grpc_cpp_plugin" milvus.proto
#
#../../cmake-build-debug/thirdparty/grpc/grpc-build/third_party/protobuf/protoc -I . --cpp_out=./gen-milvus milvus.proto
#
#
#../../cmake-build-debug/thirdparty/grpc/grpc-build/third_party/protobuf/protoc -I . --grpc_out=./gen-milvus --plugin=protoc-gen-grpc="../../cmake-build-debug/thirdparty/grpc/grpc-build/grpc_cpp_plugin" hello.proto
#
#../../cmake-build-debug/thirdparty/grpc/grpc-build/third_party/protobuf/protoc -I . --cpp_out=./gen-milvus hello.proto
#
#../../cmake-build-debug/thirdparty/grpc/grpc-build/third_party/protobuf/protoc -I . --grpc_out=./gen-milvus --plugin=protoc-gen-grpc="../../cmake-build-debug/thirdparty/grpc/grpc-build/grpc_cpp_plugin" master.proto
#
#../../cmake-build-debug/thirdparty/grpc/grpc-build/third_party/protobuf/protoc -I . --cpp_out=./gen-milvus master.proto
../../cmake-build-debug/thirdparty/grpc/grpc-build/third_party/protobuf/protoc -I . --grpc_out=./gen-status --plugin=protoc-gen-grpc="../../cmake-build-debug/thirdparty/grpc/grpc-build/grpc_cpp_plugin" status.proto
../../cmake-build-debug/thirdparty/grpc/grpc-build/third_party/protobuf/protoc -I . --cpp_out=./gen-status status.proto
../../cmake-build-debug/thirdparty/grpc/grpc-build/third_party/protobuf/protoc -I . --grpc_out=./gen-milvus --plugin=protoc-gen-grpc="../../cmake-build-debug/thirdparty/grpc/grpc-build/grpc_cpp_plugin" milvus.proto
../../cmake-build-debug/thirdparty/grpc/grpc-build/third_party/protobuf/protoc -I . --cpp_out=./gen-milvus milvus.proto
../../cmake-build-debug/thirdparty/grpc/grpc-build/third_party/protobuf/protoc -I . --grpc_out=./gen-milvus --plugin=protoc-gen-grpc="../../cmake-build-debug/thirdparty/grpc/grpc-build/grpc_cpp_plugin" hello.proto
../../cmake-build-debug/thirdparty/grpc/grpc-build/third_party/protobuf/protoc -I . --cpp_out=./gen-milvus hello.proto
../../cmake-build-debug/thirdparty/grpc/grpc-build/third_party/protobuf/protoc -I . --grpc_out=./gen-milvus --plugin=protoc-gen-grpc="../../cmake-build-debug/thirdparty/grpc/grpc-build/grpc_cpp_plugin" master.proto
../../cmake-build-debug/thirdparty/grpc/grpc-build/third_party/protobuf/protoc -I . --cpp_out=./gen-milvus master.proto
# TODO: spilt milvus and pulsar proto
../../cmake-build-debug/thirdparty/grpc/grpc-build/third_party/protobuf/protoc -I . --grpc_out=./gen-milvus --plugin=protoc-gen-grpc="../../cmake-build-debug/thirdparty/grpc/grpc-build/grpc_cpp_plugin" suvlim.proto
../../cmake-build-debug/thirdparty/grpc/grpc-build/third_party/protobuf/protoc -I . --cpp_out=./gen-milvus suvlim.proto

16962
proxy/src/grpc/gen-milvus/milvus.pb.cc
View File

File diff suppressed because it is too large Load Diff

11538
proxy/src/grpc/gen-milvus/milvus.pb.h
View File

File diff suppressed because it is too large Load Diff

6
proxy/src/grpc/gen-milvus/milvus.grpc.pb.cc → proxy/src/grpc/gen-milvus/suvlim.grpc.pb.cc
View File

@ -1,9 +1,9 @@
// Generated by the gRPC C++ plugin.
// If you make any local change, they will be lost.
// source: milvus.proto
// source: suvlim.proto
#include "milvus.pb.h"
#include "milvus.grpc.pb.h"
#include "suvlim.pb.h"
#include "suvlim.grpc.pb.h"
#include <functional>
#include <grpcpp/impl/codegen/async_stream.h>

10
proxy/src/grpc/gen-milvus/milvus.grpc.pb.h → proxy/src/grpc/gen-milvus/suvlim.grpc.pb.h
View File

@ -1,10 +1,10 @@
// Generated by the gRPC C++ plugin.
// If you make any local change, they will be lost.
// source: milvus.proto
#ifndef GRPC_milvus_2eproto__INCLUDED
#define GRPC_milvus_2eproto__INCLUDED
// source: suvlim.proto
#ifndef GRPC_suvlim_2eproto__INCLUDED
#define GRPC_suvlim_2eproto__INCLUDED
#include "milvus.pb.h"
#include "suvlim.pb.h"
#include <functional>
#include <grpcpp/impl/codegen/async_generic_service.h>
@ -4553,4 +4553,4 @@ class MilvusService final {
} // namespace milvus
#endif // GRPC_milvus_2eproto__INCLUDED
#endif // GRPC_suvlim_2eproto__INCLUDED

24803
proxy/src/grpc/gen-milvus/suvlim.pb.cc
View File

File diff suppressed because it is too large Load Diff

11687
proxy/src/grpc/gen-milvus/suvlim.pb.h
View File

File diff suppressed because it is too large Load Diff

2
proxy/src/grpc/milvus.proto
View File

@ -1,3 +1,5 @@
// Not used now, please using suvlim.proto
syntax = "proto3";
import "status.proto";

81
proxy/src/grpc/suvlim.proto
View File

@ -1,38 +1,40 @@
syntax = "proto3";
import "status.proto";
package milvus.grpc;
enum ErrorCode {
SUCCESS = 0;
UNEXPECTED_ERROR = 1;
CONNECT_FAILED = 2;
PERMISSION_DENIED = 3;
COLLECTION_NOT_EXISTS = 4;
ILLEGAL_ARGUMENT = 5;
ILLEGAL_DIMENSION = 7;
ILLEGAL_INDEX_TYPE = 8;
ILLEGAL_COLLECTION_NAME = 9;
ILLEGAL_TOPK = 10;
ILLEGAL_ROWRECORD = 11;
ILLEGAL_VECTOR_ID = 12;
ILLEGAL_SEARCH_RESULT = 13;
FILE_NOT_FOUND = 14;
META_FAILED = 15;
CACHE_FAILED = 16;
CANNOT_CREATE_FOLDER = 17;
CANNOT_CREATE_FILE = 18;
CANNOT_DELETE_FOLDER = 19;
CANNOT_DELETE_FILE = 20;
BUILD_INDEX_ERROR = 21;
ILLEGAL_NLIST = 22;
ILLEGAL_METRIC_TYPE = 23;
OUT_OF_MEMORY = 24;
}
message Status {
ErrorCode error_code = 1;
string reason = 2;
}
//
//enum ErrorCode {
// SUCCESS = 0;
// UNEXPECTED_ERROR = 1;
// CONNECT_FAILED = 2;
// PERMISSION_DENIED = 3;
// COLLECTION_NOT_EXISTS = 4;
// ILLEGAL_ARGUMENT = 5;
// ILLEGAL_DIMENSION = 7;
// ILLEGAL_INDEX_TYPE = 8;
// ILLEGAL_COLLECTION_NAME = 9;
// ILLEGAL_TOPK = 10;
// ILLEGAL_ROWRECORD = 11;
// ILLEGAL_VECTOR_ID = 12;
// ILLEGAL_SEARCH_RESULT = 13;
// FILE_NOT_FOUND = 14;
// META_FAILED = 15;
// CACHE_FAILED = 16;
// CANNOT_CREATE_FOLDER = 17;
// CANNOT_CREATE_FILE = 18;
// CANNOT_DELETE_FOLDER = 19;
// CANNOT_DELETE_FILE = 20;
// BUILD_INDEX_ERROR = 21;
// ILLEGAL_NLIST = 22;
// ILLEGAL_METRIC_TYPE = 23;
// OUT_OF_MEMORY = 24;
//}
//
//message Status {
// ErrorCode error_code = 1;
// string reason = 2;
//}
/**
* @brief Field data type
@ -670,6 +672,8 @@ service MilvusService {
enum OpType {
INSERT = 0;
DELETE = 1;
SEARCH = 2;
SEARCH_RESULT = 3;
}
message InsertOrDeleteMsg {
@ -695,14 +699,10 @@ message SearchMsg {
repeated KeyValuePair extra_params = 7;
}
enum SyncType {
READ = 0;
WRITE = 1;
}
message TimeSyncMsg{
int64 peer_Id = 1;
int64 Timestamp = 2;
SyncType sync_type = 3;
int64 ClientId = 1;
int64 Timestamp = 2;
OpType MsgType = 3;
}
message SegmentRecord {
@ -711,6 +711,7 @@ message SegmentRecord {
}
message Key2SegMsg {
int64 client_id = 1;
SegmentRecord records = 2;
int64 client_id = 1;
SegmentRecord records = 2;
OpType msg_type = 3;
}

4
proxy/src/pulsar/message_client/CMakeLists.txt
View File

@ -1,8 +1,10 @@
set(src-cpp
Client.cpp
ClientV2.cpp
Consumer.cpp
Producer.cpp
pb/suvlim.pb.cc)
${PROJECT_SOURCE_DIR}/src/grpc/gen-milvus/suvlim.pb.cc
)
add_library(message_client_cpp SHARED
${src-cpp}

2
proxy/src/pulsar/message_client/Client.cpp
View File

@ -1,6 +1,7 @@
#include "Client.h"
namespace milvus {
namespace message_client {
MsgClient::MsgClient(const std::string &serviceUrl) : pulsar::Client(serviceUrl) {}
@ -8,4 +9,5 @@ MsgClient::MsgClient(const std::string &serviceUrl) : pulsar::Client(serviceUrl)
MsgClient::MsgClient(const std::string &serviceUrl, const pulsar::ClientConfiguration& clientConfiguration)
: pulsar::Client(serviceUrl, clientConfiguration) {}
}
}

2
proxy/src/pulsar/message_client/Client.h
View File

@ -3,6 +3,7 @@
#include "pulsar/Client.h"
#include "pulsar/ClientConfiguration.h"
namespace milvus {
namespace message_client {
using Result = pulsar::Result;
@ -22,3 +23,4 @@ private:
};
}
}

87
proxy/src/pulsar/message_client/ClientV2.cpp Normal file
View File

@ -0,0 +1,87 @@
#include "ClientV2.h"
#include "pulsar/Result.h"
namespace milvus {
namespace {
int64_t gen_channe_id(int64_t uid) {
// TODO: murmur3 hash from pulsar source code
return 0;
}
}
}
namespace milvus::message_client {
MsgClientV2::MsgClientV2(int64_t client_id, std::string &service_url, const pulsar::ClientConfiguration &config)
: client_id_(client_id), service_url_(service_url) {}
Status MsgClientV2::Init(const std::string &mut_topic, const std::string &query_topic,
const std::string &result_topic) {
auto pulsar_client = std::make_shared<pulsar::Client>(service_url_);
mut_producer_ = std::make_shared<MsgProducer>(pulsar_client, mut_topic);
query_producer_ = std::make_shared<MsgProducer>(pulsar_client, mut_topic);
consumer_ = std::make_shared<MsgConsumer>(pulsar_client, "default");
auto result = consumer_->subscribe(result_topic);
if (result != pulsar::Result::ResultOk) {
return Status(SERVER_UNEXPECTED_ERROR, pulsar::strResult(result));
}
return Status::OK();
}
void MsgClientV2::GetQueryResult(int64_t query_id) {
throw std::exception();
}
Status MsgClientV2::SendMutMessage(const milvus::grpc::InsertParam &request) {
// may have retry policy?
auto row_count = request.rows_data_size();
// TODO: Get the segment from master
int64_t segment = 0;
milvus::grpc::InsertOrDeleteMsg mut_msg;
for (auto i = 0; i < row_count; i++) {
mut_msg.set_op(milvus::grpc::OpType::INSERT);
mut_msg.set_uid(GetUniqueQId());
mut_msg.set_client_id(client_id_);
auto channel_id = gen_channe_id(request.entity_id_array(i));
mut_msg.set_channel_id(channel_id);
mut_msg.set_collection_name(request.collection_name());
mut_msg.set_partition_tag(request.partition_tag());
mut_msg.set_segment_id(segment);
mut_msg.mutable_rows_data()->CopyFrom(request.rows_data(i));
mut_msg.mutable_extra_params()->CopyFrom(request.extra_params());
auto result = mut_producer_->send(mut_msg.SerializeAsString());
if (result != pulsar::ResultOk) {
// TODO: error code
return Status(DB_ERROR, pulsar::strResult(result));
}
}
return Status::OK();
}
Status MsgClientV2::SendMutMessage(const milvus::grpc::DeleteByIDParam &request) {
milvus::grpc::InsertOrDeleteMsg mut_msg;
for (auto id: request.id_array()) {
mut_msg.set_op(milvus::grpc::OpType::DELETE);
mut_msg.set_uid(GetUniqueQId());
mut_msg.set_client_id(client_id_);
mut_msg.set_uid(id);
mut_msg.set_collection_name(request.collection_name());
auto result = mut_producer_->send(mut_msg.SerializeAsString());
if (result != pulsar::ResultOk) {
// TODO: error code
return Status(DB_ERROR, pulsar::strResult(result));
}
}
return Status::OK();
}
MsgClientV2::~MsgClientV2() {
mut_producer_->close();
query_producer_->close();
consumer_->close();
}
}

42
proxy/src/pulsar/message_client/ClientV2.h Normal file
View File

@ -0,0 +1,42 @@
#pragma once
#include "src/utils/Status.h"
#include "Producer.h"
#include "Consumer.h"
#include "grpc/gen-milvus/suvlim.pb.h"
namespace milvus::message_client {
class MsgClientV2 {
public:
MsgClientV2(int64_t client_id, std::string &service_url, const pulsar::ClientConfiguration& config = pulsar::ClientConfiguration());
~MsgClientV2();
// When using MsgClient, make sure it init successfully
Status Init(const std::string &mut_topic,
const std::string &query_topic, const std::string &result_topic);
// unpackage batch insert or delete request, and delivery message to pulsar per row
Status SendMutMessage(const milvus::grpc::InsertParam &request);
Status SendMutMessage(const milvus::grpc::DeleteByIDParam &request);
//
Status SendQueryMessage(const milvus::grpc::SearchParam &request);
void GetQueryResult(int64_t query_id);
private:
int64_t GetUniqueQId() {
return q_id_.fetch_add(1);
}
private:
std::atomic<int64_t> q_id_ = 0;
int64_t client_id_;
std::string service_url_;
std::shared_ptr<MsgConsumer> consumer_;
std::shared_ptr<MsgProducer> mut_producer_;
std::shared_ptr<MsgProducer> query_producer_;
};
}

65
proxy/src/pulsar/message_client/Consumer.cpp
View File

@ -1,10 +1,11 @@
#include "Consumer.h"
#include "pb/pulsar.pb.h"
#include "src/grpc/gen-milvus/suvlim.pb.h"
namespace milvus {
namespace message_client {
MsgConsumer::MsgConsumer(std::shared_ptr<MsgClient> &client, std::string subscription_name, const ConsumerConfiguration conf)
MsgConsumer::MsgConsumer(std::shared_ptr<pulsar::Client> &client, std::string subscription_name, const ConsumerConfiguration conf)
:client_(client), config_(conf), subscription_name_(subscription_name){}
Result MsgConsumer::subscribe(const std::string &topic) {
@ -24,35 +25,36 @@ Result MsgConsumer::receive(Message &msg) {
}
std::shared_ptr<void> MsgConsumer::receive_proto(ConsumerType consumer_type) {
Message msg;
receive(msg);
acknowledge(msg);
switch (consumer_type) {
case INSERT: {
pb::InsertMsg insert_msg;
insert_msg.ParseFromString(msg.getDataAsString());
auto message = std::make_shared<pb::InsertMsg>(insert_msg);
return std::shared_ptr<void>(message);
}
case DELETE: {
pb::DeleteMsg delete_msg;
delete_msg.ParseFromString(msg.getDataAsString());
auto message = std::make_shared<pb::DeleteMsg>(delete_msg);
return std::shared_ptr<void>(message);
}
case SEARCH_RESULT: {
pb::SearchResultMsg search_res_msg;
search_res_msg.ParseFromString(msg.getDataAsString());
auto message = std::make_shared<pb::SearchResultMsg>(search_res_msg);
return std::shared_ptr<void>(message);
}
case TEST:
pb::TestData test_msg;
test_msg.ParseFromString(msg.getDataAsString());
auto message = std::make_shared<pb::TestData>(test_msg);
return std::shared_ptr<void>(message);
}
return nullptr;
throw std::exception();
// Message msg;
// receive(msg);
// acknowledge(msg);
// switch (consumer_type) {
// case INSERT: {
// pb::InsertMsg insert_msg;
// insert_msg.ParseFromString(msg.getDataAsString());
// auto message = std::make_shared<pb::InsertMsg>(insert_msg);
// return std::shared_ptr<void>(message);
// }
// case DELETE: {
// pb::DeleteMsg delete_msg;
// delete_msg.ParseFromString(msg.getDataAsString());
// auto message = std::make_shared<pb::DeleteMsg>(delete_msg);
// return std::shared_ptr<void>(message);
// }
// case SEARCH_RESULT: {
// pb::SearchResultMsg search_res_msg;
// search_res_msg.ParseFromString(msg.getDataAsString());
// auto message = std::make_shared<pb::SearchResultMsg>(search_res_msg);
// return std::shared_ptr<void>(message);
// }
// case TEST:
// pb::TestData test_msg;
// test_msg.ParseFromString(msg.getDataAsString());
// auto message = std::make_shared<pb::TestData>(test_msg);
// return std::shared_ptr<void>(message);
// }
// return nullptr;
}
Result MsgConsumer::close() {
@ -64,3 +66,4 @@ Result MsgConsumer::acknowledge(const Message &message) {
}
}
}

6
proxy/src/pulsar/message_client/Consumer.h
View File

@ -3,6 +3,7 @@
#include "pulsar/Consumer.h"
#include "Client.h"
namespace milvus {
namespace message_client {
enum ConsumerType {
@ -17,7 +18,7 @@ using ConsumerConfiguration = pulsar::ConsumerConfiguration;
class MsgConsumer{
public:
MsgConsumer(std::shared_ptr<message_client::MsgClient> &client, std::string consumer_name,
MsgConsumer(std::shared_ptr<pulsar::Client> &client, std::string consumer_name,
const pulsar::ConsumerConfiguration conf = ConsumerConfiguration());
Result subscribe(const std::string& topic);
@ -33,9 +34,10 @@ public:
private:
Consumer consumer_;
std::shared_ptr<MsgClient> client_;
std::shared_ptr<pulsar::Client> client_;
ConsumerConfiguration config_;
std::string subscription_name_;
};
}
}

35
proxy/src/pulsar/message_client/Producer.cpp
View File

@ -1,27 +1,30 @@
#include "Producer.h"
namespace milvus {
namespace message_client {
MsgProducer::MsgProducer(std::shared_ptr<MsgClient> &client, const std::string &topic, const ProducerConfiguration conf) : client_(client), config_(conf){
createProducer(topic);
}
MsgProducer::MsgProducer(std::shared_ptr<pulsar::Client> &client, const std::string &topic,
const ProducerConfiguration& conf) : client_(client), config_(conf) {
createProducer(topic);
}
Result MsgProducer::createProducer(const std::string &topic) {
return client_->createProducer(topic, producer_);
}
Result MsgProducer::createProducer(const std::string &topic) {
return client_->createProducer(topic, producer_);
}
Result MsgProducer::send(const Message &msg) {
return producer_.send(msg);
}
Result MsgProducer::send(const Message &msg) {
return producer_.send(msg);
}
Result MsgProducer::send(const std::string &msg) {
auto pulsar_msg = pulsar::MessageBuilder().setContent(msg).build();
return send(pulsar_msg);
}
Result MsgProducer::send(const std::string &msg) {
auto pulsar_msg = pulsar::MessageBuilder().setContent(msg).build();
return send(pulsar_msg);
}
Result MsgProducer::close() {
return producer_.close();
}
Result MsgProducer::close() {
return producer_.close();
}
}
}

29
proxy/src/pulsar/message_client/Producer.h
View File

@ -2,32 +2,35 @@
#include "pulsar/Producer.h"
#include "Client.h"
#include "pb/suvlim.pb.h"
#include "src/grpc/gen-milvus/suvlim.pb.h"
namespace milvus {
namespace message_client {
using Producer = pulsar::Producer;
using ProducerConfiguration = pulsar::ProducerConfiguration;
class MsgProducer{
public:
MsgProducer(std::shared_ptr<MsgClient> &client, const std::string &topic, const ProducerConfiguration conf = ProducerConfiguration());
class MsgProducer {
public:
MsgProducer(std::shared_ptr<pulsar::Client> &client, const std::string &topic,
const ProducerConfiguration &conf = ProducerConfiguration());
Result createProducer(const std::string& topic);
Result send(const Message& msg);
Result send(const std::string& msg);
Result send(const suvlim::grpc::InsertOrDeleteMsg& msg);
Result send(const suvlim::grpc::SearchMsg& msg);
Result Send(const suvlim::grpc::GetEntityIDsParam)
Result createProducer(const std::string &topic);
Result send(const Message &msg);
Result send(const std::string &msg);
Result send(const milvus::grpc::InsertOrDeleteMsg &msg);
Result send(const milvus::grpc::SearchMsg &msg);
Result Send(const milvus::grpc::GetEntityIDsParam);
Result close();
const Producer&
const Producer &
producer() const { return producer_; }
private:
private:
Producer producer_;
std::shared_ptr<MsgClient> client_;
std::shared_ptr<pulsar::Client> client_;
ProducerConfiguration config_;
};
}
}

4
proxy/src/pulsar/message_client/pb/build.sh
View File

@ -1,4 +0,0 @@
#!/usr/bin/env bash
#../../../../cmake-build-debug/thirdparty/grpc/grpc-build/third_party/protobuf/protoc -I=./ --cpp_out=./ pulsar.proto
protoc -I=./ --cpp_out=./ suvlim.proto

744
proxy/src/pulsar/message_client/pb/milvus.proto
View File

@ -1,744 +0,0 @@
syntax = "proto3";
package pb;
enum ErrorCode {
SUCCESS = 0;
UNEXPECTED_ERROR = 1;
CONNECT_FAILED = 2;
PERMISSION_DENIED = 3;
COLLECTION_NOT_EXISTS = 4;
ILLEGAL_ARGUMENT = 5;
ILLEGAL_DIMENSION = 7;
ILLEGAL_INDEX_TYPE = 8;
ILLEGAL_COLLECTION_NAME = 9;
ILLEGAL_TOPK = 10;
ILLEGAL_ROWRECORD = 11;
ILLEGAL_VECTOR_ID = 12;
ILLEGAL_SEARCH_RESULT = 13;
FILE_NOT_FOUND = 14;
META_FAILED = 15;
CACHE_FAILED = 16;
CANNOT_CREATE_FOLDER = 17;
CANNOT_CREATE_FILE = 18;
CANNOT_DELETE_FOLDER = 19;
CANNOT_DELETE_FILE = 20;
BUILD_INDEX_ERROR = 21;
ILLEGAL_NLIST = 22;
ILLEGAL_METRIC_TYPE = 23;
OUT_OF_MEMORY = 24;
}
message Status {
ErrorCode error_code = 1;
string reason = 2;
}
/**
* @brief Field data type
*/
enum DataType {
NONE = 0;
BOOL = 1;
INT8 = 2;
INT16 = 3;
INT32 = 4;
INT64 = 5;
FLOAT = 10;
DOUBLE = 11;
STRING = 20;
VECTOR_BINARY = 100;
VECTOR_FLOAT = 101;
}
/**
* @brief General usage
*/
message KeyValuePair {
string key = 1;
string value = 2;
}
/**
* @brief Collection name
*/
message CollectionName {
string collection_name = 1;
}
/**
* @brief Collection name list
*/
message CollectionNameList {
Status status = 1;
repeated string collection_names = 2;
}
/**
* @brief Field name
*/
message FieldName {
string collection_name = 1;
string field_name = 2;
}
/**
* @brief Collection mapping
* @extra_params: key-value pair for extra parameters of the collection
* typically usage:
* extra_params["params"] = {segment_row_count: 1000000, auto_id: true}
* Note:
* the segment_row_count specify segment row count limit for merging
* the auto_id = true means entity id is auto-generated by milvus
*/
message Mapping {
Status status = 1;
string collection_name = 2;
repeated FieldParam fields = 3;
repeated KeyValuePair extra_params = 4;
}
/**
* @brief Collection mapping list
*/
message MappingList {
Status status = 1;
repeated Mapping mapping_list = 2;
}
/**
* @brief Parameters of partition
*/
message PartitionParam {
string collection_name = 1;
string tag = 2;
}
/**
* @brief Partition list
*/
message PartitionList {
Status status = 1;
repeated string partition_tag_array = 2;
}
/**
* @brief Vector row record
*/
message VectorRowRecord {
repeated float float_data = 1; //float vector data
bytes binary_data = 2; //binary vector data
}
/**
* @brief Attribute record
*/
message AttrRecord {
repeated int32 int32_value = 1;
repeated int64 int64_value = 2;
repeated float float_value = 3;
repeated double double_value = 4;
}
/**
* @brief Vector records
*/
message VectorRecord {
repeated VectorRowRecord records = 1;
}
/**
* @brief Field values
*/
message FieldValue {
string field_name = 1;
DataType type = 2;
AttrRecord attr_record = 3;
VectorRecord vector_record = 4;
}
/**
* @brief Parameters for insert action
*/
message InsertParam {
string collection_name = 1;
repeated FieldValue fields = 2;
repeated int64 entity_id_array = 3; //optional
string partition_tag = 4;
repeated KeyValuePair extra_params = 5;
}
/**
* @brief Entity ids
*/
message EntityIds {
Status status = 1;
repeated int64 entity_id_array = 2;
}
/**
* @brief Search vector parameters
*/
message VectorParam {
string json = 1;
VectorRecord row_record = 2;
}
/**
* @brief Parameters for search action
* @dsl example:
* {
* "query": {
* "bool": {
* "must": [
* {
* "must":[
* {
* "should": [
* {
* "term": {
* "gender": ["male"]
* }
* },
* {
* "range": {
* "height": {"gte": "170.0", "lte": "180.0"}
* }
* }
* ]
* },
* {
* "must_not": [
* {
* "term": {
* "age": [20, 21, 22, 23, 24, 25]
* }
* },
* {
* "Range": {
* "weight": {"lte": "100"}
* }
* }
* ]
* }
* ]
* },
* {
* "must": [
* {
* "vector": {
* "face_img": {
* "topk": 10,
* "metric_type": "L2",
* "query": [],
* "params": {
* "nprobe": 10
* }
* }
* }
* }
* ]
* }
* ]
* }
* },
* "fields": ["age", "face_img"]
* }
*/
message SearchParam {
string collection_name = 1;
repeated string partition_tag_array = 2;
repeated VectorParam vector_param = 3;
string dsl = 4;
repeated KeyValuePair extra_params = 5;
}
/**
* @brief Parameters for searching in segments
*/
message SearchInSegmentParam {
repeated string file_id_array = 1;
SearchParam search_param = 2;
}
/**
* @brief Entities
*/
message Entities {
Status status = 1;
repeated int64 ids = 2;
repeated bool valid_row = 3;
repeated FieldValue fields = 4;
}
/**
* @brief Query result
*/
message QueryResult {
Status status = 1;
Entities entities = 2;
int64 row_num = 3;
repeated float scores = 4;
repeated float distances = 5;
repeated KeyValuePair extra_params = 6;
}
/**
* @brief Server string Reply
*/
message StringReply {
Status status = 1;
string string_reply = 2;
}
/**
* @brief Server bool Reply
*/
message BoolReply {
Status status = 1;
bool bool_reply = 2;
}
/**
* @brief Return collection row count
*/
message CollectionRowCount {
Status status = 1;
int64 collection_row_count = 2;
}
/**
* @brief Server command parameters
*/
message Command {
string cmd = 1;
}
/**
* @brief Index params
* @collection_name: target collection
* @field_name: target field
* @index_name: a name for index provided by user, unique within this field
* @extra_params: index parameters in json format
* for vector field:
* extra_params["index_type"] = one of the values: FLAT, IVF_LAT, IVF_SQ8, NSGMIX, IVFSQ8H,
* PQ, HNSW, HNSW_SQ8NM, ANNOY
* extra_params["metric_type"] = one of the values: L2, IP, HAMMING, JACCARD, TANIMOTO
* SUBSTRUCTURE, SUPERSTRUCTURE
* extra_params["params"] = extra parameters for index, for example ivflat: {nlist: 2048}
* for structured field:
* extra_params["index_type"] = one of the values: SORTED
*/
message IndexParam {
Status status = 1;
string collection_name = 2;
string field_name = 3;
string index_name = 4;
repeated KeyValuePair extra_params = 5;
}
/**
* @brief Parameters for flush action
*/
message FlushParam {
repeated string collection_name_array = 1;
}
/**
* @brief Parameters for flush action
*/
message CompactParam {
string collection_name = 1;
double threshold = 2;
}
/**
* @brief Parameters for deleting entities by id
*/
message DeleteByIDParam {
string collection_name = 1;
repeated int64 id_array = 2;
}
/**
* @brief Return collection stats
* @json_info: collection stats in json format, typically, the format is like:
* {
* row_count: xxx,
* data_size: xxx,
* partitions: [
* {
* tag: xxx,
* id: xxx,
* row_count: xxx,
* data_size: xxx,
* segments: [
* {
* id: xxx,
* row_count: xxx,
* data_size: xxx,
* files: [
* {
* field: xxx,
* name: xxx,
* index_type: xxx,
* path: xxx,
* data_size: xxx,
* }
* ]
* }
* ]
* }
* ]
* }
*/
message CollectionInfo {
Status status = 1;
string json_info = 2;
}
/**
* @brief Parameters for returning entities id of a segment
*/
message GetEntityIDsParam {
string collection_name = 1;
int64 segment_id = 2;
}
/**
* @brief Entities identity
*/
message EntityIdentity {
string collection_name = 1;
repeated int64 id_array = 2;
repeated string field_names = 3;
}
/********************************************SearchPB interface***************************************************/
/**
* @brief Vector field parameters
*/
message VectorFieldParam {
int64 dimension = 1;
}
/**
* @brief Field type
*/
message FieldType {
oneof value {
DataType data_type = 1;
VectorFieldParam vector_param = 2;
}
}
/**
* @brief Field parameters
*/
message FieldParam {
uint64 id = 1;
string name = 2;
DataType type = 3;
repeated KeyValuePair index_params = 4;
repeated KeyValuePair extra_params = 5;
}
/**
* @brief Vector field record
*/
message VectorFieldRecord {
repeated VectorRowRecord value = 1;
}
///////////////////////////////////////////////////////////////////
message TermQuery {
string field_name = 1;
repeated int64 int_value = 2;
repeated double double_value = 3;
int64 value_num = 4;
float boost = 5;
repeated KeyValuePair extra_params = 6;
}
enum CompareOperator {
LT = 0;
LTE = 1;
EQ = 2;
GT = 3;
GTE = 4;
NE = 5;
}
message CompareExpr {
CompareOperator operator = 1;
string operand = 2;
}
message RangeQuery {
string field_name = 1;
repeated CompareExpr operand = 2;
float boost = 3;
repeated KeyValuePair extra_params = 4;
}
message VectorQuery {
string field_name = 1;
float query_boost = 2;
repeated VectorRowRecord records = 3;
int64 topk = 4;
repeated KeyValuePair extra_params = 5;
}
enum Occur {
INVALID = 0;
MUST = 1;
SHOULD = 2;
MUST_NOT = 3;
}
message BooleanQuery {
Occur occur = 1;
repeated GeneralQuery general_query = 2;
}
message GeneralQuery {
oneof query {
BooleanQuery boolean_query = 1;
TermQuery term_query = 2;
RangeQuery range_query = 3;
VectorQuery vector_query = 4;
}
}
message SearchParamPB {
string collection_name = 1;
repeated string partition_tag_array = 2;
GeneralQuery general_query = 3;
repeated KeyValuePair extra_params = 4;
}
service MilvusService {
/**
* @brief This method is used to create collection
*
* @param CollectionSchema, use to provide collection information to be created.
*
* @return Status
*/
rpc CreateCollection(Mapping) returns (Status){}
/**
* @brief This method is used to test collection existence.
*
* @param CollectionName, collection name is going to be tested.
*
* @return BoolReply
*/
rpc HasCollection(CollectionName) returns (BoolReply) {}
/**
* @brief This method is used to get collection schema.
*
* @param CollectionName, target collection name.
*
* @return CollectionSchema
*/
rpc DescribeCollection(CollectionName) returns (Mapping) {}
/**
* @brief This method is used to get collection schema.
*
* @param CollectionName, target collection name.
*
* @return CollectionRowCount
*/
rpc CountCollection(CollectionName) returns (CollectionRowCount) {}
/**
* @brief This method is used to list all collections.
*
* @param Command, dummy parameter.
*
* @return CollectionNameList
*/
rpc ShowCollections(Command) returns (CollectionNameList) {}
/**
* @brief This method is used to get collection detail information.
*
* @param CollectionName, target collection name.
*
* @return CollectionInfo
*/
rpc ShowCollectionInfo(CollectionName) returns (CollectionInfo) {}
/**
* @brief This method is used to delete collection.
*
* @param CollectionName, collection name is going to be deleted.
*
* @return Status
*/
rpc DropCollection(CollectionName) returns (Status) {}
/**
* @brief This method is used to build index by collection in sync mode.
*
* @param IndexParam, index paramters.
*
* @return Status
*/
rpc CreateIndex(IndexParam) returns (Status) {}
/**
* @brief This method is used to describe index
*
* @param IndexParam, target index.
*
* @return IndexParam
*/
rpc DescribeIndex(IndexParam) returns (IndexParam) {}
/**
* @brief This method is used to drop index
*
* @param IndexParam, target field. if the IndexParam.field_name is empty, will drop all index of the collection
*
* @return Status
*/
rpc DropIndex(IndexParam) returns (Status) {}
/**
* @brief This method is used to create partition
*
* @param PartitionParam, partition parameters.
*
* @return Status
*/
rpc CreatePartition(PartitionParam) returns (Status) {}
/**
* @brief This method is used to test partition existence.
*
* @param PartitionParam, target partition.
*
* @return BoolReply
*/
rpc HasPartition(PartitionParam) returns (BoolReply) {}
/**
* @brief This method is used to show partition information
*
* @param CollectionName, target collection name.
*
* @return PartitionList
*/
rpc ShowPartitions(CollectionName) returns (PartitionList) {}
/**
* @brief This method is used to drop partition
*
* @param PartitionParam, target partition.
*
* @return Status
*/
rpc DropPartition(PartitionParam) returns (Status) {}
/**
* @brief This method is used to add vector array to collection.
*
* @param InsertParam, insert parameters.
*
* @return VectorIds
*/
rpc Insert(InsertParam) returns (EntityIds) {}
/**
* @brief This method is used to get entities data by id array.
*
* @param EntitiesIdentity, target entity id array.
*
* @return EntitiesData
*/
rpc GetEntityByID(EntityIdentity) returns (Entities) {}
/**
* @brief This method is used to get vector ids from a segment
*
* @param GetVectorIDsParam, target collection and segment
*
* @return VectorIds
*/
rpc GetEntityIDs(GetEntityIDsParam) returns (EntityIds) {}
/**
* @brief This method is used to query vector in collection.
*
* @param SearchParam, search parameters.
*
* @return KQueryResult
*/
rpc Search(SearchParam) returns (QueryResult) {}
/**
* @brief This method is used to query vector in specified files.
*
* @param SearchInSegmentParam, target segments to search.
*
* @return TopKQueryResult
*/
rpc SearchInSegment(SearchInSegmentParam) returns (QueryResult) {}
/**
* @brief This method is used to give the server status.
*
* @param Command, command string
*
* @return StringReply
*/
rpc Cmd(Command) returns (StringReply) {}
/**
* @brief This method is used to delete vector by id
*
* @param DeleteByIDParam, delete parameters.
*
* @return status
*/
rpc DeleteByID(DeleteByIDParam) returns (Status) {}
/**
* @brief This method is used to preload collection
*
* @param CollectionName, target collection name.
*
* @return Status
*/
rpc PreloadCollection(CollectionName) returns (Status) {}
/**
* @brief This method is used to flush buffer into storage.
*
* @param FlushParam, flush parameters
*
* @return Status
*/
rpc Flush(FlushParam) returns (Status) {}
/**
* @brief This method is used to compact collection
*
* @param CompactParam, compact parameters
*
* @return Status
*/
rpc Compact(CompactParam) returns (Status) {}
/********************************New Interface********************************************/
rpc SearchPB(SearchParamPB) returns (QueryResult) {}
}

6462
proxy/src/pulsar/message_client/pb/pulsar.pb.cc
View File

File diff suppressed because it is too large Load Diff

3046
proxy/src/pulsar/message_client/pb/pulsar.pb.h
View File

File diff suppressed because it is too large Load Diff

157
proxy/src/pulsar/message_client/pb/pulsar.proto
View File

@ -1,157 +0,0 @@
syntax = "proto3";
package pb;
enum ErrorCode {
SUCCESS = 0;
UNEXPECTED_ERROR = 1;
CONNECT_FAILED = 2;
PERMISSION_DENIED = 3;
COLLECTION_NOT_EXISTS = 4;
ILLEGAL_ARGUMENT = 5;
ILLEGAL_DIMENSION = 7;
ILLEGAL_INDEX_TYPE = 8;
ILLEGAL_COLLECTION_NAME = 9;
ILLEGAL_TOPK = 10;
ILLEGAL_ROWRECORD = 11;
ILLEGAL_VECTOR_ID = 12;
ILLEGAL_SEARCH_RESULT = 13;
FILE_NOT_FOUND = 14;
META_FAILED = 15;
CACHE_FAILED = 16;
CANNOT_CREATE_FOLDER = 17;
CANNOT_CREATE_FILE = 18;
CANNOT_DELETE_FOLDER = 19;
CANNOT_DELETE_FILE = 20;
BUILD_INDEX_ERROR = 21;
ILLEGAL_NLIST = 22;
ILLEGAL_METRIC_TYPE = 23;
OUT_OF_MEMORY = 24;
}
message Status {
ErrorCode error_code = 1;
string reason = 2;
}
enum DataType {
NONE = 0;
BOOL = 1;
INT8 = 2;
INT16 = 3;
INT32 = 4;
INT64 = 5;
FLOAT = 10;
DOUBLE = 11;
STRING = 20;
VECTOR_BINARY = 100;
VECTOR_FLOAT = 101;
}
enum OpType {
Insert = 0;
Delete = 1;
Search = 2;
TimeSync = 3;
Key2Seg = 4;
Statistics = 5;
}
message SegmentRecord {
repeated string seg_info = 1;
}
message VectorRowRecord {
repeated float float_data = 1; //float vector data
bytes binary_data = 2; //binary vector data
}
message AttrRecord {
repeated int32 int32_value = 1;
repeated int64 int64_value = 2;
repeated float float_value = 3;
repeated double double_value = 4;
}
message VectorRecord {
repeated VectorRowRecord records = 1;
}
message VectorParam {
string json = 1;
VectorRecord row_record = 2;
}
message FieldValue {
string field_name = 1;
DataType type = 2;
AttrRecord attr_record = 3;
VectorRecord vector_record = 4;
}
message Cell {
oneof value{
int32 int32_value = 1;
int64 int64_value = 2;
float float_value = 3;
double double_value = 4;
VectorRowRecord vec = 5;
}
}
message RowValue{
repeated Cell cell = 2;
}
message PulsarMessage {
string collection_name = 1;
repeated FieldValue fields = 2;
int64 entity_id = 3;
string partition_tag = 4;
VectorParam vector_param =5;
SegmentRecord segments = 6;
int64 timestamp = 7;
int64 client_id = 8;
OpType msg_type = 9;
string topic_name = 10;
int64 partition_id = 11;
}
//message PulsarMessages {
// string collection_name = 1;
// repeated FieldValue fields = 2;
// repeated int64 entity_id = 3;
// string partition_tag = 4;
// repeated VectorParam vector_param =5;
// repeated SegmentRecord segments = 6;
// repeated int64 timestamp = 7;
// repeated int64 client_id = 8;
// OpType msg_type = 9;
// repeated string topic_name = 10;
// repeated int64 partition_id = 11;
//}
message TestData {
string id = 1;
string name = 2;
}
message InsertMsg {
int64 client_id = 1;
}
message DeleteMsg {
int64 client_id = 1;
}
message SearchMsg {
int64 client_id = 1;
}
message SearchResultMsg {
int64 client_id = 1;
}

21782
proxy/src/pulsar/message_client/pb/suvlim.pb.cc
View File

File diff suppressed because it is too large Load Diff

10343
proxy/src/pulsar/message_client/pb/suvlim.pb.h
View File

File diff suppressed because it is too large Load Diff

715
proxy/src/pulsar/message_client/pb/suvlim.proto
View File

@ -1,715 +0,0 @@
syntax = "proto3";
package suvlim.grpc;
enum ErrorCode {
SUCCESS = 0;
UNEXPECTED_ERROR = 1;
CONNECT_FAILED = 2;
PERMISSION_DENIED = 3;
COLLECTION_NOT_EXISTS = 4;
ILLEGAL_ARGUMENT = 5;
ILLEGAL_DIMENSION = 7;
ILLEGAL_INDEX_TYPE = 8;
ILLEGAL_COLLECTION_NAME = 9;
ILLEGAL_TOPK = 10;
ILLEGAL_ROWRECORD = 11;
ILLEGAL_VECTOR_ID = 12;
ILLEGAL_SEARCH_RESULT = 13;
FILE_NOT_FOUND = 14;
META_FAILED = 15;
CACHE_FAILED = 16;
CANNOT_CREATE_FOLDER = 17;
CANNOT_CREATE_FILE = 18;
CANNOT_DELETE_FOLDER = 19;
CANNOT_DELETE_FILE = 20;
BUILD_INDEX_ERROR = 21;
ILLEGAL_NLIST = 22;
ILLEGAL_METRIC_TYPE = 23;
OUT_OF_MEMORY = 24;
}
message Status {
ErrorCode error_code = 1;
string reason = 2;
}
/**
* @brief Field data type
*/
enum DataType {
NONE = 0;
BOOL = 1;
INT8 = 2;
INT16 = 3;
INT32 = 4;
INT64 = 5;
FLOAT = 10;
DOUBLE = 11;
STRING = 20;
VECTOR_BINARY = 100;
VECTOR_FLOAT = 101;
}
/**
* @brief General usage
*/
message KeyValuePair {
string key = 1;
string value = 2;
}
/**
* @brief Collection name
*/
message CollectionName {
string collection_name = 1;
}
/**
* @brief Collection name list
*/
message CollectionNameList {
Status status = 1;
repeated string collection_names = 2;
}
/**
* @brief Field name
*/
message FieldName {
string collection_name = 1;
string field_name = 2;
}
/**
* @brief Collection mapping
* @extra_params: key-value pair for extra parameters of the collection
* typically usage:
* extra_params["params"] = {segment_row_count: 1000000, auto_id: true}
* Note:
* the segment_row_count specify segment row count limit for merging
* the auto_id = true means entity id is auto-generated by milvus
*/
message Mapping {
Status status = 1;
string collection_name = 2;
repeated FieldParam fields = 3;
repeated KeyValuePair extra_params = 4;
}
/**
* @brief Collection mapping list
*/
message MappingList {
Status status = 1;
repeated Mapping mapping_list = 2;
}
/**
* @brief Parameters of partition
*/
message PartitionParam {
string collection_name = 1;
string tag = 2;
}
/**
* @brief Partition list
*/
message PartitionList {
Status status = 1;
repeated string partition_tag_array = 2;
}
/**
* @brief Vector row record
*/
message VectorRowRecord {
repeated float float_data = 1; //float vector data
bytes binary_data = 2; //binary vector data
}
message EntityIds {
Status status = 1;
repeated int64 entity_id_array = 2;
}
message VectorRecord {
repeated VectorRowRecord records = 1;
}
message VectorParam {
string json = 1;
VectorRecord row_record = 2;
}
//////////////////////////row schema and data///////////////////////////////////
/**
* @brief schema
*/
message FieldMeta {
string field_name = 1;
DataType type = 2;
int64 dim = 3;
}
message Schema {
repeated FieldMeta field_metas = 1;
}
message RowData {
bytes blob = 1;
}
//////////////////////suvlim-proxy///////////////////////////////////
message InsertParam {
string collection_name = 1;
Schema schema = 2;
repeated RowData rows_data = 3;
repeated int64 entity_id_array = 4; //optional
string partition_tag = 5;
repeated KeyValuePair extra_params = 6;
}
message SearchParam {
string collection_name = 1;
repeated VectorParam vector_param = 2;
string dsl = 3; //optional
repeated string partition_tag = 4; //why
repeated KeyValuePair extra_params = 5;
}
message SearchInSegmentParam {
repeated string file_id_array = 1;
SearchParam search_param = 2;
}
message Entities {
Status status = 1;
repeated int64 ids = 2;
repeated bool valid_row = 3;
repeated RowData rows_data = 4;
}
///////////////////////////milvus-server///////////////////////////
/**
* @brief Query result
*/
message QueryResult {
Status status = 1;
Entities entities = 2;
int64 row_num = 3;
repeated float scores = 4;
repeated float distances = 5;
repeated KeyValuePair extra_params = 6;
}
/**
* @brief Server string Reply
*/
message StringReply {
Status status = 1;
string string_reply = 2;
}
/**
* @brief Server bool Reply
*/
message BoolReply {
Status status = 1;
bool bool_reply = 2;
}
/**
* @brief Return collection row count
*/
message CollectionRowCount {
Status status = 1;
int64 collection_row_count = 2;
}
/**
* @brief Server command parameters
*/
message Command {
string cmd = 1;
}
/**
* @brief Index params
* @collection_name: target collection
* @field_name: target field
* @index_name: a name for index provided by user, unique within this field
* @extra_params: index parameters in json format
* for vector field:
* extra_params["index_type"] = one of the values: FLAT, IVF_LAT, IVF_SQ8, NSGMIX, IVFSQ8H,
* PQ, HNSW, HNSW_SQ8NM, ANNOY
* extra_params["metric_type"] = one of the values: L2, IP, HAMMING, JACCARD, TANIMOTO
* SUBSTRUCTURE, SUPERSTRUCTURE
* extra_params["params"] = extra parameters for index, for example ivflat: {nlist: 2048}
* for structured field:
* extra_params["index_type"] = one of the values: SORTED
*/
message IndexParam {
Status status = 1;
string collection_name = 2;
string field_name = 3;
string index_name = 4;
repeated KeyValuePair extra_params = 5;
}
/**
* @brief Parameters for flush action
*/
message FlushParam {
repeated string collection_name_array = 1;
}
/**
* @brief Parameters for flush action
*/
message CompactParam {
string collection_name = 1;
double threshold = 2;
}
/**
* @brief Parameters for deleting entities by id
*/
message DeleteByIDParam {
string collection_name = 1;
repeated int64 id_array = 2;
}
/**
* @brief Return collection stats
* @json_info: collection stats in json format, typically, the format is like:
* {
* row_count: xxx,
* data_size: xxx,
* partitions: [
* {
* tag: xxx,
* id: xxx,
* row_count: xxx,
* data_size: xxx,
* segments: [
* {
* id: xxx,
* row_count: xxx,
* data_size: xxx,
* files: [
* {
* field: xxx,
* name: xxx,
* index_type: xxx,
* path: xxx,
* data_size: xxx,
* }
* ]
* }
* ]
* }
* ]
* }
*/
message CollectionInfo {
Status status = 1;
string json_info = 2;
}
/**
* @brief Parameters for returning entities id of a segment
*/
message GetEntityIDsParam {
string collection_name = 1;
int64 segment_id = 2;
}
/**
* @brief Entities identity
*/
message EntityIdentity {
string collection_name = 1;
repeated int64 id_array = 2;
repeated string field_names = 3;
}
/********************************************SearchPB interface***************************************************/
/**
* @brief Vector field parameters
*/
message VectorFieldParam {
int64 dimension = 1;
}
/**
* @brief Field type
*/
message FieldType {
oneof value {
DataType data_type = 1;
VectorFieldParam vector_param = 2;
}
}
/**
* @brief Field parameters
*/
message FieldParam {
uint64 id = 1;
string name = 2;
DataType type = 3;
repeated KeyValuePair index_params = 4;
repeated KeyValuePair extra_params = 5;
}
/**
* @brief Vector field record
*/
message VectorFieldRecord {
repeated VectorRowRecord value = 1;
}
///////////////////////////////////////////////////////////////////
message TermQuery {
string field_name = 1;
repeated int64 int_value = 2;
repeated double double_value = 3;
int64 value_num = 4;
float boost = 5;
repeated KeyValuePair extra_params = 6;
}
enum CompareOperator {
LT = 0;
LTE = 1;
EQ = 2;
GT = 3;
GTE = 4;
NE = 5;
}
message CompareExpr {
CompareOperator operator = 1;
string operand = 2;
}
message RangeQuery {
string field_name = 1;
repeated CompareExpr operand = 2;
float boost = 3;
repeated KeyValuePair extra_params = 4;
}
message VectorQuery {
string field_name = 1;
float query_boost = 2;
repeated VectorRowRecord records = 3;
int64 topk = 4;
repeated KeyValuePair extra_params = 5;
}
enum Occur {
INVALID = 0;
MUST = 1;
SHOULD = 2;
MUST_NOT = 3;
}
message BooleanQuery {
Occur occur = 1;
repeated GeneralQuery general_query = 2;
}
message GeneralQuery {
oneof query {
BooleanQuery boolean_query = 1;
TermQuery term_query = 2;
RangeQuery range_query = 3;
VectorQuery vector_query = 4;
}
}
message SearchParamPB {
string collection_name = 1;
repeated string partition_tag_array = 2;
GeneralQuery general_query = 3;
repeated KeyValuePair extra_params = 4;
}
service MilvusService {
/**
* @brief This method is used to create collection
*
* @param CollectionSchema, use to provide collection information to be created.
*
* @return Status
*/
rpc CreateCollection(Mapping) returns (Status){}
/**
* @brief This method is used to test collection existence.
*
* @param CollectionName, collection name is going to be tested.
*
* @return BoolReply
*/
rpc HasCollection(CollectionName) returns (BoolReply) {}
/**
* @brief This method is used to get collection schema.
*
* @param CollectionName, target collection name.
*
* @return CollectionSchema
*/
rpc DescribeCollection(CollectionName) returns (Mapping) {}
/**
* @brief This method is used to get collection schema.
*
* @param CollectionName, target collection name.
*
* @return CollectionRowCount
*/
rpc CountCollection(CollectionName) returns (CollectionRowCount) {}
/**
* @brief This method is used to list all collections.
*
* @param Command, dummy parameter.
*
* @return CollectionNameList
*/
rpc ShowCollections(Command) returns (CollectionNameList) {}
/**
* @brief This method is used to get collection detail information.
*
* @param CollectionName, target collection name.
*
* @return CollectionInfo
*/
rpc ShowCollectionInfo(CollectionName) returns (CollectionInfo) {}
/**
* @brief This method is used to delete collection.
*
* @param CollectionName, collection name is going to be deleted.
*
* @return Status
*/
rpc DropCollection(CollectionName) returns (Status) {}
/**
* @brief This method is used to build index by collection in sync mode.
*
* @param IndexParam, index paramters.
*
* @return Status
*/
rpc CreateIndex(IndexParam) returns (Status) {}
/**
* @brief This method is used to describe index
*
* @param IndexParam, target index.
*
* @return IndexParam
*/
rpc DescribeIndex(IndexParam) returns (IndexParam) {}
/**
* @brief This method is used to drop index
*
* @param IndexParam, target field. if the IndexParam.field_name is empty, will drop all index of the collection
*
* @return Status
*/
rpc DropIndex(IndexParam) returns (Status) {}
/**
* @brief This method is used to create partition
*
* @param PartitionParam, partition parameters.
*
* @return Status
*/
rpc CreatePartition(PartitionParam) returns (Status) {}
/**
* @brief This method is used to test partition existence.
*
* @param PartitionParam, target partition.
*
* @return BoolReply
*/
rpc HasPartition(PartitionParam) returns (BoolReply) {}
/**
* @brief This method is used to show partition information
*
* @param CollectionName, target collection name.
*
* @return PartitionList
*/
rpc ShowPartitions(CollectionName) returns (PartitionList) {}
/**
* @brief This method is used to drop partition
*
* @param PartitionParam, target partition.
*
* @return Status
*/
rpc DropPartition(PartitionParam) returns (Status) {}
/**
* @brief This method is used to add vector array to collection.
*
* @param InsertParam, insert parameters.
*
* @return VectorIds
*/
rpc Insert(InsertParam) returns (EntityIds) {}
/**
* @brief This method is used to get entities data by id array.
*
* @param EntitiesIdentity, target entity id array.
*
* @return EntitiesData
*/
rpc GetEntityByID(EntityIdentity) returns (Entities) {}
/**
* @brief This method is used to get vector ids from a segment
*
* @param GetVectorIDsParam, target collection and segment
*
* @return VectorIds
*/
rpc GetEntityIDs(GetEntityIDsParam) returns (EntityIds) {}
/**
* @brief This method is used to query vector in collection.
*
* @param SearchParam, search parameters.
*
* @return KQueryResult
*/
rpc Search(SearchParam) returns (QueryResult) {}
/**
* @brief This method is used to query vector in specified files.
*
* @param SearchInSegmentParam, target segments to search.
*
* @return TopKQueryResult
*/
rpc SearchInSegment(SearchInSegmentParam) returns (QueryResult) {}
/**
* @brief This method is used to give the server status.
*
* @param Command, command string
*
* @return StringReply
*/
rpc Cmd(Command) returns (StringReply) {}
/**
* @brief This method is used to delete vector by id
*
* @param DeleteByIDParam, delete parameters.
*
* @return status
*/
rpc DeleteByID(DeleteByIDParam) returns (Status) {}
/**
* @brief This method is used to preload collection
*
* @param CollectionName, target collection name.
*
* @return Status
*/
rpc PreloadCollection(CollectionName) returns (Status) {}
/**
* @brief This method is used to flush buffer into storage.
*
* @param FlushParam, flush parameters
*
* @return Status
*/
rpc Flush(FlushParam) returns (Status) {}
/**
* @brief This method is used to compact collection
*
* @param CompactParam, compact parameters
*
* @return Status
*/
rpc Compact(CompactParam) returns (Status) {}
/********************************New Interface********************************************/
rpc SearchPB(SearchParamPB) returns (QueryResult) {}
}
////////////////////pulsar//////////////////////////////////////
enum OpType {
INSERT = 0;
DELETE = 1;
SEARCH = 2;
SEARCH_RESULT = 3;
}
message InsertOrDeleteMsg {
string collection_name = 1;
RowData rows_data = 2;
int64 uid = 3; //optional
string partition_tag = 4;
int64 timestamp =5;
int64 segment_id = 6;
int64 channel_id = 7;
OpType op = 8;
int64 client_id = 9;
repeated KeyValuePair extra_params = 10;
}
message SearchMsg {
string collection_name = 1;
VectorRowRecord records = 2;
string partition_tag = 3;
int64 uid = 4;
int64 timestamp =5;
int64 client_id = 6;
repeated KeyValuePair extra_params = 7;
}
message TimeSyncMsg{
int64 ClientId = 1;
int64 Timestamp = 2;
OpType MsgType = 3;
}
message SegmentRecord {
int64 uid = 1;
repeated int64 segment_id = 2;
}
message Key2SegMsg {
int64 client_id = 1;
SegmentRecord records = 2;
OpType msg_type = 3;
}

4
proxy/src/server/CMakeLists.txt
View File

@ -11,8 +11,8 @@
# or implied. See the License for the specific language governing permissions and limitations under the License.
#-------------------------------------------------------------------------------
set( GRPC_SERVICE_FILES ${MILVUS_ENGINE_SRC}/grpc/gen-milvus/milvus.grpc.pb.cc
${MILVUS_ENGINE_SRC}/grpc/gen-milvus/milvus.pb.cc
set( GRPC_SERVICE_FILES ${MILVUS_ENGINE_SRC}/grpc/gen-milvus/suvlim.grpc.pb.cc
${MILVUS_ENGINE_SRC}/grpc/gen-milvus/suvlim.pb.cc
${MILVUS_ENGINE_SRC}/grpc/gen-status/status.grpc.pb.cc
${MILVUS_ENGINE_SRC}/grpc/gen-status/status.pb.cc
${MILVUS_ENGINE_SRC}/grpc/gen-milvus/hello.grpc.pb.cc

2
proxy/src/server/delivery/request/Types.h
View File

@ -12,7 +12,7 @@
#pragma once
#include "db/Types.h"
#include "grpc/gen-milvus/milvus.grpc.pb.h"
#include "grpc/gen-milvus/suvlim.grpc.pb.h"
#include "grpc/gen-status/status.grpc.pb.h"
#include "grpc/gen-status/status.pb.h"
#include "query/GeneralQuery.h"

6
proxy/src/server/grpc_impl/GrpcRequestHandler.cpp
View File

@ -213,6 +213,10 @@ GrpcRequestHandler::GrpcRequestHandler(const std::shared_ptr<opentracing::Tracer
: tracer_(tracer), random_num_generator_() {
std::random_device random_device;
random_num_generator_.seed(random_device());
// TODO: do not hardcode pulsar message configure and init
std::string pulsar_server_address = "pulsar://localhost:6650";
msg_client_ = std::make_shared<message_client::MsgClientV2>(0, pulsar_server_address);
msg_client_->Init("topic-mut", "topic-query", "topic-result");
}
void
@ -722,6 +726,8 @@ GrpcRequestHandler::Insert(::grpc::ServerContext* context, const ::milvus::grpc:
CHECK_NULLPTR_RETURN(request);
LOG_SERVER_INFO_ << LogOut("Request [%s] %s begin.", GetContext(context)->ReqID().c_str(), __func__);
// TODO: add pulsar message proucer
return ::grpc::Status::OK;
}

9
proxy/src/server/grpc_impl/GrpcRequestHandler.h
View File

@ -20,13 +20,16 @@
#include <string>
#include <unordered_map>
#include "query/BinaryQuery.h"
#include "grpc/gen-milvus/milvus.grpc.pb.h"
#include "grpc/gen-milvus/suvlim.grpc.pb.h"
#include "grpc/gen-status/status.pb.h"
#include "opentracing/tracer.h"
#include "server/context/Context.h"
#include "server/delivery/ReqHandler.h"
#include "server/grpc_impl/interceptor/GrpcInterceptorHookHandler.h"
#include "src/utils/Status.h"
#include "pulsar/message_client/Consumer.h"
#include "pulsar/message_client/Producer.h"
#include "pulsar/message_client/ClientV2.h"
namespace milvus {
namespace server {
@ -331,8 +334,12 @@ class GrpcRequestHandler final : public ::milvus::grpc::MilvusService::Service,
ProcessLeafQueryJson(const nlohmann::json& json, query::BooleanQueryPtr& query, std::string& field_name);
private:
// May remove req_handler ?
ReqHandler req_handler_;
// delivery and receive pulsar message
std::shared_ptr<message_client::MsgClientV2> msg_client_;
std::unordered_map<std::string, std::shared_ptr<Context>> context_map_;
std::shared_ptr<opentracing::Tracer> tracer_;

2
proxy/src/server/grpc_impl/GrpcServer.cpp
View File

@ -99,7 +99,7 @@ GrpcServer::StartService() {
HelloService helloService;
builder.RegisterService(&helloService);
// report address to mmaster
// report address to master
auto reportClient = new ReportClient(::grpc::CreateChannel("192.168.2.28:50051",
::grpc::InsecureChannelCredentials()));
auto status = reportClient->ReportAddress();

2
proxy/src/version.h
View File

@ -13,4 +13,4 @@
#define BUILD_TYPE "Debug"
#define BUILD_TIME "2020-09-03 16:41.13"
#define LAST_COMMIT_ID "a97d3e925e993f4a2a7409a9734dcc788f109767"
#define LAST_COMMIT_ID "a97d3e925e993f4a2a7409a9734dcc788f109767"