Former-commit-id: aee405b98f44290051fe022387c42bcb6d68cc76
This commit is contained in:
kun yu 2019-07-22 10:14:23 +08:00
parent 43ad4f0587
commit 48064dfdee
44 changed files with 17211 additions and 4 deletions

View File

@ -8,6 +8,7 @@
aux_source_directory(cache cache_files)
aux_source_directory(config config_files)
aux_source_directory(server server_files)
aux_source_directory(grpcserver grpcserver_files)
aux_source_directory(utils utils_files)
aux_source_directory(db db_files)
aux_source_directory(wrapper wrapper_files)
@ -40,6 +41,10 @@ set(service_files
metrics/SystemInfo.h
server/ThreadPoolServer.cpp
server/ThreadPoolServer.h
grpc/gen-milvus/milvus.grpc.pb.cc
grpc/gen-milvus/milvus.pb.cc
grpc/gen-status/status.grpc.pb.cc
grpc/gen-status/status.pb.cc
)
set(engine_files
@ -63,11 +68,16 @@ include_directories(/usr/include)
include_directories("${CUDA_TOOLKIT_ROOT_DIR}/include")
include_directories(thrift/gen-cpp)
include_directories(/usr/include/mysql)
include_directories(grpc/gen-status)
include_directories(grpc/gen-milvus)
set(third_party_libs
easyloggingpp
sqlite
thrift
grpc
grpc++
grpcpp_channelz
yaml-cpp
libgpufaiss.a
faiss
@ -158,7 +168,8 @@ set(server_libs
add_executable(milvus_server
${config_files}
${server_files}
# ${server_files}
${grpcserver_files}
${utils_files}
${service_files}
${metrics_files}
@ -189,4 +200,5 @@ install(FILES
${CMAKE_BINARY_DIR}/mysqlpp_ep-prefix/src/mysqlpp_ep/lib/${CMAKE_SHARED_LIBRARY_PREFIX}mysqlpp${CMAKE_SHARED_LIBRARY_SUFFIX}.3.2.4
DESTINATION lib) #need to copy libmysqlpp.so
add_subdirectory(sdk)
#add_subdirectory(sdk)
add_subdirectory(grpcsdk)

9
cpp/src/grpc/cpp_gen.sh Normal file
View File

@ -0,0 +1,9 @@
#!/bin/bash
protoc -I . --grpc_out=./gen-status --plugin=protoc-gen-grpc=`which grpc_cpp_plugin` status.proto
protoc -I . --cpp_out=./gen-status status.proto
protoc -I . --grpc_out=./gen-milvus --plugin=protoc-gen-grpc=`which grpc_cpp_plugin` milvus.proto
protoc -I . --cpp_out=./gen-milvus milvus.proto

View File

@ -0,0 +1,405 @@
// Generated by the gRPC C++ plugin.
// If you make any local change, they will be lost.
// source: milvus.proto
#include "milvus.pb.h"
#include "milvus.grpc.pb.h"
#include <functional>
#include <grpcpp/impl/codegen/async_stream.h>
#include <grpcpp/impl/codegen/async_unary_call.h>
#include <grpcpp/impl/codegen/channel_interface.h>
#include <grpcpp/impl/codegen/client_unary_call.h>
#include <grpcpp/impl/codegen/client_callback.h>
#include <grpcpp/impl/codegen/method_handler_impl.h>
#include <grpcpp/impl/codegen/rpc_service_method.h>
#include <grpcpp/impl/codegen/server_callback.h>
#include <grpcpp/impl/codegen/service_type.h>
#include <grpcpp/impl/codegen/sync_stream.h>
namespace milvus {
namespace grpc {
static const char* MilvusService_method_names[] = {
"/milvus.grpc.MilvusService/CreateTable",
"/milvus.grpc.MilvusService/HasTable",
"/milvus.grpc.MilvusService/DropTable",
"/milvus.grpc.MilvusService/BuildIndex",
"/milvus.grpc.MilvusService/InsertVector",
"/milvus.grpc.MilvusService/SearchVector",
"/milvus.grpc.MilvusService/SearchVectorInFiles",
"/milvus.grpc.MilvusService/DescribeTable",
"/milvus.grpc.MilvusService/GetTableRowCount",
"/milvus.grpc.MilvusService/ShowTables",
"/milvus.grpc.MilvusService/Ping",
};
std::unique_ptr< MilvusService::Stub> MilvusService::NewStub(const std::shared_ptr< ::grpc::ChannelInterface>& channel, const ::grpc::StubOptions& options) {
(void)options;
std::unique_ptr< MilvusService::Stub> stub(new MilvusService::Stub(channel));
return stub;
}
MilvusService::Stub::Stub(const std::shared_ptr< ::grpc::ChannelInterface>& channel)
: channel_(channel), rpcmethod_CreateTable_(MilvusService_method_names[0], ::grpc::internal::RpcMethod::NORMAL_RPC, channel)
, rpcmethod_HasTable_(MilvusService_method_names[1], ::grpc::internal::RpcMethod::NORMAL_RPC, channel)
, rpcmethod_DropTable_(MilvusService_method_names[2], ::grpc::internal::RpcMethod::NORMAL_RPC, channel)
, rpcmethod_BuildIndex_(MilvusService_method_names[3], ::grpc::internal::RpcMethod::NORMAL_RPC, channel)
, rpcmethod_InsertVector_(MilvusService_method_names[4], ::grpc::internal::RpcMethod::NORMAL_RPC, channel)
, rpcmethod_SearchVector_(MilvusService_method_names[5], ::grpc::internal::RpcMethod::SERVER_STREAMING, channel)
, rpcmethod_SearchVectorInFiles_(MilvusService_method_names[6], ::grpc::internal::RpcMethod::SERVER_STREAMING, channel)
, rpcmethod_DescribeTable_(MilvusService_method_names[7], ::grpc::internal::RpcMethod::NORMAL_RPC, channel)
, rpcmethod_GetTableRowCount_(MilvusService_method_names[8], ::grpc::internal::RpcMethod::NORMAL_RPC, channel)
, rpcmethod_ShowTables_(MilvusService_method_names[9], ::grpc::internal::RpcMethod::SERVER_STREAMING, channel)
, rpcmethod_Ping_(MilvusService_method_names[10], ::grpc::internal::RpcMethod::NORMAL_RPC, channel)
{}
::grpc::Status MilvusService::Stub::CreateTable(::grpc::ClientContext* context, const ::milvus::grpc::TableSchema& request, ::milvus::Status* response) {
return ::grpc::internal::BlockingUnaryCall(channel_.get(), rpcmethod_CreateTable_, context, request, response);
}
void MilvusService::Stub::experimental_async::CreateTable(::grpc::ClientContext* context, const ::milvus::grpc::TableSchema* request, ::milvus::Status* response, std::function<void(::grpc::Status)> f) {
return ::grpc::internal::CallbackUnaryCall(stub_->channel_.get(), stub_->rpcmethod_CreateTable_, context, request, response, std::move(f));
}
void MilvusService::Stub::experimental_async::CreateTable(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::Status* response, std::function<void(::grpc::Status)> f) {
return ::grpc::internal::CallbackUnaryCall(stub_->channel_.get(), stub_->rpcmethod_CreateTable_, context, request, response, std::move(f));
}
::grpc::ClientAsyncResponseReader< ::milvus::Status>* MilvusService::Stub::AsyncCreateTableRaw(::grpc::ClientContext* context, const ::milvus::grpc::TableSchema& request, ::grpc::CompletionQueue* cq) {
return ::grpc::internal::ClientAsyncResponseReaderFactory< ::milvus::Status>::Create(channel_.get(), cq, rpcmethod_CreateTable_, context, request, true);
}
::grpc::ClientAsyncResponseReader< ::milvus::Status>* MilvusService::Stub::PrepareAsyncCreateTableRaw(::grpc::ClientContext* context, const ::milvus::grpc::TableSchema& request, ::grpc::CompletionQueue* cq) {
return ::grpc::internal::ClientAsyncResponseReaderFactory< ::milvus::Status>::Create(channel_.get(), cq, rpcmethod_CreateTable_, context, request, false);
}
::grpc::Status MilvusService::Stub::HasTable(::grpc::ClientContext* context, const ::milvus::grpc::TableName& request, ::milvus::grpc::BoolReply* response) {
return ::grpc::internal::BlockingUnaryCall(channel_.get(), rpcmethod_HasTable_, context, request, response);
}
void MilvusService::Stub::experimental_async::HasTable(::grpc::ClientContext* context, const ::milvus::grpc::TableName* request, ::milvus::grpc::BoolReply* response, std::function<void(::grpc::Status)> f) {
return ::grpc::internal::CallbackUnaryCall(stub_->channel_.get(), stub_->rpcmethod_HasTable_, context, request, response, std::move(f));
}
void MilvusService::Stub::experimental_async::HasTable(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::BoolReply* response, std::function<void(::grpc::Status)> f) {
return ::grpc::internal::CallbackUnaryCall(stub_->channel_.get(), stub_->rpcmethod_HasTable_, context, request, response, std::move(f));
}
::grpc::ClientAsyncResponseReader< ::milvus::grpc::BoolReply>* MilvusService::Stub::AsyncHasTableRaw(::grpc::ClientContext* context, const ::milvus::grpc::TableName& request, ::grpc::CompletionQueue* cq) {
return ::grpc::internal::ClientAsyncResponseReaderFactory< ::milvus::grpc::BoolReply>::Create(channel_.get(), cq, rpcmethod_HasTable_, context, request, true);
}
::grpc::ClientAsyncResponseReader< ::milvus::grpc::BoolReply>* MilvusService::Stub::PrepareAsyncHasTableRaw(::grpc::ClientContext* context, const ::milvus::grpc::TableName& request, ::grpc::CompletionQueue* cq) {
return ::grpc::internal::ClientAsyncResponseReaderFactory< ::milvus::grpc::BoolReply>::Create(channel_.get(), cq, rpcmethod_HasTable_, context, request, false);
}
::grpc::Status MilvusService::Stub::DropTable(::grpc::ClientContext* context, const ::milvus::grpc::TableName& request, ::milvus::Status* response) {
return ::grpc::internal::BlockingUnaryCall(channel_.get(), rpcmethod_DropTable_, context, request, response);
}
void MilvusService::Stub::experimental_async::DropTable(::grpc::ClientContext* context, const ::milvus::grpc::TableName* request, ::milvus::Status* response, std::function<void(::grpc::Status)> f) {
return ::grpc::internal::CallbackUnaryCall(stub_->channel_.get(), stub_->rpcmethod_DropTable_, context, request, response, std::move(f));
}
void MilvusService::Stub::experimental_async::DropTable(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::Status* response, std::function<void(::grpc::Status)> f) {
return ::grpc::internal::CallbackUnaryCall(stub_->channel_.get(), stub_->rpcmethod_DropTable_, context, request, response, std::move(f));
}
::grpc::ClientAsyncResponseReader< ::milvus::Status>* MilvusService::Stub::AsyncDropTableRaw(::grpc::ClientContext* context, const ::milvus::grpc::TableName& request, ::grpc::CompletionQueue* cq) {
return ::grpc::internal::ClientAsyncResponseReaderFactory< ::milvus::Status>::Create(channel_.get(), cq, rpcmethod_DropTable_, context, request, true);
}
::grpc::ClientAsyncResponseReader< ::milvus::Status>* MilvusService::Stub::PrepareAsyncDropTableRaw(::grpc::ClientContext* context, const ::milvus::grpc::TableName& request, ::grpc::CompletionQueue* cq) {
return ::grpc::internal::ClientAsyncResponseReaderFactory< ::milvus::Status>::Create(channel_.get(), cq, rpcmethod_DropTable_, context, request, false);
}
::grpc::Status MilvusService::Stub::BuildIndex(::grpc::ClientContext* context, const ::milvus::grpc::TableName& request, ::milvus::Status* response) {
return ::grpc::internal::BlockingUnaryCall(channel_.get(), rpcmethod_BuildIndex_, context, request, response);
}
void MilvusService::Stub::experimental_async::BuildIndex(::grpc::ClientContext* context, const ::milvus::grpc::TableName* request, ::milvus::Status* response, std::function<void(::grpc::Status)> f) {
return ::grpc::internal::CallbackUnaryCall(stub_->channel_.get(), stub_->rpcmethod_BuildIndex_, context, request, response, std::move(f));
}
void MilvusService::Stub::experimental_async::BuildIndex(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::Status* response, std::function<void(::grpc::Status)> f) {
return ::grpc::internal::CallbackUnaryCall(stub_->channel_.get(), stub_->rpcmethod_BuildIndex_, context, request, response, std::move(f));
}
::grpc::ClientAsyncResponseReader< ::milvus::Status>* MilvusService::Stub::AsyncBuildIndexRaw(::grpc::ClientContext* context, const ::milvus::grpc::TableName& request, ::grpc::CompletionQueue* cq) {
return ::grpc::internal::ClientAsyncResponseReaderFactory< ::milvus::Status>::Create(channel_.get(), cq, rpcmethod_BuildIndex_, context, request, true);
}
::grpc::ClientAsyncResponseReader< ::milvus::Status>* MilvusService::Stub::PrepareAsyncBuildIndexRaw(::grpc::ClientContext* context, const ::milvus::grpc::TableName& request, ::grpc::CompletionQueue* cq) {
return ::grpc::internal::ClientAsyncResponseReaderFactory< ::milvus::Status>::Create(channel_.get(), cq, rpcmethod_BuildIndex_, context, request, false);
}
::grpc::Status MilvusService::Stub::InsertVector(::grpc::ClientContext* context, const ::milvus::grpc::InsertInfos& request, ::milvus::grpc::VectorIds* response) {
return ::grpc::internal::BlockingUnaryCall(channel_.get(), rpcmethod_InsertVector_, context, request, response);
}
void MilvusService::Stub::experimental_async::InsertVector(::grpc::ClientContext* context, const ::milvus::grpc::InsertInfos* request, ::milvus::grpc::VectorIds* response, std::function<void(::grpc::Status)> f) {
return ::grpc::internal::CallbackUnaryCall(stub_->channel_.get(), stub_->rpcmethod_InsertVector_, context, request, response, std::move(f));
}
void MilvusService::Stub::experimental_async::InsertVector(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::VectorIds* response, std::function<void(::grpc::Status)> f) {
return ::grpc::internal::CallbackUnaryCall(stub_->channel_.get(), stub_->rpcmethod_InsertVector_, context, request, response, std::move(f));
}
::grpc::ClientAsyncResponseReader< ::milvus::grpc::VectorIds>* MilvusService::Stub::AsyncInsertVectorRaw(::grpc::ClientContext* context, const ::milvus::grpc::InsertInfos& request, ::grpc::CompletionQueue* cq) {
return ::grpc::internal::ClientAsyncResponseReaderFactory< ::milvus::grpc::VectorIds>::Create(channel_.get(), cq, rpcmethod_InsertVector_, context, request, true);
}
::grpc::ClientAsyncResponseReader< ::milvus::grpc::VectorIds>* MilvusService::Stub::PrepareAsyncInsertVectorRaw(::grpc::ClientContext* context, const ::milvus::grpc::InsertInfos& request, ::grpc::CompletionQueue* cq) {
return ::grpc::internal::ClientAsyncResponseReaderFactory< ::milvus::grpc::VectorIds>::Create(channel_.get(), cq, rpcmethod_InsertVector_, context, request, false);
}
::grpc::ClientReader< ::milvus::grpc::TopKQueryResult>* MilvusService::Stub::SearchVectorRaw(::grpc::ClientContext* context, const ::milvus::grpc::SearchVectorInfos& request) {
return ::grpc::internal::ClientReaderFactory< ::milvus::grpc::TopKQueryResult>::Create(channel_.get(), rpcmethod_SearchVector_, context, request);
}
void MilvusService::Stub::experimental_async::SearchVector(::grpc::ClientContext* context, ::milvus::grpc::SearchVectorInfos* request, ::grpc::experimental::ClientReadReactor< ::milvus::grpc::TopKQueryResult>* reactor) {
::grpc::internal::ClientCallbackReaderFactory< ::milvus::grpc::TopKQueryResult>::Create(stub_->channel_.get(), stub_->rpcmethod_SearchVector_, context, request, reactor);
}
::grpc::ClientAsyncReader< ::milvus::grpc::TopKQueryResult>* MilvusService::Stub::AsyncSearchVectorRaw(::grpc::ClientContext* context, const ::milvus::grpc::SearchVectorInfos& request, ::grpc::CompletionQueue* cq, void* tag) {
return ::grpc::internal::ClientAsyncReaderFactory< ::milvus::grpc::TopKQueryResult>::Create(channel_.get(), cq, rpcmethod_SearchVector_, context, request, true, tag);
}
::grpc::ClientAsyncReader< ::milvus::grpc::TopKQueryResult>* MilvusService::Stub::PrepareAsyncSearchVectorRaw(::grpc::ClientContext* context, const ::milvus::grpc::SearchVectorInfos& request, ::grpc::CompletionQueue* cq) {
return ::grpc::internal::ClientAsyncReaderFactory< ::milvus::grpc::TopKQueryResult>::Create(channel_.get(), cq, rpcmethod_SearchVector_, context, request, false, nullptr);
}
::grpc::ClientReader< ::milvus::grpc::TopKQueryResult>* MilvusService::Stub::SearchVectorInFilesRaw(::grpc::ClientContext* context, const ::milvus::grpc::SearchVectorInFilesInfos& request) {
return ::grpc::internal::ClientReaderFactory< ::milvus::grpc::TopKQueryResult>::Create(channel_.get(), rpcmethod_SearchVectorInFiles_, context, request);
}
void MilvusService::Stub::experimental_async::SearchVectorInFiles(::grpc::ClientContext* context, ::milvus::grpc::SearchVectorInFilesInfos* request, ::grpc::experimental::ClientReadReactor< ::milvus::grpc::TopKQueryResult>* reactor) {
::grpc::internal::ClientCallbackReaderFactory< ::milvus::grpc::TopKQueryResult>::Create(stub_->channel_.get(), stub_->rpcmethod_SearchVectorInFiles_, context, request, reactor);
}
::grpc::ClientAsyncReader< ::milvus::grpc::TopKQueryResult>* MilvusService::Stub::AsyncSearchVectorInFilesRaw(::grpc::ClientContext* context, const ::milvus::grpc::SearchVectorInFilesInfos& request, ::grpc::CompletionQueue* cq, void* tag) {
return ::grpc::internal::ClientAsyncReaderFactory< ::milvus::grpc::TopKQueryResult>::Create(channel_.get(), cq, rpcmethod_SearchVectorInFiles_, context, request, true, tag);
}
::grpc::ClientAsyncReader< ::milvus::grpc::TopKQueryResult>* MilvusService::Stub::PrepareAsyncSearchVectorInFilesRaw(::grpc::ClientContext* context, const ::milvus::grpc::SearchVectorInFilesInfos& request, ::grpc::CompletionQueue* cq) {
return ::grpc::internal::ClientAsyncReaderFactory< ::milvus::grpc::TopKQueryResult>::Create(channel_.get(), cq, rpcmethod_SearchVectorInFiles_, context, request, false, nullptr);
}
::grpc::Status MilvusService::Stub::DescribeTable(::grpc::ClientContext* context, const ::milvus::grpc::TableName& request, ::milvus::grpc::TableSchema* response) {
return ::grpc::internal::BlockingUnaryCall(channel_.get(), rpcmethod_DescribeTable_, context, request, response);
}
void MilvusService::Stub::experimental_async::DescribeTable(::grpc::ClientContext* context, const ::milvus::grpc::TableName* request, ::milvus::grpc::TableSchema* response, std::function<void(::grpc::Status)> f) {
return ::grpc::internal::CallbackUnaryCall(stub_->channel_.get(), stub_->rpcmethod_DescribeTable_, context, request, response, std::move(f));
}
void MilvusService::Stub::experimental_async::DescribeTable(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::TableSchema* response, std::function<void(::grpc::Status)> f) {
return ::grpc::internal::CallbackUnaryCall(stub_->channel_.get(), stub_->rpcmethod_DescribeTable_, context, request, response, std::move(f));
}
::grpc::ClientAsyncResponseReader< ::milvus::grpc::TableSchema>* MilvusService::Stub::AsyncDescribeTableRaw(::grpc::ClientContext* context, const ::milvus::grpc::TableName& request, ::grpc::CompletionQueue* cq) {
return ::grpc::internal::ClientAsyncResponseReaderFactory< ::milvus::grpc::TableSchema>::Create(channel_.get(), cq, rpcmethod_DescribeTable_, context, request, true);
}
::grpc::ClientAsyncResponseReader< ::milvus::grpc::TableSchema>* MilvusService::Stub::PrepareAsyncDescribeTableRaw(::grpc::ClientContext* context, const ::milvus::grpc::TableName& request, ::grpc::CompletionQueue* cq) {
return ::grpc::internal::ClientAsyncResponseReaderFactory< ::milvus::grpc::TableSchema>::Create(channel_.get(), cq, rpcmethod_DescribeTable_, context, request, false);
}
::grpc::Status MilvusService::Stub::GetTableRowCount(::grpc::ClientContext* context, const ::milvus::grpc::TableName& request, ::milvus::grpc::TableRowCount* response) {
return ::grpc::internal::BlockingUnaryCall(channel_.get(), rpcmethod_GetTableRowCount_, context, request, response);
}
void MilvusService::Stub::experimental_async::GetTableRowCount(::grpc::ClientContext* context, const ::milvus::grpc::TableName* request, ::milvus::grpc::TableRowCount* response, std::function<void(::grpc::Status)> f) {
return ::grpc::internal::CallbackUnaryCall(stub_->channel_.get(), stub_->rpcmethod_GetTableRowCount_, context, request, response, std::move(f));
}
void MilvusService::Stub::experimental_async::GetTableRowCount(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::TableRowCount* response, std::function<void(::grpc::Status)> f) {
return ::grpc::internal::CallbackUnaryCall(stub_->channel_.get(), stub_->rpcmethod_GetTableRowCount_, context, request, response, std::move(f));
}
::grpc::ClientAsyncResponseReader< ::milvus::grpc::TableRowCount>* MilvusService::Stub::AsyncGetTableRowCountRaw(::grpc::ClientContext* context, const ::milvus::grpc::TableName& request, ::grpc::CompletionQueue* cq) {
return ::grpc::internal::ClientAsyncResponseReaderFactory< ::milvus::grpc::TableRowCount>::Create(channel_.get(), cq, rpcmethod_GetTableRowCount_, context, request, true);
}
::grpc::ClientAsyncResponseReader< ::milvus::grpc::TableRowCount>* MilvusService::Stub::PrepareAsyncGetTableRowCountRaw(::grpc::ClientContext* context, const ::milvus::grpc::TableName& request, ::grpc::CompletionQueue* cq) {
return ::grpc::internal::ClientAsyncResponseReaderFactory< ::milvus::grpc::TableRowCount>::Create(channel_.get(), cq, rpcmethod_GetTableRowCount_, context, request, false);
}
::grpc::ClientReader< ::milvus::grpc::TableName>* MilvusService::Stub::ShowTablesRaw(::grpc::ClientContext* context, const ::milvus::grpc::Command& request) {
return ::grpc::internal::ClientReaderFactory< ::milvus::grpc::TableName>::Create(channel_.get(), rpcmethod_ShowTables_, context, request);
}
void MilvusService::Stub::experimental_async::ShowTables(::grpc::ClientContext* context, ::milvus::grpc::Command* request, ::grpc::experimental::ClientReadReactor< ::milvus::grpc::TableName>* reactor) {
::grpc::internal::ClientCallbackReaderFactory< ::milvus::grpc::TableName>::Create(stub_->channel_.get(), stub_->rpcmethod_ShowTables_, context, request, reactor);
}
::grpc::ClientAsyncReader< ::milvus::grpc::TableName>* MilvusService::Stub::AsyncShowTablesRaw(::grpc::ClientContext* context, const ::milvus::grpc::Command& request, ::grpc::CompletionQueue* cq, void* tag) {
return ::grpc::internal::ClientAsyncReaderFactory< ::milvus::grpc::TableName>::Create(channel_.get(), cq, rpcmethod_ShowTables_, context, request, true, tag);
}
::grpc::ClientAsyncReader< ::milvus::grpc::TableName>* MilvusService::Stub::PrepareAsyncShowTablesRaw(::grpc::ClientContext* context, const ::milvus::grpc::Command& request, ::grpc::CompletionQueue* cq) {
return ::grpc::internal::ClientAsyncReaderFactory< ::milvus::grpc::TableName>::Create(channel_.get(), cq, rpcmethod_ShowTables_, context, request, false, nullptr);
}
::grpc::Status MilvusService::Stub::Ping(::grpc::ClientContext* context, const ::milvus::grpc::Command& request, ::milvus::grpc::ServerStatus* response) {
return ::grpc::internal::BlockingUnaryCall(channel_.get(), rpcmethod_Ping_, context, request, response);
}
void MilvusService::Stub::experimental_async::Ping(::grpc::ClientContext* context, const ::milvus::grpc::Command* request, ::milvus::grpc::ServerStatus* response, std::function<void(::grpc::Status)> f) {
return ::grpc::internal::CallbackUnaryCall(stub_->channel_.get(), stub_->rpcmethod_Ping_, context, request, response, std::move(f));
}
void MilvusService::Stub::experimental_async::Ping(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::ServerStatus* response, std::function<void(::grpc::Status)> f) {
return ::grpc::internal::CallbackUnaryCall(stub_->channel_.get(), stub_->rpcmethod_Ping_, context, request, response, std::move(f));
}
::grpc::ClientAsyncResponseReader< ::milvus::grpc::ServerStatus>* MilvusService::Stub::AsyncPingRaw(::grpc::ClientContext* context, const ::milvus::grpc::Command& request, ::grpc::CompletionQueue* cq) {
return ::grpc::internal::ClientAsyncResponseReaderFactory< ::milvus::grpc::ServerStatus>::Create(channel_.get(), cq, rpcmethod_Ping_, context, request, true);
}
::grpc::ClientAsyncResponseReader< ::milvus::grpc::ServerStatus>* MilvusService::Stub::PrepareAsyncPingRaw(::grpc::ClientContext* context, const ::milvus::grpc::Command& request, ::grpc::CompletionQueue* cq) {
return ::grpc::internal::ClientAsyncResponseReaderFactory< ::milvus::grpc::ServerStatus>::Create(channel_.get(), cq, rpcmethod_Ping_, context, request, false);
}
MilvusService::Service::Service() {
AddMethod(new ::grpc::internal::RpcServiceMethod(
MilvusService_method_names[0],
::grpc::internal::RpcMethod::NORMAL_RPC,
new ::grpc::internal::RpcMethodHandler< MilvusService::Service, ::milvus::grpc::TableSchema, ::milvus::Status>(
std::mem_fn(&MilvusService::Service::CreateTable), this)));
AddMethod(new ::grpc::internal::RpcServiceMethod(
MilvusService_method_names[1],
::grpc::internal::RpcMethod::NORMAL_RPC,
new ::grpc::internal::RpcMethodHandler< MilvusService::Service, ::milvus::grpc::TableName, ::milvus::grpc::BoolReply>(
std::mem_fn(&MilvusService::Service::HasTable), this)));
AddMethod(new ::grpc::internal::RpcServiceMethod(
MilvusService_method_names[2],
::grpc::internal::RpcMethod::NORMAL_RPC,
new ::grpc::internal::RpcMethodHandler< MilvusService::Service, ::milvus::grpc::TableName, ::milvus::Status>(
std::mem_fn(&MilvusService::Service::DropTable), this)));
AddMethod(new ::grpc::internal::RpcServiceMethod(
MilvusService_method_names[3],
::grpc::internal::RpcMethod::NORMAL_RPC,
new ::grpc::internal::RpcMethodHandler< MilvusService::Service, ::milvus::grpc::TableName, ::milvus::Status>(
std::mem_fn(&MilvusService::Service::BuildIndex), this)));
AddMethod(new ::grpc::internal::RpcServiceMethod(
MilvusService_method_names[4],
::grpc::internal::RpcMethod::NORMAL_RPC,
new ::grpc::internal::RpcMethodHandler< MilvusService::Service, ::milvus::grpc::InsertInfos, ::milvus::grpc::VectorIds>(
std::mem_fn(&MilvusService::Service::InsertVector), this)));
AddMethod(new ::grpc::internal::RpcServiceMethod(
MilvusService_method_names[5],
::grpc::internal::RpcMethod::SERVER_STREAMING,
new ::grpc::internal::ServerStreamingHandler< MilvusService::Service, ::milvus::grpc::SearchVectorInfos, ::milvus::grpc::TopKQueryResult>(
std::mem_fn(&MilvusService::Service::SearchVector), this)));
AddMethod(new ::grpc::internal::RpcServiceMethod(
MilvusService_method_names[6],
::grpc::internal::RpcMethod::SERVER_STREAMING,
new ::grpc::internal::ServerStreamingHandler< MilvusService::Service, ::milvus::grpc::SearchVectorInFilesInfos, ::milvus::grpc::TopKQueryResult>(
std::mem_fn(&MilvusService::Service::SearchVectorInFiles), this)));
AddMethod(new ::grpc::internal::RpcServiceMethod(
MilvusService_method_names[7],
::grpc::internal::RpcMethod::NORMAL_RPC,
new ::grpc::internal::RpcMethodHandler< MilvusService::Service, ::milvus::grpc::TableName, ::milvus::grpc::TableSchema>(
std::mem_fn(&MilvusService::Service::DescribeTable), this)));
AddMethod(new ::grpc::internal::RpcServiceMethod(
MilvusService_method_names[8],
::grpc::internal::RpcMethod::NORMAL_RPC,
new ::grpc::internal::RpcMethodHandler< MilvusService::Service, ::milvus::grpc::TableName, ::milvus::grpc::TableRowCount>(
std::mem_fn(&MilvusService::Service::GetTableRowCount), this)));
AddMethod(new ::grpc::internal::RpcServiceMethod(
MilvusService_method_names[9],
::grpc::internal::RpcMethod::SERVER_STREAMING,
new ::grpc::internal::ServerStreamingHandler< MilvusService::Service, ::milvus::grpc::Command, ::milvus::grpc::TableName>(
std::mem_fn(&MilvusService::Service::ShowTables), this)));
AddMethod(new ::grpc::internal::RpcServiceMethod(
MilvusService_method_names[10],
::grpc::internal::RpcMethod::NORMAL_RPC,
new ::grpc::internal::RpcMethodHandler< MilvusService::Service, ::milvus::grpc::Command, ::milvus::grpc::ServerStatus>(
std::mem_fn(&MilvusService::Service::Ping), this)));
}
MilvusService::Service::~Service() {
}
::grpc::Status MilvusService::Service::CreateTable(::grpc::ServerContext* context, const ::milvus::grpc::TableSchema* request, ::milvus::Status* response) {
(void) context;
(void) request;
(void) response;
return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, "");
}
::grpc::Status MilvusService::Service::HasTable(::grpc::ServerContext* context, const ::milvus::grpc::TableName* request, ::milvus::grpc::BoolReply* response) {
(void) context;
(void) request;
(void) response;
return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, "");
}
::grpc::Status MilvusService::Service::DropTable(::grpc::ServerContext* context, const ::milvus::grpc::TableName* request, ::milvus::Status* response) {
(void) context;
(void) request;
(void) response;
return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, "");
}
::grpc::Status MilvusService::Service::BuildIndex(::grpc::ServerContext* context, const ::milvus::grpc::TableName* request, ::milvus::Status* response) {
(void) context;
(void) request;
(void) response;
return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, "");
}
::grpc::Status MilvusService::Service::InsertVector(::grpc::ServerContext* context, const ::milvus::grpc::InsertInfos* request, ::milvus::grpc::VectorIds* response) {
(void) context;
(void) request;
(void) response;
return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, "");
}
::grpc::Status MilvusService::Service::SearchVector(::grpc::ServerContext* context, const ::milvus::grpc::SearchVectorInfos* request, ::grpc::ServerWriter< ::milvus::grpc::TopKQueryResult>* writer) {
(void) context;
(void) request;
(void) writer;
return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, "");
}
::grpc::Status MilvusService::Service::SearchVectorInFiles(::grpc::ServerContext* context, const ::milvus::grpc::SearchVectorInFilesInfos* request, ::grpc::ServerWriter< ::milvus::grpc::TopKQueryResult>* writer) {
(void) context;
(void) request;
(void) writer;
return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, "");
}
::grpc::Status MilvusService::Service::DescribeTable(::grpc::ServerContext* context, const ::milvus::grpc::TableName* request, ::milvus::grpc::TableSchema* response) {
(void) context;
(void) request;
(void) response;
return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, "");
}
::grpc::Status MilvusService::Service::GetTableRowCount(::grpc::ServerContext* context, const ::milvus::grpc::TableName* request, ::milvus::grpc::TableRowCount* response) {
(void) context;
(void) request;
(void) response;
return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, "");
}
::grpc::Status MilvusService::Service::ShowTables(::grpc::ServerContext* context, const ::milvus::grpc::Command* request, ::grpc::ServerWriter< ::milvus::grpc::TableName>* writer) {
(void) context;
(void) request;
(void) writer;
return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, "");
}
::grpc::Status MilvusService::Service::Ping(::grpc::ServerContext* context, const ::milvus::grpc::Command* request, ::milvus::grpc::ServerStatus* response) {
(void) context;
(void) request;
(void) response;
return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, "");
}
} // namespace milvus
} // namespace grpc

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

View File

@ -0,0 +1,22 @@
// Generated by the gRPC C++ plugin.
// If you make any local change, they will be lost.
// source: status.proto
#include "status.pb.h"
#include "status.grpc.pb.h"
#include <functional>
#include <grpcpp/impl/codegen/async_stream.h>
#include <grpcpp/impl/codegen/async_unary_call.h>
#include <grpcpp/impl/codegen/channel_interface.h>
#include <grpcpp/impl/codegen/client_unary_call.h>
#include <grpcpp/impl/codegen/client_callback.h>
#include <grpcpp/impl/codegen/method_handler_impl.h>
#include <grpcpp/impl/codegen/rpc_service_method.h>
#include <grpcpp/impl/codegen/server_callback.h>
#include <grpcpp/impl/codegen/service_type.h>
#include <grpcpp/impl/codegen/sync_stream.h>
namespace milvus {
} // namespace milvus

View File

@ -0,0 +1,35 @@
// Generated by the gRPC C++ plugin.
// If you make any local change, they will be lost.
// source: status.proto
#ifndef GRPC_status_2eproto__INCLUDED
#define GRPC_status_2eproto__INCLUDED
#include "status.pb.h"
#include <functional>
#include <grpcpp/impl/codegen/async_generic_service.h>
#include <grpcpp/impl/codegen/async_stream.h>
#include <grpcpp/impl/codegen/async_unary_call.h>
#include <grpcpp/impl/codegen/client_callback.h>
#include <grpcpp/impl/codegen/method_handler_impl.h>
#include <grpcpp/impl/codegen/proto_utils.h>
#include <grpcpp/impl/codegen/rpc_method.h>
#include <grpcpp/impl/codegen/server_callback.h>
#include <grpcpp/impl/codegen/service_type.h>
#include <grpcpp/impl/codegen/status.h>
#include <grpcpp/impl/codegen/stub_options.h>
#include <grpcpp/impl/codegen/sync_stream.h>
namespace grpc {
class CompletionQueue;
class Channel;
class ServerCompletionQueue;
class ServerContext;
} // namespace grpc
namespace milvus {
} // namespace milvus
#endif // GRPC_status_2eproto__INCLUDED

View File

@ -0,0 +1,494 @@
// Generated by the protocol buffer compiler. DO NOT EDIT!
// source: status.proto
#include "status.pb.h"
#include <algorithm>
#include <google/protobuf/stubs/common.h>
#include <google/protobuf/io/coded_stream.h>
#include <google/protobuf/extension_set.h>
#include <google/protobuf/wire_format_lite_inl.h>
#include <google/protobuf/descriptor.h>
#include <google/protobuf/generated_message_reflection.h>
#include <google/protobuf/reflection_ops.h>
#include <google/protobuf/wire_format.h>
// @@protoc_insertion_point(includes)
#include <google/protobuf/port_def.inc>
namespace milvus {
class StatusDefaultTypeInternal {
public:
::google::protobuf::internal::ExplicitlyConstructed<Status> _instance;
} _Status_default_instance_;
} // namespace milvus
static void InitDefaultsStatus_status_2eproto() {
GOOGLE_PROTOBUF_VERIFY_VERSION;
{
void* ptr = &::milvus::_Status_default_instance_;
new (ptr) ::milvus::Status();
::google::protobuf::internal::OnShutdownDestroyMessage(ptr);
}
::milvus::Status::InitAsDefaultInstance();
}
::google::protobuf::internal::SCCInfo<0> scc_info_Status_status_2eproto =
{{ATOMIC_VAR_INIT(::google::protobuf::internal::SCCInfoBase::kUninitialized), 0, InitDefaultsStatus_status_2eproto}, {}};
void InitDefaults_status_2eproto() {
::google::protobuf::internal::InitSCC(&scc_info_Status_status_2eproto.base);
}
::google::protobuf::Metadata file_level_metadata_status_2eproto[1];
const ::google::protobuf::EnumDescriptor* file_level_enum_descriptors_status_2eproto[1];
constexpr ::google::protobuf::ServiceDescriptor const** file_level_service_descriptors_status_2eproto = nullptr;
const ::google::protobuf::uint32 TableStruct_status_2eproto::offsets[] PROTOBUF_SECTION_VARIABLE(protodesc_cold) = {
~0u, // no _has_bits_
PROTOBUF_FIELD_OFFSET(::milvus::Status, _internal_metadata_),
~0u, // no _extensions_
~0u, // no _oneof_case_
~0u, // no _weak_field_map_
PROTOBUF_FIELD_OFFSET(::milvus::Status, error_code_),
PROTOBUF_FIELD_OFFSET(::milvus::Status, reason_),
};
static const ::google::protobuf::internal::MigrationSchema schemas[] PROTOBUF_SECTION_VARIABLE(protodesc_cold) = {
{ 0, -1, sizeof(::milvus::Status)},
};
static ::google::protobuf::Message const * const file_default_instances[] = {
reinterpret_cast<const ::google::protobuf::Message*>(&::milvus::_Status_default_instance_),
};
::google::protobuf::internal::AssignDescriptorsTable assign_descriptors_table_status_2eproto = {
{}, AddDescriptors_status_2eproto, "status.proto", schemas,
file_default_instances, TableStruct_status_2eproto::offsets,
file_level_metadata_status_2eproto, 1, file_level_enum_descriptors_status_2eproto, file_level_service_descriptors_status_2eproto,
};
const char descriptor_table_protodef_status_2eproto[] =
"\n\014status.proto\022\006milvus\"\?\n\006Status\022%\n\nerro"
"r_code\030\001 \001(\0162\021.milvus.ErrorCode\022\016\n\006reaso"
"n\030\002 \001(\t*\354\003\n\tErrorCode\022\013\n\007SUCCESS\020\000\022\024\n\020UN"
"EXPECTED_ERROR\020\001\022\022\n\016CONNECT_FAILED\020\002\022\025\n\021"
"PERMISSION_DENIED\020\003\022\024\n\020TABLE_NOT_EXISTS\020"
"\004\022\024\n\020ILLEGAL_ARGUMENT\020\005\022\021\n\rILLEGAL_RANGE"
"\020\006\022\025\n\021ILLEGAL_DIMENSION\020\007\022\026\n\022ILLEGAL_IND"
"EX_TYPE\020\010\022\026\n\022ILLEGAL_TABLE_NAME\020\t\022\020\n\014ILL"
"EGAL_TOPK\020\n\022\025\n\021ILLEGAL_ROWRECORD\020\013\022\025\n\021IL"
"LEGAL_VECTOR_ID\020\014\022\031\n\025ILLEGAL_SEARCH_RESU"
"LT\020\r\022\022\n\016FILE_NOT_FOUND\020\016\022\017\n\013META_FAILED\020"
"\017\022\020\n\014CACHE_FAILED\020\020\022\030\n\024CANNOT_CREATE_FOL"
"DER\020\021\022\026\n\022CANNOT_CREATE_FILE\020\022\022\030\n\024CANNOT_"
"DELETE_FOLDER\020\023\022\026\n\022CANNOT_DELETE_FILE\020\024\022"
"\025\n\021BUILD_INDEX_ERROR\020\025b\006proto3"
;
::google::protobuf::internal::DescriptorTable descriptor_table_status_2eproto = {
false, InitDefaults_status_2eproto,
descriptor_table_protodef_status_2eproto,
"status.proto", &assign_descriptors_table_status_2eproto, 590,
};
void AddDescriptors_status_2eproto() {
static constexpr ::google::protobuf::internal::InitFunc deps[1] =
{
};
::google::protobuf::internal::AddDescriptors(&descriptor_table_status_2eproto, deps, 0);
}
// Force running AddDescriptors() at dynamic initialization time.
static bool dynamic_init_dummy_status_2eproto = []() { AddDescriptors_status_2eproto(); return true; }();
namespace milvus {
const ::google::protobuf::EnumDescriptor* ErrorCode_descriptor() {
::google::protobuf::internal::AssignDescriptors(&assign_descriptors_table_status_2eproto);
return file_level_enum_descriptors_status_2eproto[0];
}
bool ErrorCode_IsValid(int value) {
switch (value) {
case 0:
case 1:
case 2:
case 3:
case 4:
case 5:
case 6:
case 7:
case 8:
case 9:
case 10:
case 11:
case 12:
case 13:
case 14:
case 15:
case 16:
case 17:
case 18:
case 19:
case 20:
case 21:
return true;
default:
return false;
}
}
// ===================================================================
void Status::InitAsDefaultInstance() {
}
class Status::HasBitSetters {
public:
};
#if !defined(_MSC_VER) || _MSC_VER >= 1900
const int Status::kErrorCodeFieldNumber;
const int Status::kReasonFieldNumber;
#endif // !defined(_MSC_VER) || _MSC_VER >= 1900
Status::Status()
: ::google::protobuf::Message(), _internal_metadata_(nullptr) {
SharedCtor();
// @@protoc_insertion_point(constructor:milvus.Status)
}
Status::Status(const Status& from)
: ::google::protobuf::Message(),
_internal_metadata_(nullptr) {
_internal_metadata_.MergeFrom(from._internal_metadata_);
reason_.UnsafeSetDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited());
if (from.reason().size() > 0) {
reason_.AssignWithDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), from.reason_);
}
error_code_ = from.error_code_;
// @@protoc_insertion_point(copy_constructor:milvus.Status)
}
void Status::SharedCtor() {
::google::protobuf::internal::InitSCC(
&scc_info_Status_status_2eproto.base);
reason_.UnsafeSetDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited());
error_code_ = 0;
}
Status::~Status() {
// @@protoc_insertion_point(destructor:milvus.Status)
SharedDtor();
}
void Status::SharedDtor() {
reason_.DestroyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited());
}
void Status::SetCachedSize(int size) const {
_cached_size_.Set(size);
}
const Status& Status::default_instance() {
::google::protobuf::internal::InitSCC(&::scc_info_Status_status_2eproto.base);
return *internal_default_instance();
}
void Status::Clear() {
// @@protoc_insertion_point(message_clear_start:milvus.Status)
::google::protobuf::uint32 cached_has_bits = 0;
// Prevent compiler warnings about cached_has_bits being unused
(void) cached_has_bits;
reason_.ClearToEmptyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited());
error_code_ = 0;
_internal_metadata_.Clear();
}
#if GOOGLE_PROTOBUF_ENABLE_EXPERIMENTAL_PARSER
const char* Status::_InternalParse(const char* begin, const char* end, void* object,
::google::protobuf::internal::ParseContext* ctx) {
auto msg = static_cast<Status*>(object);
::google::protobuf::int32 size; (void)size;
int depth; (void)depth;
::google::protobuf::uint32 tag;
::google::protobuf::internal::ParseFunc parser_till_end; (void)parser_till_end;
auto ptr = begin;
while (ptr < end) {
ptr = ::google::protobuf::io::Parse32(ptr, &tag);
GOOGLE_PROTOBUF_PARSER_ASSERT(ptr);
switch (tag >> 3) {
// .milvus.ErrorCode error_code = 1;
case 1: {
if (static_cast<::google::protobuf::uint8>(tag) != 8) goto handle_unusual;
::google::protobuf::uint64 val = ::google::protobuf::internal::ReadVarint(&ptr);
msg->set_error_code(static_cast<::milvus::ErrorCode>(val));
GOOGLE_PROTOBUF_PARSER_ASSERT(ptr);
break;
}
// string reason = 2;
case 2: {
if (static_cast<::google::protobuf::uint8>(tag) != 18) goto handle_unusual;
ptr = ::google::protobuf::io::ReadSize(ptr, &size);
GOOGLE_PROTOBUF_PARSER_ASSERT(ptr);
ctx->extra_parse_data().SetFieldName("milvus.Status.reason");
object = msg->mutable_reason();
if (size > end - ptr + ::google::protobuf::internal::ParseContext::kSlopBytes) {
parser_till_end = ::google::protobuf::internal::GreedyStringParserUTF8;
goto string_till_end;
}
GOOGLE_PROTOBUF_PARSER_ASSERT(::google::protobuf::internal::StringCheckUTF8(ptr, size, ctx));
::google::protobuf::internal::InlineGreedyStringParser(object, ptr, size, ctx);
ptr += size;
break;
}
default: {
handle_unusual:
if ((tag & 7) == 4 || tag == 0) {
ctx->EndGroup(tag);
return ptr;
}
auto res = UnknownFieldParse(tag, {_InternalParse, msg},
ptr, end, msg->_internal_metadata_.mutable_unknown_fields(), ctx);
ptr = res.first;
GOOGLE_PROTOBUF_PARSER_ASSERT(ptr != nullptr);
if (res.second) return ptr;
}
} // switch
} // while
return ptr;
string_till_end:
static_cast<::std::string*>(object)->clear();
static_cast<::std::string*>(object)->reserve(size);
goto len_delim_till_end;
len_delim_till_end:
return ctx->StoreAndTailCall(ptr, end, {_InternalParse, msg},
{parser_till_end, object}, size);
}
#else // GOOGLE_PROTOBUF_ENABLE_EXPERIMENTAL_PARSER
bool Status::MergePartialFromCodedStream(
::google::protobuf::io::CodedInputStream* input) {
#define DO_(EXPRESSION) if (!PROTOBUF_PREDICT_TRUE(EXPRESSION)) goto failure
::google::protobuf::uint32 tag;
// @@protoc_insertion_point(parse_start:milvus.Status)
for (;;) {
::std::pair<::google::protobuf::uint32, bool> p = input->ReadTagWithCutoffNoLastTag(127u);
tag = p.first;
if (!p.second) goto handle_unusual;
switch (::google::protobuf::internal::WireFormatLite::GetTagFieldNumber(tag)) {
// .milvus.ErrorCode error_code = 1;
case 1: {
if (static_cast< ::google::protobuf::uint8>(tag) == (8 & 0xFF)) {
int value = 0;
DO_((::google::protobuf::internal::WireFormatLite::ReadPrimitive<
int, ::google::protobuf::internal::WireFormatLite::TYPE_ENUM>(
input, &value)));
set_error_code(static_cast< ::milvus::ErrorCode >(value));
} else {
goto handle_unusual;
}
break;
}
// string reason = 2;
case 2: {
if (static_cast< ::google::protobuf::uint8>(tag) == (18 & 0xFF)) {
DO_(::google::protobuf::internal::WireFormatLite::ReadString(
input, this->mutable_reason()));
DO_(::google::protobuf::internal::WireFormatLite::VerifyUtf8String(
this->reason().data(), static_cast<int>(this->reason().length()),
::google::protobuf::internal::WireFormatLite::PARSE,
"milvus.Status.reason"));
} else {
goto handle_unusual;
}
break;
}
default: {
handle_unusual:
if (tag == 0) {
goto success;
}
DO_(::google::protobuf::internal::WireFormat::SkipField(
input, tag, _internal_metadata_.mutable_unknown_fields()));
break;
}
}
}
success:
// @@protoc_insertion_point(parse_success:milvus.Status)
return true;
failure:
// @@protoc_insertion_point(parse_failure:milvus.Status)
return false;
#undef DO_
}
#endif // GOOGLE_PROTOBUF_ENABLE_EXPERIMENTAL_PARSER
void Status::SerializeWithCachedSizes(
::google::protobuf::io::CodedOutputStream* output) const {
// @@protoc_insertion_point(serialize_start:milvus.Status)
::google::protobuf::uint32 cached_has_bits = 0;
(void) cached_has_bits;
// .milvus.ErrorCode error_code = 1;
if (this->error_code() != 0) {
::google::protobuf::internal::WireFormatLite::WriteEnum(
1, this->error_code(), output);
}
// string reason = 2;
if (this->reason().size() > 0) {
::google::protobuf::internal::WireFormatLite::VerifyUtf8String(
this->reason().data(), static_cast<int>(this->reason().length()),
::google::protobuf::internal::WireFormatLite::SERIALIZE,
"milvus.Status.reason");
::google::protobuf::internal::WireFormatLite::WriteStringMaybeAliased(
2, this->reason(), output);
}
if (_internal_metadata_.have_unknown_fields()) {
::google::protobuf::internal::WireFormat::SerializeUnknownFields(
_internal_metadata_.unknown_fields(), output);
}
// @@protoc_insertion_point(serialize_end:milvus.Status)
}
::google::protobuf::uint8* Status::InternalSerializeWithCachedSizesToArray(
::google::protobuf::uint8* target) const {
// @@protoc_insertion_point(serialize_to_array_start:milvus.Status)
::google::protobuf::uint32 cached_has_bits = 0;
(void) cached_has_bits;
// .milvus.ErrorCode error_code = 1;
if (this->error_code() != 0) {
target = ::google::protobuf::internal::WireFormatLite::WriteEnumToArray(
1, this->error_code(), target);
}
// string reason = 2;
if (this->reason().size() > 0) {
::google::protobuf::internal::WireFormatLite::VerifyUtf8String(
this->reason().data(), static_cast<int>(this->reason().length()),
::google::protobuf::internal::WireFormatLite::SERIALIZE,
"milvus.Status.reason");
target =
::google::protobuf::internal::WireFormatLite::WriteStringToArray(
2, this->reason(), target);
}
if (_internal_metadata_.have_unknown_fields()) {
target = ::google::protobuf::internal::WireFormat::SerializeUnknownFieldsToArray(
_internal_metadata_.unknown_fields(), target);
}
// @@protoc_insertion_point(serialize_to_array_end:milvus.Status)
return target;
}
size_t Status::ByteSizeLong() const {
// @@protoc_insertion_point(message_byte_size_start:milvus.Status)
size_t total_size = 0;
if (_internal_metadata_.have_unknown_fields()) {
total_size +=
::google::protobuf::internal::WireFormat::ComputeUnknownFieldsSize(
_internal_metadata_.unknown_fields());
}
::google::protobuf::uint32 cached_has_bits = 0;
// Prevent compiler warnings about cached_has_bits being unused
(void) cached_has_bits;
// string reason = 2;
if (this->reason().size() > 0) {
total_size += 1 +
::google::protobuf::internal::WireFormatLite::StringSize(
this->reason());
}
// .milvus.ErrorCode error_code = 1;
if (this->error_code() != 0) {
total_size += 1 +
::google::protobuf::internal::WireFormatLite::EnumSize(this->error_code());
}
int cached_size = ::google::protobuf::internal::ToCachedSize(total_size);
SetCachedSize(cached_size);
return total_size;
}
void Status::MergeFrom(const ::google::protobuf::Message& from) {
// @@protoc_insertion_point(generalized_merge_from_start:milvus.Status)
GOOGLE_DCHECK_NE(&from, this);
const Status* source =
::google::protobuf::DynamicCastToGenerated<Status>(
&from);
if (source == nullptr) {
// @@protoc_insertion_point(generalized_merge_from_cast_fail:milvus.Status)
::google::protobuf::internal::ReflectionOps::Merge(from, this);
} else {
// @@protoc_insertion_point(generalized_merge_from_cast_success:milvus.Status)
MergeFrom(*source);
}
}
void Status::MergeFrom(const Status& from) {
// @@protoc_insertion_point(class_specific_merge_from_start:milvus.Status)
GOOGLE_DCHECK_NE(&from, this);
_internal_metadata_.MergeFrom(from._internal_metadata_);
::google::protobuf::uint32 cached_has_bits = 0;
(void) cached_has_bits;
if (from.reason().size() > 0) {
reason_.AssignWithDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), from.reason_);
}
if (from.error_code() != 0) {
set_error_code(from.error_code());
}
}
void Status::CopyFrom(const ::google::protobuf::Message& from) {
// @@protoc_insertion_point(generalized_copy_from_start:milvus.Status)
if (&from == this) return;
Clear();
MergeFrom(from);
}
void Status::CopyFrom(const Status& from) {
// @@protoc_insertion_point(class_specific_copy_from_start:milvus.Status)
if (&from == this) return;
Clear();
MergeFrom(from);
}
bool Status::IsInitialized() const {
return true;
}
void Status::Swap(Status* other) {
if (other == this) return;
InternalSwap(other);
}
void Status::InternalSwap(Status* other) {
using std::swap;
_internal_metadata_.Swap(&other->_internal_metadata_);
reason_.Swap(&other->reason_, &::google::protobuf::internal::GetEmptyStringAlreadyInited(),
GetArenaNoVirtual());
swap(error_code_, other->error_code_);
}
::google::protobuf::Metadata Status::GetMetadata() const {
::google::protobuf::internal::AssignDescriptors(&::assign_descriptors_table_status_2eproto);
return ::file_level_metadata_status_2eproto[kIndexInFileMessages];
}
// @@protoc_insertion_point(namespace_scope)
} // namespace milvus
namespace google {
namespace protobuf {
template<> PROTOBUF_NOINLINE ::milvus::Status* Arena::CreateMaybeMessage< ::milvus::Status >(Arena* arena) {
return Arena::CreateInternal< ::milvus::Status >(arena);
}
} // namespace protobuf
} // namespace google
// @@protoc_insertion_point(global_scope)
#include <google/protobuf/port_undef.inc>

View File

@ -0,0 +1,333 @@
// Generated by the protocol buffer compiler. DO NOT EDIT!
// source: status.proto
#ifndef PROTOBUF_INCLUDED_status_2eproto
#define PROTOBUF_INCLUDED_status_2eproto
#include <limits>
#include <string>
#include <google/protobuf/port_def.inc>
#if PROTOBUF_VERSION < 3007000
#error This file was generated by a newer version of protoc which is
#error incompatible with your Protocol Buffer headers. Please update
#error your headers.
#endif
#if 3007000 < PROTOBUF_MIN_PROTOC_VERSION
#error This file was generated by an older version of protoc which is
#error incompatible with your Protocol Buffer headers. Please
#error regenerate this file with a newer version of protoc.
#endif
#include <google/protobuf/port_undef.inc>
#include <google/protobuf/io/coded_stream.h>
#include <google/protobuf/arena.h>
#include <google/protobuf/arenastring.h>
#include <google/protobuf/generated_message_table_driven.h>
#include <google/protobuf/generated_message_util.h>
#include <google/protobuf/inlined_string_field.h>
#include <google/protobuf/metadata.h>
#include <google/protobuf/message.h>
#include <google/protobuf/repeated_field.h> // IWYU pragma: export
#include <google/protobuf/extension_set.h> // IWYU pragma: export
#include <google/protobuf/generated_enum_reflection.h>
#include <google/protobuf/unknown_field_set.h>
// @@protoc_insertion_point(includes)
#include <google/protobuf/port_def.inc>
#define PROTOBUF_INTERNAL_EXPORT_status_2eproto
// Internal implementation detail -- do not use these members.
struct TableStruct_status_2eproto {
static const ::google::protobuf::internal::ParseTableField entries[]
PROTOBUF_SECTION_VARIABLE(protodesc_cold);
static const ::google::protobuf::internal::AuxillaryParseTableField aux[]
PROTOBUF_SECTION_VARIABLE(protodesc_cold);
static const ::google::protobuf::internal::ParseTable schema[1]
PROTOBUF_SECTION_VARIABLE(protodesc_cold);
static const ::google::protobuf::internal::FieldMetadata field_metadata[];
static const ::google::protobuf::internal::SerializationTable serialization_table[];
static const ::google::protobuf::uint32 offsets[];
};
void AddDescriptors_status_2eproto();
namespace milvus {
class Status;
class StatusDefaultTypeInternal;
extern StatusDefaultTypeInternal _Status_default_instance_;
} // namespace milvus
namespace google {
namespace protobuf {
template<> ::milvus::Status* Arena::CreateMaybeMessage<::milvus::Status>(Arena*);
} // namespace protobuf
} // namespace google
namespace milvus {
enum ErrorCode {
SUCCESS = 0,
UNEXPECTED_ERROR = 1,
CONNECT_FAILED = 2,
PERMISSION_DENIED = 3,
TABLE_NOT_EXISTS = 4,
ILLEGAL_ARGUMENT = 5,
ILLEGAL_RANGE = 6,
ILLEGAL_DIMENSION = 7,
ILLEGAL_INDEX_TYPE = 8,
ILLEGAL_TABLE_NAME = 9,
ILLEGAL_TOPK = 10,
ILLEGAL_ROWRECORD = 11,
ILLEGAL_VECTOR_ID = 12,
ILLEGAL_SEARCH_RESULT = 13,
FILE_NOT_FOUND = 14,
META_FAILED = 15,
CACHE_FAILED = 16,
CANNOT_CREATE_FOLDER = 17,
CANNOT_CREATE_FILE = 18,
CANNOT_DELETE_FOLDER = 19,
CANNOT_DELETE_FILE = 20,
BUILD_INDEX_ERROR = 21,
ErrorCode_INT_MIN_SENTINEL_DO_NOT_USE_ = std::numeric_limits<::google::protobuf::int32>::min(),
ErrorCode_INT_MAX_SENTINEL_DO_NOT_USE_ = std::numeric_limits<::google::protobuf::int32>::max()
};
bool ErrorCode_IsValid(int value);
const ErrorCode ErrorCode_MIN = SUCCESS;
const ErrorCode ErrorCode_MAX = BUILD_INDEX_ERROR;
const int ErrorCode_ARRAYSIZE = ErrorCode_MAX + 1;
const ::google::protobuf::EnumDescriptor* ErrorCode_descriptor();
inline const ::std::string& ErrorCode_Name(ErrorCode value) {
return ::google::protobuf::internal::NameOfEnum(
ErrorCode_descriptor(), value);
}
inline bool ErrorCode_Parse(
const ::std::string& name, ErrorCode* value) {
return ::google::protobuf::internal::ParseNamedEnum<ErrorCode>(
ErrorCode_descriptor(), name, value);
}
// ===================================================================
class Status final :
public ::google::protobuf::Message /* @@protoc_insertion_point(class_definition:milvus.Status) */ {
public:
Status();
virtual ~Status();
Status(const Status& from);
inline Status& operator=(const Status& from) {
CopyFrom(from);
return *this;
}
#if LANG_CXX11
Status(Status&& from) noexcept
: Status() {
*this = ::std::move(from);
}
inline Status& operator=(Status&& from) noexcept {
if (GetArenaNoVirtual() == from.GetArenaNoVirtual()) {
if (this != &from) InternalSwap(&from);
} else {
CopyFrom(from);
}
return *this;
}
#endif
static const ::google::protobuf::Descriptor* descriptor() {
return default_instance().GetDescriptor();
}
static const Status& default_instance();
static void InitAsDefaultInstance(); // FOR INTERNAL USE ONLY
static inline const Status* internal_default_instance() {
return reinterpret_cast<const Status*>(
&_Status_default_instance_);
}
static constexpr int kIndexInFileMessages =
0;
void Swap(Status* other);
friend void swap(Status& a, Status& b) {
a.Swap(&b);
}
// implements Message ----------------------------------------------
inline Status* New() const final {
return CreateMaybeMessage<Status>(nullptr);
}
Status* New(::google::protobuf::Arena* arena) const final {
return CreateMaybeMessage<Status>(arena);
}
void CopyFrom(const ::google::protobuf::Message& from) final;
void MergeFrom(const ::google::protobuf::Message& from) final;
void CopyFrom(const Status& from);
void MergeFrom(const Status& from);
PROTOBUF_ATTRIBUTE_REINITIALIZES void Clear() final;
bool IsInitialized() const final;
size_t ByteSizeLong() const final;
#if GOOGLE_PROTOBUF_ENABLE_EXPERIMENTAL_PARSER
static const char* _InternalParse(const char* begin, const char* end, void* object, ::google::protobuf::internal::ParseContext* ctx);
::google::protobuf::internal::ParseFunc _ParseFunc() const final { return _InternalParse; }
#else
bool MergePartialFromCodedStream(
::google::protobuf::io::CodedInputStream* input) final;
#endif // GOOGLE_PROTOBUF_ENABLE_EXPERIMENTAL_PARSER
void SerializeWithCachedSizes(
::google::protobuf::io::CodedOutputStream* output) const final;
::google::protobuf::uint8* InternalSerializeWithCachedSizesToArray(
::google::protobuf::uint8* target) const final;
int GetCachedSize() const final { return _cached_size_.Get(); }
private:
void SharedCtor();
void SharedDtor();
void SetCachedSize(int size) const final;
void InternalSwap(Status* other);
private:
inline ::google::protobuf::Arena* GetArenaNoVirtual() const {
return nullptr;
}
inline void* MaybeArenaPtr() const {
return nullptr;
}
public:
::google::protobuf::Metadata GetMetadata() const final;
// nested types ----------------------------------------------------
// accessors -------------------------------------------------------
// string reason = 2;
void clear_reason();
static const int kReasonFieldNumber = 2;
const ::std::string& reason() const;
void set_reason(const ::std::string& value);
#if LANG_CXX11
void set_reason(::std::string&& value);
#endif
void set_reason(const char* value);
void set_reason(const char* value, size_t size);
::std::string* mutable_reason();
::std::string* release_reason();
void set_allocated_reason(::std::string* reason);
// .milvus.ErrorCode error_code = 1;
void clear_error_code();
static const int kErrorCodeFieldNumber = 1;
::milvus::ErrorCode error_code() const;
void set_error_code(::milvus::ErrorCode value);
// @@protoc_insertion_point(class_scope:milvus.Status)
private:
class HasBitSetters;
::google::protobuf::internal::InternalMetadataWithArena _internal_metadata_;
::google::protobuf::internal::ArenaStringPtr reason_;
int error_code_;
mutable ::google::protobuf::internal::CachedSize _cached_size_;
friend struct ::TableStruct_status_2eproto;
};
// ===================================================================
// ===================================================================
#ifdef __GNUC__
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wstrict-aliasing"
#endif // __GNUC__
// Status
// .milvus.ErrorCode error_code = 1;
inline void Status::clear_error_code() {
error_code_ = 0;
}
inline ::milvus::ErrorCode Status::error_code() const {
// @@protoc_insertion_point(field_get:milvus.Status.error_code)
return static_cast< ::milvus::ErrorCode >(error_code_);
}
inline void Status::set_error_code(::milvus::ErrorCode value) {
error_code_ = value;
// @@protoc_insertion_point(field_set:milvus.Status.error_code)
}
// string reason = 2;
inline void Status::clear_reason() {
reason_.ClearToEmptyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited());
}
inline const ::std::string& Status::reason() const {
// @@protoc_insertion_point(field_get:milvus.Status.reason)
return reason_.GetNoArena();
}
inline void Status::set_reason(const ::std::string& value) {
reason_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), value);
// @@protoc_insertion_point(field_set:milvus.Status.reason)
}
#if LANG_CXX11
inline void Status::set_reason(::std::string&& value) {
reason_.SetNoArena(
&::google::protobuf::internal::GetEmptyStringAlreadyInited(), ::std::move(value));
// @@protoc_insertion_point(field_set_rvalue:milvus.Status.reason)
}
#endif
inline void Status::set_reason(const char* value) {
GOOGLE_DCHECK(value != nullptr);
reason_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), ::std::string(value));
// @@protoc_insertion_point(field_set_char:milvus.Status.reason)
}
inline void Status::set_reason(const char* value, size_t size) {
reason_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(),
::std::string(reinterpret_cast<const char*>(value), size));
// @@protoc_insertion_point(field_set_pointer:milvus.Status.reason)
}
inline ::std::string* Status::mutable_reason() {
// @@protoc_insertion_point(field_mutable:milvus.Status.reason)
return reason_.MutableNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited());
}
inline ::std::string* Status::release_reason() {
// @@protoc_insertion_point(field_release:milvus.Status.reason)
return reason_.ReleaseNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited());
}
inline void Status::set_allocated_reason(::std::string* reason) {
if (reason != nullptr) {
} else {
}
reason_.SetAllocatedNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), reason);
// @@protoc_insertion_point(field_set_allocated:milvus.Status.reason)
}
#ifdef __GNUC__
#pragma GCC diagnostic pop
#endif // __GNUC__
// @@protoc_insertion_point(namespace_scope)
} // namespace milvus
namespace google {
namespace protobuf {
template <> struct is_proto_enum< ::milvus::ErrorCode> : ::std::true_type {};
template <>
inline const EnumDescriptor* GetEnumDescriptor< ::milvus::ErrorCode>() {
return ::milvus::ErrorCode_descriptor();
}
} // namespace protobuf
} // namespace google
// @@protoc_insertion_point(global_scope)
#include <google/protobuf/port_undef.inc>
#endif // PROTOBUF_INCLUDED_status_2eproto

250
cpp/src/grpc/milvus.proto Normal file
View File

@ -0,0 +1,250 @@
syntax = "proto3";
import "status.proto";
package milvus.grpc;
/**
* @brief Table Name
*/
message TableName {
milvus.Status status = 1;
string table_name = 2;
}
/**
* @brief Table Schema
*/
message TableSchema {
TableName table_name = 1;
int32 index_type = 2;
int64 dimension = 3;
bool store_raw_vector = 4;
}
/**
* @brief Range Schema
*/
message Range {
string start_value = 1;
string end_value = 2;
}
/**
* @brief Record inserted
*/
message RowRecord {
repeated float vector_data = 1; //binary in thrift
}
/**
* @brief Infos to be inserted
*/
message InsertInfos {
string table_name = 1;
repeated RowRecord row_record_array = 2;
}
/**
* @brief Vector ids
*/
message VectorIds {
milvus.Status status = 1;
repeated int64 vector_id_array = 2;
}
/**
* @brief Infos for searching vector
*/
message SearchVectorInfos {
string table_name = 1;
repeated RowRecord query_record_array = 2;
repeated Range query_range_array = 3;
int64 topk = 4;
}
/**
* @brief Infos for searching vector in files
*/
message SearchVectorInFilesInfos {
repeated string file_id_array = 1;
SearchVectorInfos search_vector_infos = 2;
}
/**
* @brief Query result infos
*/
message QueryResult {
int64 id = 1;
double distance = 2;
}
/**
* @brief TopK query result
*/
message TopKQueryResult {
milvus.Status status = 1;
repeated QueryResult query_result_arrays = 2;
}
/**
* @brief Server String Reply
*/
message StringReply {
milvus.Status status = 1;
string string_reply = 2;
}
/**
* @brief Server bool Reply
*/
message BoolReply {
milvus.Status status = 1;
bool bool_reply = 2;
}
/**
* @brief Return table row count
*/
message TableRowCount {
milvus.Status status = 1;
int64 table_row_count = 2;
}
/**
* @brief Give Server Command
*/
message Command {
string cmd = 1;
}
/**
* @brief Give Server Command
*/
message ServerStatus{
milvus.Status status = 1;
string info = 2;
}
service MilvusService {
/**
* @brief Create table method
*
* This method is used to create table
*
* @param param, use to provide table information to be created.
*
*/
rpc CreateTable(TableSchema) returns (milvus.Status){}
/**
* @brief Test table existence method
*
* This method is used to test table existence.
*
* @param table_name, table name is going to be tested.
*
*/
rpc HasTable(TableName) returns (BoolReply) {}
/**
* @brief Delete table method
*
* This method is used to delete table.
*
* @param table_name, table name is going to be deleted.
*
*/
rpc DropTable(TableName) returns (milvus.Status) {}
/**
* @brief Build index by table method
*
* This method is used to build index by table in sync mode.
*
* @param table_name, table is going to be built index.
*
*/
rpc BuildIndex(TableName) returns (milvus.Status) {}
/**
* @brief Add vector array to table
*
* This method is used to add vector array to table.
*
* @param table_name, table_name is inserted.
* @param record_array, vector array is inserted.
*
* @return vector id array
*/
rpc InsertVector(InsertInfos) returns (VectorIds) {}
/**
* @brief Query vector
*
* This method is used to query vector in table.
*
* @param table_name, table_name is queried.
* @param query_record_array, all vector are going to be queried.
* @param query_range_array, optional ranges for conditional search. If not specified, search whole table
* @param topk, how many similarity vectors will be searched.
*
* @return query result array.
*/
rpc SearchVector(SearchVectorInfos) returns (stream TopKQueryResult) {}
/**
* @brief Internal use query interface
*
* This method is used to query vector in specified files.
*
* @param file_id_array, specified files id array, queried.
* @param query_record_array, all vector are going to be queried.
* @param query_range_array, optional ranges for conditional search. If not specified, search whole table
* @param topk, how many similarity vectors will be searched.
*
* @return query result array.
*/
rpc SearchVectorInFiles(SearchVectorInFilesInfos) returns (stream TopKQueryResult) {}
/**
* @brief Get table schema
*
* This method is used to get table schema.
*
* @param table_name, target table name.
*
* @return table schema
*/
rpc DescribeTable(TableName) returns (TableSchema) {}
/**
* @brief Get table schema
*
* This method is used to get table schema.
*
* @param table_name, target table name.
*
* @return table schema
*/
rpc GetTableRowCount(TableName) returns (TableRowCount) {}
/**
* @brief List all tables in database
*
* This method is used to list all tables.
*
*
* @return table names.
*/
rpc ShowTables(Command) returns (stream TableName) {}
/**
* @brief Give the server status
*
* This method is used to give the server status.
*
* @return Server status.
*/
rpc Ping(Command) returns (ServerStatus) {}
}

33
cpp/src/grpc/status.proto Normal file
View File

@ -0,0 +1,33 @@
syntax = "proto3";
package milvus;
enum ErrorCode {
SUCCESS = 0;
UNEXPECTED_ERROR = 1;
CONNECT_FAILED = 2;
PERMISSION_DENIED = 3;
TABLE_NOT_EXISTS = 4;
ILLEGAL_ARGUMENT = 5;
ILLEGAL_RANGE = 6;
ILLEGAL_DIMENSION = 7;
ILLEGAL_INDEX_TYPE = 8;
ILLEGAL_TABLE_NAME = 9;
ILLEGAL_TOPK = 10;
ILLEGAL_ROWRECORD = 11;
ILLEGAL_VECTOR_ID = 12;
ILLEGAL_SEARCH_RESULT = 13;
FILE_NOT_FOUND = 14;
META_FAILED = 15;
CACHE_FAILED = 16;
CANNOT_CREATE_FOLDER = 17;
CANNOT_CREATE_FILE = 18;
CANNOT_DELETE_FOLDER = 19;
CANNOT_DELETE_FILE = 20;
BUILD_INDEX_ERROR = 21;
}
message Status {
ErrorCode error_code = 1;
string reason = 2;
}

View File

@ -0,0 +1,38 @@
#-------------------------------------------------------------------------------
# Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved
# Unauthorized copying of this file, via any medium is strictly prohibited.
# Proprietary and confidential.
#-------------------------------------------------------------------------------
aux_source_directory(src/interface interface_files)
aux_source_directory(src/client client_files)
aux_source_directory(src/util util_files)
include_directories(src)
include_directories(include)
include_directories(/usr/include)
include_directories(${CMAKE_SOURCE_DIR}/src/grpc/gen-milvus)
include_directories(${CMAKE_SOURCE_DIR}/src/grpc/gen-status)
include_directories(/usr/local/include)
set(service_files
${CMAKE_SOURCE_DIR}/src/grpc/gen-milvus/milvus.grpc.pb.cc
${CMAKE_SOURCE_DIR}/src/grpc/gen-milvus/milvus.pb.cc
${CMAKE_SOURCE_DIR}/src/grpc/gen-status/status.grpc.pb.cc
${CMAKE_SOURCE_DIR}/src/grpc/gen-status/status.pb.cc
)
add_library(milvus_grpc_sdk STATIC
${interface_files}
${client_files}
${util_files}
${service_files}
)
target_link_libraries(milvus_grpc_sdk
${third_party_libs}
)
add_subdirectory(examples)
install(TARGETS milvus_grpc_sdk DESTINATION lib)

View File

@ -0,0 +1,24 @@
#-------------------------------------------------------------------------------
# Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved
# Unauthorized copying of this file, via any medium is strictly prohibited.
# Proprietary and confidential.
#-------------------------------------------------------------------------------
aux_source_directory(src src_files)
include_directories(src)
include_directories(../include)
link_directories(${CMAKE_BINARY_DIR})
add_executable(grpc_sdk_simple
./main.cpp
${src_files}
)
target_link_libraries(grpc_sdk_simple
milvus_grpc_sdk
pthread
)
install(TARGETS grpc_sdk_simple DESTINATION bin)

View File

@ -0,0 +1,68 @@
////////////////////////////////////////////////////////////////////////////////
// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved
// Unauthorized copying of this file, via any medium is strictly prohibited.
// Proprietary and confidential.
////////////////////////////////////////////////////////////////////////////////
#include <getopt.h>
#include <libgen.h>
#include <cstring>
#include <string>
#include "src/ClientTest.h"
void print_help(const std::string &app_name);
int
main(int argc, char *argv[]) {
printf("Client start...\n");
std::string app_name = basename(argv[0]);
static struct option long_options[] = {{"server", optional_argument, 0, 's'},
{"port", optional_argument, 0, 'p'},
{"help", no_argument, 0, 'h'},
{NULL, 0, 0, 0}};
int option_index = 0;
std::string address = "127.0.0.1", port = "19530";
app_name = argv[0];
int value;
while ((value = getopt_long(argc, argv, "s:p:h", long_options, &option_index)) != -1) {
switch (value) {
case 's': {
char *address_ptr = strdup(optarg);
address = address_ptr;
free(address_ptr);
break;
}
case 'p': {
char *port_ptr = strdup(optarg);
port = port_ptr;
free(port_ptr);
break;
}
case 'h':
default:
print_help(app_name);
return EXIT_SUCCESS;
}
}
ClientTest test;
test.Test(address, port);
printf("Client stop...\n");
return 0;
}
void
print_help(const std::string &app_name) {
printf("\n Usage: %s [OPTIONS]\n\n", app_name.c_str());
printf(" Options:\n");
printf(" -s --server Server address, default 127.0.0.1\n");
printf(" -p --port Server port, default 19530\n");
printf(" -h --help Print help information\n");
printf("\n");
}

View File

@ -0,0 +1,291 @@
/*******************************************************************************
* Copyright (Zilliz) - All Rights Reserved
* Unauthorized copying of this file, via any medium is strictly prohibited.
* Proprietary and confidential.
******************************************************************************/
#include "ClientTest.h"
#include "MilvusApi.h"
#include <iostream>
#include <time.h>
#include <chrono>
#include <thread>
#include <unistd.h>
using namespace ::zilliz::milvus;
namespace {
std::string GetTableName();
static const std::string TABLE_NAME = GetTableName();
static constexpr int64_t TABLE_DIMENSION = 512;
static constexpr int64_t BATCH_ROW_COUNT = 100000;
static constexpr int64_t NQ = 10;
static constexpr int64_t TOP_K = 10;
static constexpr int64_t SEARCH_TARGET = 5000; //change this value, result is different
static constexpr int64_t ADD_VECTOR_LOOP = 5;
static constexpr int64_t SECONDS_EACH_HOUR = 3600;
#define BLOCK_SPLITER std::cout << "===========================================" << std::endl;
void PrintTableSchema(const TableSchema& tb_schema) {
BLOCK_SPLITER
std::cout << "Table name: " << tb_schema.table_name.table_name << std::endl;
std::cout << "Table index type: " << (int)tb_schema.index_type << std::endl;
std::cout << "Table dimension: " << tb_schema.dimension << std::endl;
std::cout << "Table store raw data: " << (tb_schema.store_raw_vector ? "true" : "false") << std::endl;
BLOCK_SPLITER
}
void PrintSearchResult(const std::vector<std::pair<int64_t, RowRecord>>& search_record_array,
const std::vector<TopKQueryResult>& topk_query_result_array) {
BLOCK_SPLITER
std::cout << "Returned result count: " << topk_query_result_array.size() << std::endl;
int32_t index = 0;
for(auto& result : topk_query_result_array) {
auto search_id = search_record_array[index].first;
index++;
std::cout << "No." << std::to_string(index) << " vector " << std::to_string(search_id)
<< " top " << std::to_string(result.query_result_arrays.size())
<< " search result:" << std::endl;
for(auto& item : result.query_result_arrays) {
std::cout << "\t" << std::to_string(item.id) << "\tdistance:" << std::to_string(item.distance);
std::cout << std::endl;
}
}
BLOCK_SPLITER
}
std::string CurrentTime() {
time_t tt;
time( &tt );
tt = tt + 8*SECONDS_EACH_HOUR;
tm* t= gmtime( &tt );
std::string str = std::to_string(t->tm_year + 1900) + "_" + std::to_string(t->tm_mon + 1)
+ "_" + std::to_string(t->tm_mday) + "_" + std::to_string(t->tm_hour)
+ "_" + std::to_string(t->tm_min) + "_" + std::to_string(t->tm_sec);
return str;
}
std::string CurrentTmDate(int64_t offset_day = 0) {
time_t tt;
time( &tt );
tt = tt + 8*SECONDS_EACH_HOUR;
tt = tt + 24*SECONDS_EACH_HOUR*offset_day;
tm* t= gmtime( &tt );
std::string str = std::to_string(t->tm_year + 1900) + "-" + std::to_string(t->tm_mon + 1)
+ "-" + std::to_string(t->tm_mday);
return str;
}
std::string GetTableName() {
static std::string s_id(CurrentTime());
return "tbl_" + s_id;
}
TableSchema BuildTableSchema() {
TableSchema tb_schema;
tb_schema.table_name.table_name = TABLE_NAME;
tb_schema.index_type = IndexType::gpu_ivfflat;
tb_schema.dimension = TABLE_DIMENSION;
tb_schema.store_raw_vector = true;
return tb_schema;
}
void BuildVectors(int64_t from, int64_t to,
std::vector<RowRecord>& vector_record_array) {
if(to <= from){
return;
}
vector_record_array.clear();
for (int64_t k = from; k < to; k++) {
RowRecord record;
record.data.resize(TABLE_DIMENSION);
for(int64_t i = 0; i < TABLE_DIMENSION; i++) {
record.data[i] = (float)(k%(i+1));
}
vector_record_array.emplace_back(record);
}
}
void Sleep(int seconds) {
std::cout << "Waiting " << seconds << " seconds ..." << std::endl;
sleep(seconds);
}
class TimeRecorder {
public:
TimeRecorder(const std::string& title)
: title_(title) {
start_ = std::chrono::system_clock::now();
}
~TimeRecorder() {
std::chrono::system_clock::time_point end = std::chrono::system_clock::now();
long span = (std::chrono::duration_cast<std::chrono::milliseconds> (end - start_)).count();
std::cout << title_ << " totally cost: " << span << " ms" << std::endl;
}
private:
std::string title_;
std::chrono::system_clock::time_point start_;
};
void CheckResult(const std::vector<std::pair<int64_t, RowRecord>>& search_record_array,
const std::vector<TopKQueryResult>& topk_query_result_array) {
BLOCK_SPLITER
int64_t index = 0;
for(auto& result : topk_query_result_array) {
auto result_id = result.query_result_arrays[0].id;
auto search_id = search_record_array[index++].first;
if(result_id != search_id) {
std::cout << "The top 1 result is wrong: " << result_id
<< " vs. " << search_id << std::endl;
} else {
std::cout << "Check result sucessfully" << std::endl;
}
}
BLOCK_SPLITER
}
void DoSearch(std::shared_ptr<Connection> conn,
const std::vector<std::pair<int64_t, RowRecord>>& search_record_array,
const std::string& phase_name) {
std::vector<Range> query_range_array;
Range rg;
rg.start_value = CurrentTmDate();
rg.end_value = CurrentTmDate(1);
query_range_array.emplace_back(rg);
std::vector<RowRecord> record_array;
for(auto& pair : search_record_array) {
record_array.push_back(pair.second);
}
std::vector<TopKQueryResult> topk_query_result_array;
{
TimeRecorder rc(phase_name);
Status stat = conn->SearchVector(TABLE_NAME, record_array, query_range_array, TOP_K, topk_query_result_array);
std::cout << "SearchVector function call status: " << stat.ToString() << std::endl;
}
PrintSearchResult(search_record_array, topk_query_result_array);
CheckResult(search_record_array, topk_query_result_array);
}
}
void
ClientTest::Test(const std::string& address, const std::string& port) {
std::shared_ptr<Connection> conn = Connection::Create();
{//connect server
ConnectParam param = {address, port};
Status stat = conn->Connect(param);
std::cout << "Connect function call status: " << stat.ToString() << std::endl;
}
{//server version
std::string version = conn->ServerVersion();
std::cout << "Server version: " << version << std::endl;
}
{//sdk version
std::string version = conn->ClientVersion();
std::cout << "SDK version: " << version << std::endl;
}
{
std::vector<std::string> tables;
Status stat = conn->ShowTables(tables);
std::cout << "ShowTables function call status: " << stat.ToString() << std::endl;
std::cout << "All tables: " << std::endl;
for(auto& table : tables) {
int64_t row_count = 0;
stat = conn->GetTableRowCount(table, row_count);
std::cout << "\t" << table << "(" << row_count << " rows)" << std::endl;
}
}
{//create table
TableSchema tb_schema = BuildTableSchema();
Status stat = conn->CreateTable(tb_schema);
std::cout << "CreateTable function call status: " << stat.ToString() << std::endl;
PrintTableSchema(tb_schema);
bool has_table = conn->HasTable(tb_schema.table_name.table_name);
if(has_table) {
std::cout << "Table is created" << std::endl;
}
}
{//describe table
TableSchema tb_schema;
Status stat = conn->DescribeTable(TABLE_NAME, tb_schema);
std::cout << "DescribeTable function call status: " << stat.ToString() << std::endl;
PrintTableSchema(tb_schema);
}
std::vector<std::pair<int64_t, RowRecord>> search_record_array;
{//insert vectors
for (int i = 0; i < ADD_VECTOR_LOOP; i++) {//add vectors
std::vector<RowRecord> record_array;
int64_t begin_index = i * BATCH_ROW_COUNT;
BuildVectors(begin_index, begin_index + BATCH_ROW_COUNT, record_array);
std::vector<int64_t> record_ids;
auto start = std::chrono::high_resolution_clock::now();
Status stat = conn->InsertVector(TABLE_NAME, record_array, record_ids);
auto finish = std::chrono::high_resolution_clock::now();
std::cout << "InsertVector cost: " << std::chrono::duration_cast<std::chrono::duration<double>>(finish - start).count() << "s\n";
std::cout << "InsertVector function call status: " << stat.ToString() << std::endl;
std::cout << "Returned id array count: " << record_ids.size() << std::endl;
if(search_record_array.size() < NQ) {
search_record_array.push_back(
std::make_pair(record_ids[SEARCH_TARGET], record_array[SEARCH_TARGET]));
}
}
}
{//search vectors without index
Sleep(2);
DoSearch(conn, search_record_array, "Search without index");
}
{//wait unit build index finish
std::cout << "Wait until build all index done" << std::endl;
Status stat = conn->BuildIndex(TABLE_NAME);
std::cout << "BuildIndex function call status: " << stat.ToString() << std::endl;
}
{//search vectors after build index finish
DoSearch(conn, search_record_array, "Search after build index finish");
}
{//delete table
Status stat = conn->DropTable(TABLE_NAME);
std::cout << "DeleteTable function call status: " << stat.ToString() << std::endl;
}
{//server status
std::string status = conn->ServerStatus();
std::cout << "Server status before disconnect: " << status << std::endl;
}
// Connection::Destroy(conn);
conn->Disconnect();
{//server status
std::string status = conn->ServerStatus();
std::cout << "Server status after disconnect: " << status << std::endl;
}
}

View File

@ -0,0 +1,13 @@
/*******************************************************************************
* Copyright (Zilliz) - All Rights Reserved
* Unauthorized copying of this file, via any medium is strictly prohibited.
* Proprietary and confidential.
******************************************************************************/
#pragma once
#include <string>
class ClientTest {
public:
void Test(const std::string& address, const std::string& port);
};

View File

@ -0,0 +1,384 @@
#pragma onceinclude_directories(/usr/include)
#include "Status.h"
#include <string>
#include <vector>
#include <memory>
#include <thread>
/** \brief Milvus SDK namespace
*/
namespace zilliz {
namespace milvus {
//enum Error_Code {
// SUCCESS = 0,
// UNEXPECTED_ERROR = 1,
// CONNECT_FAILED = 2,
// PERMISSION_DENIED = 3,
// TABLE_NOT_EXISTS = 4,
// ILLEGAL_ARGUMENT = 5,
// ILLEGAL_RANGE = 6,
// ILLEGAL_DIMENSION = 7,
// ILLEGAL_INDEX_TYPE = 8,
// ILLEGAL_TABLE_NAME = 9,
// ILLEGAL_TOPK = 10,
// ILLEGAL_ROWRECORD = 11,
// ILLEGAL_VECTOR_ID = 12,
// ILLEGAL_SEARCH_RESULT = 13,
// FILE_NOT_FOUND = 14,
// META_FAILED = 15,
// CACHE_FAILED = 16,
// CANNOT_CREATE_FOLDER = 17,
// CANNOT_CREATE_FILE = 18,
// CANNOT_DELETE_FOLDER = 19,
// CANNOT_DELETE_FILE = 20,
//};
/**
* @brief Index Type
*/
enum class IndexType {
invalid = 0,
cpu_idmap,
gpu_ivfflat,
gpu_ivfsq8,
};
/**
* @brief Connect API parameter
*/
struct ConnectParam {
std::string ip_address; ///< Server IP address
std::string port; ///< Server PORT
};
/**
* @brief Status for return
*/
struct StatusMsg {
// Error_Code errorCode;
std::string reason;
};
/**
* @brief Table Name
*/
struct TableName {
StatusMsg status;
std::string table_name;
};
/**
* @brief Table Schema
*/
struct TableSchema {
TableName table_name; ///< Table name
IndexType index_type = IndexType::invalid; ///< Index type
int64_t dimension = 0; ///< Vector dimension, must be a positive value
bool store_raw_vector = false; ///< Is vector raw data stored in the table
};
/**
* @brief Range information
* for DATE partition, the format is like: 'year-month-day'
*/
struct Range {
std::string start_value; ///< Range start
std::string end_value; ///< Range stop
};
/**
* @brief Record inserted
*/
struct RowRecord {
std::vector<float> data; ///< Vector raw data
};
//struct InsertInfos {
// std::string table_name;
// std::vector<RowRecord> row_record_array;
//};
/**
* @brief Vector ids for return
*/
struct VectorIds {
StatusMsg status;
std::vector<int64_t> vector_id_array;
};
/**
* @brief Infos for searching vector
*/
struct SearchVectorInfos {
std::string table_name;
std::vector<RowRecord> query_record_array;
std::vector<Range> query_range_array;
int64_t topk;
};
/**
* @brief Infos for searching vector in files
*/
struct SearchVectorInFilesInfos {
std::vector<std::string> file_id_array;
SearchVectorInfos search_vector_infos;
};
/**
* @brief Query result
*/
struct QueryResult {
int64_t id; ///< Output result
double distance; ///< Vector similarity distance
};
/**
* @brief TopK query result
*/
struct TopKQueryResult {
StatusMsg status;
std::vector<QueryResult> query_result_arrays; ///< TopK query result
};
/**
* @brief Server bool Reply
*/
struct BoolReply {
StatusMsg status;
bool bool_reply;
};
/**
* @brief Return table row count
*/
struct TableRowCount {
StatusMsg status;
int64_t table_row_count;
};
/**
* @brief SDK main class
*/
class Connection {
public:
/**
* @brief CreateConnection
*
* Create a connection instance and return it's shared pointer
*
* @return Connection instance pointer
*/
static std::shared_ptr<Connection>
Create();
/**
* @brief DestroyConnection
*
* Destroy the connection instance
*
* @param connection, the shared pointer to the instance to be destroyed
*
* @return if destroy is successful
*/
static Status
Destroy(std::shared_ptr<Connection> connection_ptr);
/**
* @brief Connect
*
* Connect function should be called before any operations
* Server will be connected after Connect return OK
*
* @param param, use to provide server information
*
* @return Indicate if connect is successful
*/
virtual Status Connect(const ConnectParam &param) = 0;
/**
* @brief Connect
*
* Connect function should be called before any operations
* Server will be connected after Connect return OK
*
* @param uri, use to provide server information, example: milvus://ipaddress:port
*
* @return Indicate if connect is successful
*/
virtual Status Connect(const std::string &uri) = 0;
/**
* @brief connected
*
* Connection status.
*
* @return Indicate if connection status
*/
virtual Status Connected() const = 0;
/**
* @brief Disconnect
*
* Server will be disconnected after Disconnect return OK
*
* @return Indicate if disconnect is successful
*/
virtual Status Disconnect() = 0;
/**
* @brief Create table method
*
* This method is used to create table
*
* @param param, use to provide table information to be created.
*
* @return Indicate if table is created successfully
*/
virtual Status CreateTable(const TableSchema &param) = 0;
/**
* @brief Test table existence method
*
* This method is used to create table
*
* @param table_name, table name is going to be tested.
*
* @return Indicate if table is cexist
*/
virtual bool HasTable(const std::string &table_name) = 0;
/**
* @brief Drop table method
*
* This method is used to drop table.
*
* @param table_name, table name is going to be dropped.
*
* @return Indicate if table is drop successfully.
*/
virtual Status DropTable(const std::string &table_name) = 0;
/**
* @brief Build index method
*
* This method is used to build index for whole table
*
* @param table_name, table name is going to be build index.
*
* @return Indicate if build index successfully.
*/
virtual Status BuildIndex(const std::string &table_name) = 0;
/**
* @brief Add vector to table
*
* This method is used to add vector array to table.
*
* @param table_name, table_name is inserted.
* @param record_array, vector array is inserted.
* @param id_array, after inserted every vector is given a id.
*
* @return Indicate if vector array are inserted successfully
*/
virtual Status InsertVector(const std::string &table_name,
const std::vector<RowRecord> &record_array,
std::vector<int64_t> &id_array) = 0;
/**
* @brief Search vector
*
* This method is used to query vector in table.
*
* @param table_name, table_name is queried.
* @param query_record_array, all vector are going to be queried.
* @param query_range_array, time ranges, if not specified, will search in whole table
* @param topk, how many similarity vectors will be searched.
* @param topk_query_result_array, result array.
*
* @return Indicate if query is successful.
*/
virtual Status SearchVector(const std::string &table_name,
const std::vector<RowRecord> &query_record_array,
const std::vector<Range> &query_range_array,
int64_t topk,
std::vector<TopKQueryResult> &topk_query_result_array) = 0;
/**
* @brief Show table description
*
* This method is used to show table information.
*
* @param table_name, which table is show.
* @param table_schema, table_schema is given when operation is successful.
*
* @return Indicate if this operation is successful.
*/
virtual Status DescribeTable(const std::string &table_name, TableSchema &table_schema) = 0;
/**
* @brief Get table row count
*
* This method is used to get table row count.
*
* @param table_name, table's name.
* @param row_count, table total row count.
*
* @return Indicate if this operation is successful.
*/
virtual Status GetTableRowCount(const std::string &table_name, int64_t &row_count) = 0;
/**
* @brief Show all tables in database
*
* This method is used to list all tables.
*
* @param table_array, all tables are push into the array.
*
* @return Indicate if this operation is successful.
*/
virtual Status ShowTables(std::vector<std::string> &table_array) = 0;
/**
* @brief Give the client version
*
* This method is used to give the client version.
*
* @return Client version.
*/
virtual std::string ClientVersion() const = 0;
/**
* @brief Give the server version
*
* This method is used to give the server version.
*
* @return Server version.
*/
virtual std::string ServerVersion() const = 0;
/**
* @brief Give the server status
*
* This method is used to give the server status.
*
* @return Server status.
*/
virtual std::string ServerStatus() const = 0;
};
}
}

View File

@ -0,0 +1,330 @@
#pragma once
#include <string>
#include <sstream>
/** \brief Milvus SDK namespace
*/
namespace zilliz {
namespace milvus {
/**
* @brief Status Code for SDK interface return
*/
enum class StatusCode {
OK = 0,
// system error section
UnknownError = 1,
NotSupported,
NotConnected,
// function error section
InvalidAgument = 1000,
};
/**
* @brief Status for SDK interface return
*/
class Status {
public:
/**
* @brief Status
*
* Default constructor.
*
*/
Status() = default;
/**
* @brief Status
*
* Destructor.
*
*/
~Status() noexcept;
/**
* @brief Status
*
* Constructor
*
* @param code, status code.
* @param message, status message.
*
*/
Status(StatusCode code, const std::string &message);
/**
* @brief Status
*
* Copy constructor
*
* @param status, status to be copied.
*
*/
inline Status(const Status &status);
/**
* @brief Status
*
* Assignment operator
*
* @param status, status to be copied.
* @return, the status is assigned.
*
*/
Status &operator=(const Status &s);
/**
* @brief Status
*
* Move constructor
*
* @param status, status to be moved.
*
*/
inline Status(Status &&s) noexcept : state_(s.state_) {};
/**
* @brief Status
*
* Move assignment operator
*
* @param status, status to be moved.
* @return, the status is moved.
*
*/
Status &operator=(Status &&s) noexcept;
/**
* @brief Status
*
* AND operator
*
* @param status, status to be AND.
* @return, the status after AND operation.
*
*/
inline Status operator&(const Status &s) const noexcept;
/**
* @brief Status
*
* AND operator
*
* @param status, status to be AND.
* @return, the status after AND operation.
*
*/
inline Status operator&(Status &&s) const noexcept;
/**
* @brief Status
*
* AND operator
*
* @param status, status to be AND.
* @return, the status after AND operation.
*
*/
inline Status &operator&=(const Status &s) noexcept;
/**
* @brief Status
*
* AND operator
*
* @param status, status to be AND.
* @return, the status after AND operation.
*
*/
inline Status &operator&=(Status &&s) noexcept;
/**
* @brief OK
*
* static OK status constructor
*
* @return, the status with OK.
*
*/
static Status OK() { return Status(); }
/**
* @brief OK
*
* static OK status constructor with a specific message
*
* @param, serveral specific messages
* @return, the status with OK.
*
*/
template<typename... Args>
static Status OK(Args &&... args) {
return Status(StatusCode::OK, MessageBuilder(std::forward<Args>(args)...));
}
/**
* @brief Invalid
*
* static Invalid status constructor with a specific message
*
* @param, serveral specific messages
* @return, the status with Invalid.
*
*/
template<typename... Args>
static Status Invalid(Args &&... args) {
return Status(StatusCode::InvalidAgument,
MessageBuilder(std::forward<Args>(args)...));
}
/**
* @brief Unknown Error
*
* static unknown error status constructor with a specific message
*
* @param, serveral specific messages
* @return, the status with unknown error.
*
*/
template<typename... Args>
static Status UnknownError(Args &&... args) {
return Status(StatusCode::UnknownError, MessageBuilder(std::forward<Args>(args)...));
}
/**
* @brief not supported Error
*
* static not supported status constructor with a specific message
*
* @param, serveral specific messages
* @return, the status with not supported error.
*
*/
template<typename... Args>
static Status NotSupported(Args &&... args) {
return Status(StatusCode::NotSupported, MessageBuilder(std::forward<Args>(args)...));
}
/**
* @brief ok
*
* Return true iff the status indicates success.
*
* @return, if the status indicates success.
*
*/
bool ok() const { return (state_ == nullptr); }
/**
* @brief IsInvalid
*
* Return true iff the status indicates invalid.
*
* @return, if the status indicates invalid.
*
*/
bool IsInvalid() const { return code() == StatusCode::InvalidAgument; }
/**
* @brief IsUnknownError
*
* Return true iff the status indicates unknown error.
*
* @return, if the status indicates unknown error.
*
*/
bool IsUnknownError() const { return code() == StatusCode::UnknownError; }
/**
* @brief IsNotSupported
*
* Return true iff the status indicates not supported.
*
* @return, if the status indicates not supported.
*
*/
bool IsNotSupported() const { return code() == StatusCode::NotSupported; }
/**
* @brief ToString
*
* Return error message string.
*
* @return, error message string.
*
*/
std::string ToString() const;
/**
* @brief CodeAsString
*
* Return a string representation of the status code.
*
* @return, a string representation of the status code.
*
*/
std::string CodeAsString() const;
/**
* @brief code
*
* Return the StatusCode value attached to this status.
*
* @return, the status code value attached to this status.
*
*/
StatusCode code() const { return ok() ? StatusCode::OK : state_->code; }
/**
* @brief message
*
* Return the specific error message attached to this status.
*
* @return, the specific error message attached to this status.
*
*/
std::string message() const { return ok() ? "" : state_->message; }
private:
struct State {
StatusCode code;
std::string message;
};
// OK status has a `nullptr` state_. Otherwise, `state_` points to
// a `State` structure containing the error code and message.
State *state_ = nullptr;
void DeleteState() {
delete state_;
state_ = nullptr;
}
void CopyFrom(const Status &s);
inline void MoveFrom(Status &s);
template<typename Head>
static void MessageBuilderRecursive(std::stringstream &stream, Head &&head) {
stream << head;
}
template<typename Head, typename... Tail>
static void MessageBuilderRecursive(std::stringstream &stream, Head &&head, Tail &&... tail) {
MessageBuilderRecursive(stream, std::forward<Head>(head));
MessageBuilderRecursive(stream, std::forward<Tail>(tail)...);
}
template<typename... Args>
static std::string MessageBuilder(Args &&... args) {
std::stringstream stream;
MessageBuilderRecursive(stream, std::forward<Args>(args)...);
return stream.str();
}
};
}
}

View File

@ -0,0 +1,313 @@
/*******************************************************************************
* Copyright (Zilliz) - All Rights Reserved
* Unauthorized copying of this file, via any medium is strictly prohibited.
* Proprietary and confidential.
******************************************************************************/
#include "ClientProxy.h"
#include "milvus.grpc.pb.h"
namespace zilliz {
namespace milvus {
Status
ClientProxy::Connect(const ConnectParam &param) {
// Disconnect();
std::string uri = param.ip_address + ":" + param.port;
channel_ = ::grpc::CreateChannel(uri, ::grpc::InsecureChannelCredentials());
client_ptr = new grpcClient(channel_);
if (channel_ != nullptr) {
connected_ = true;
}
return Status::OK();
}
Status
ClientProxy::Connect(const std::string &uri) {
// Disconnect();
size_t index = uri.find_first_of(":", 0);
if ((index == std::string::npos)) {
return Status::Invalid("Invalid uri");
}
ConnectParam param;
param.ip_address = uri.substr(0, index);
param.port = uri.substr(index + 1);
return Connect(param);
}
Status
ClientProxy::Connected() const {
try {
std::string info;
client_ptr->Ping(info, "");
} catch (std::exception &ex) {
return Status(StatusCode::NotConnected, "connection lost: " + std::string(ex.what()));
}
return Status::OK();
}
Status
ClientProxy::Disconnect() {
connected_ = false;
// delete client_ptr;
return Status::OK();
}
std::string
ClientProxy::ClientVersion() const {
return "";
}
Status
ClientProxy::CreateTable(const TableSchema &param) {
try {
::milvus::grpc::TableSchema schema;
// ::milvus::grpc::TableName *grpc_tablename = new ::milvus::grpc::TableName;
// grpc_tablename->set_table_name(param.table_name.table_name);
// schema.set_allocated_table_name(grpc_tablename);
schema.mutable_table_name()->set_table_name(param.table_name.table_name);
schema.set_index_type((int) param.index_type);
schema.set_dimension(param.dimension);
schema.set_store_raw_vector(param.store_raw_vector);
client_ptr->CreateTable(schema);//stub call
} catch (std::exception &ex) {
return Status(StatusCode::UnknownError, "failed to create table: " + std::string(ex.what()));
}
return Status::OK();
}
bool
ClientProxy::HasTable(const std::string &table_name) {
::milvus::grpc::TableName grpc_table_name;
grpc_table_name.set_table_name(table_name);
return client_ptr->HasTable(grpc_table_name);
}
Status
ClientProxy::DropTable(const std::string &table_name) {
try {
::milvus::grpc::TableName grpc_table_name;
grpc_table_name.set_table_name(table_name);
client_ptr->DropTable(grpc_table_name);
} catch (std::exception &ex) {
return Status(StatusCode::UnknownError, "failed to drop table: " + std::string(ex.what()));
}
return Status::OK();
}
Status
ClientProxy::BuildIndex(const std::string &table_name) {
try {
::milvus::grpc::TableName grpc_table_name;
grpc_table_name.set_table_name(table_name);
client_ptr->BuildIndex(grpc_table_name);
} catch (std::exception &ex) {
return Status(StatusCode::UnknownError, "failed to build index: " + std::string(ex.what()));
}
return Status::OK();
}
Status
ClientProxy::InsertVector(const std::string &table_name,
const std::vector<RowRecord> &record_array,
std::vector<int64_t> &id_array) {
try {
// ::milvus::grpc::InsertInfos insert_infos;
// insert_infos.set_table_name(table_name);
//
// for (auto &record : record_array) {
// ::milvus::grpc::RowRecord *grpc_record = insert_infos.add_row_record_array();
// for (size_t i = 0; i < record.data.size(); i++) {
// grpc_record->add_vector_data(record.data[i]);
// }
// }
//
// ::milvus::grpc::VectorIds vector_ids;
//
// std::cout << "*****************************************************\n";
// auto start = std::chrono::high_resolution_clock::now();
// //Single thread
// client_ptr->InsertVector(vector_ids, insert_infos);
// auto finish = std::chrono::high_resolution_clock::now();
// std::cout << "InsertVector cost: " << std::chrono::duration_cast<std::chrono::duration<double>>(finish - start).count() << "s\n";
// std::cout << "*****************************************************\n";
//
// for (size_t i = 0; i < vector_ids.vector_id_array_size(); i++) {
// id_array.push_back(vector_ids.vector_id_array(i));
// }
////////////////////////////////////////////////////////////////////////////
//multithread
std::vector<std::thread> threads;
// int thread_count = std::thread::hardware_concurrency();
int thread_count = 10;
::milvus::grpc::InsertInfos *insert_info_array = new ::milvus::grpc::InsertInfos[thread_count];
::milvus::grpc::VectorIds *vector_ids_array = new ::milvus::grpc::VectorIds[thread_count];
int64_t record_count = record_array.size() / thread_count;
for (size_t i = 0; i < thread_count; i++) {
insert_info_array[i].set_table_name(table_name);
for (size_t j = i * record_count; j < record_count * (i + 1); j++) {
::milvus::grpc::RowRecord *grpc_record = insert_info_array[i].add_row_record_array();
for (size_t k = 0; k < record_array[j].data.size(); k++) {
grpc_record->add_vector_data(record_array[j].data[k]);
}
}
}
std::cout << "*****************************************************\n";
auto start = std::chrono::high_resolution_clock::now();
for (size_t j = 0; j < thread_count; j++) {
threads.push_back(
std::thread(&grpcClient::InsertVector, client_ptr, std::ref(vector_ids_array[j]), insert_info_array[j]));
}
std::for_each(threads.begin(), threads.end(), std::mem_fn(&std::thread::join));
auto finish = std::chrono::high_resolution_clock::now();
std::cout << "InsertVector cost: " << std::chrono::duration_cast<std::chrono::duration<double>>(finish - start).count() << "s\n";
std::cout << "*****************************************************\n";
for (size_t i = 0; i < thread_count; i++) {
for (size_t j = 0; j < vector_ids_array[i].vector_id_array_size(); j++) {
id_array.push_back(vector_ids_array[i].vector_id_array(j));
}
}
} catch (std::exception &ex) {
return Status(StatusCode::UnknownError, "failed to add vector: " + std::string(ex.what()));
}
return Status::OK();
}
Status
ClientProxy::SearchVector(const std::string &table_name,
const std::vector<RowRecord> &query_record_array,
const std::vector<Range> &query_range_array,
int64_t topk,
std::vector<TopKQueryResult> &topk_query_result_array) {
try {
//step 1: convert vectors data
::milvus::grpc::SearchVectorInfos search_vector_infos;
search_vector_infos.set_table_name(table_name);
search_vector_infos.set_topk(topk);
for (auto &record : query_record_array) {
::milvus::grpc::RowRecord *row_record = search_vector_infos.add_query_record_array();
for (size_t i = 0; i < record.data.size(); i++) {
row_record->add_vector_data(record.data[i]);
}
}
//step 2: convert range array
for (auto &range : query_range_array) {
::milvus::grpc::Range *grpc_range = search_vector_infos.add_query_range_array();
grpc_range->set_start_value(range.start_value);
grpc_range->set_end_value(range.end_value);
}
//step 3: search vectors
std::vector<::milvus::grpc::TopKQueryResult> result_array;
client_ptr->SearchVector(result_array, search_vector_infos);
//step 4: convert result array
for (auto &grpc_topk_result : result_array) {
TopKQueryResult result;
for (size_t i = 0; i < grpc_topk_result.query_result_arrays_size(); i++) {
QueryResult query_result;
query_result.id = grpc_topk_result.query_result_arrays(i).id();
query_result.distance = grpc_topk_result.query_result_arrays(i).distance();
result.query_result_arrays.emplace_back(query_result);
}
topk_query_result_array.emplace_back(result);
}
} catch (std::exception &ex) {
return Status(StatusCode::UnknownError, "failed to search vectors: " + std::string(ex.what()));
}
return Status::OK();
}
Status
ClientProxy::DescribeTable(const std::string &table_name, TableSchema &table_schema) {
try {
::milvus::grpc::TableSchema grpc_schema;
client_ptr->DescribeTable(grpc_schema, table_name);
table_schema.table_name.table_name = grpc_schema.table_name().table_name();
table_schema.index_type = (IndexType) grpc_schema.index_type();
table_schema.dimension = grpc_schema.dimension();
table_schema.store_raw_vector = grpc_schema.store_raw_vector();
} catch (std::exception &ex) {
return Status(StatusCode::UnknownError, "failed to describe table: " + std::string(ex.what()));
}
return Status::OK();
}
Status
ClientProxy::GetTableRowCount(const std::string &table_name, int64_t &row_count) {
try {
row_count = client_ptr->GetTableRowCount(table_name);
} catch (std::exception &ex) {
return Status(StatusCode::UnknownError, "failed to show tables: " + std::string(ex.what()));
}
return Status::OK();
}
Status
ClientProxy::ShowTables(std::vector<std::string> &table_array) {
try {
client_ptr->ShowTables(table_array);
} catch (std::exception &ex) {
return Status(StatusCode::UnknownError, "failed to show tables: " + std::string(ex.what()));
}
return Status::OK();
}
std::string
ClientProxy::ServerVersion() const {
try {
std::string version;
client_ptr->Ping(version, "version");
return version;
} catch (std::exception &ex) {
return "";
}
}
std::string
ClientProxy::ServerStatus() const {
if (connected_ == false) {
return "not connected to server";
}
try {
std::string dummy;
client_ptr->Ping(dummy, "");
return "server alive";
} catch (std::exception &ex) {
return "connection lost";
}
}
}
}

View File

@ -0,0 +1,66 @@
/*******************************************************************************
* Copyright (Zilliz) - All Rights Reserved
* Unauthorized copying of this file, via any medium is strictly prohibited.
* Proprietary and confidential.
******************************************************************************/
#pragma once
#include "MilvusApi.h"
#include "grpcClient.h"
namespace zilliz {
namespace milvus {
class ClientProxy : public Connection {
public:
// Implementations of the Connection interface
virtual Status Connect(const ConnectParam &param) override;
virtual Status Connect(const std::string &uri) override;
virtual Status Connected() const override;
virtual Status Disconnect() override;
virtual Status CreateTable(const TableSchema &param) override;
virtual bool HasTable(const std::string &table_name) override;
virtual Status DropTable(const std::string &table_name) override;
virtual Status BuildIndex(const std::string &table_name) override;
virtual Status InsertVector(const std::string &table_name,
const std::vector<RowRecord> &record_array,
std::vector<int64_t> &id_array) override;
virtual Status SearchVector(const std::string &table_name,
const std::vector<RowRecord> &query_record_array,
const std::vector<Range> &query_range_array,
int64_t topk,
std::vector<TopKQueryResult> &topk_query_result_array) override;
virtual Status DescribeTable(const std::string &table_name, TableSchema &table_schema) override;
virtual Status GetTableRowCount(const std::string &table_name, int64_t &row_count) override;
virtual Status ShowTables(std::vector<std::string> &table_array) override;
virtual std::string ClientVersion() const override;
virtual std::string ServerVersion() const override;
virtual std::string ServerStatus() const override;
private:
std::shared_ptr<::grpc::Channel> channel_;
private:
grpcClient *client_ptr;
// std::shared_ptr<grpcClient> client_ptr;
bool connected_ = false;
};
}
}

View File

@ -0,0 +1,168 @@
/*******************************************************************************
* Copyright (Zilliz) - All Rights Reserved
* Unauthorized copying of this file, via any medium is strictly prohibited.
* Proprietary and confidential.
******************************************************************************/
#include <grpc/grpc.h>
#include <grpcpp/channel.h>
#include <grpcpp/client_context.h>
#include <grpcpp/create_channel.h>
#include <grpcpp/security/credentials.h>
#include "grpcClient.h"
using grpc::Channel;
using grpc::ClientContext;
using grpc::ClientReader;
using grpc::ClientReaderWriter;
using grpc::ClientWriter;
using grpc::Status;
namespace zilliz {
namespace milvus {
grpcClient::grpcClient(std::shared_ptr<::grpc::Channel> channel)
: stub_(::milvus::grpc::MilvusService::NewStub(channel)) {
}
grpcClient::~grpcClient() {
}
void
grpcClient::CreateTable(const ::milvus::grpc::TableSchema table_schema) {
ClientContext context;
::milvus::Status response;
::grpc::Status status = stub_->CreateTable(&context, table_schema, &response);
if (!status.ok()) {
std::cout << "CreateTable rpc failed!\n";
}
}
bool
grpcClient::HasTable(const ::milvus::grpc::TableName table_name) {
ClientContext context;
::milvus::grpc::BoolReply response;
::grpc::Status status = stub_->HasTable(&context, table_name, &response);
if (!status.ok()) {
std::cout << "HasTable rpc failed!\n";
}
return response.bool_reply();
}
void
grpcClient::DropTable(const ::milvus::grpc::TableName table_name) {
ClientContext context;
::milvus::Status response;
::grpc::Status status = stub_->DropTable(&context, table_name, &response);
if (!status.ok()) {
std::cout << "DropTable rpc failed!\n";
}
}
void
grpcClient::BuildIndex(const ::milvus::grpc::TableName table_name) {
ClientContext context;
::milvus::Status response;
::grpc::Status status = stub_->BuildIndex(&context, table_name, &response);
if (!status.ok()) {
std::cout << "BuildIndex rpc failed!\n";
}
}
void
grpcClient::InsertVector(::milvus::grpc::VectorIds& vector_ids, const ::milvus::grpc::InsertInfos insert_infos) {
ClientContext context;
::grpc::Status status = stub_->InsertVector(&context, insert_infos, &vector_ids);
// std::cout << vector_ids.vector_id_array_size();
if (!status.ok()) {
std::cout << "InsertVector rpc failed!\n";
}
}
void
grpcClient::SearchVector(std::vector<::milvus::grpc::TopKQueryResult>& result_array,
const ::milvus::grpc::SearchVectorInfos search_vector_infos) {
::milvus::grpc::TopKQueryResult query_result;
ClientContext context;
std::unique_ptr<ClientReader<::milvus::grpc::TopKQueryResult> > reader(
stub_->SearchVector(&context, search_vector_infos));
while (reader->Read(&query_result)) {
result_array.emplace_back(query_result);
}
::grpc::Status status = reader->Finish();
if (!status.ok()) {
std::cout << "SearchVector rpc failed!\n";
}
}
void
grpcClient::DescribeTable(::milvus::grpc::TableSchema &grpc_schema, const std::string table_name) {
ClientContext context;
::milvus::grpc::TableName grpc_tablename;
grpc_tablename.set_table_name(table_name);
::grpc::Status status = stub_->DescribeTable(&context, grpc_tablename, &grpc_schema);
if (!status.ok()) {
std::cout << "DescribeTable rpc failed!\n";
}
}
int64_t
grpcClient::GetTableRowCount(const std::string table_name) {
ClientContext context;
::milvus::grpc::TableRowCount response;
::milvus::grpc::TableName grpc_tablename;
::grpc::Status status = stub_->GetTableRowCount(&context, grpc_tablename, &response);
if (!status.ok()) {
std::cout << "DescribeTable rpc failed!\n";
}
return response.table_row_count();
}
void
grpcClient::ShowTables(std::vector<std::string> &table_array) {
ClientContext context;
::milvus::grpc::Command command;
std::unique_ptr<ClientReader<::milvus::grpc::TableName> > reader(
stub_->ShowTables(&context, command));
::milvus::grpc::TableName table_name;
while (reader->Read(&table_name)) {
table_array.emplace_back(table_name.table_name());
}
::grpc::Status status = reader->Finish();
if (!status.ok()) {
std::cout << "ShowTables rpc failed!\n";
}
}
void
grpcClient::Ping(std::string &result, const std::string cmd) {
ClientContext context;
::milvus::grpc::ServerStatus response;
::milvus::grpc::Command command;
command.set_cmd(cmd);
::grpc::Status status = stub_->Ping(&context, command, &response);
result = response.info();
if (!status.ok()) {
std::cout << "Ping rpc failed!\n";
}
}
void
grpcClient::Disconnect() {
}
}
}

View File

@ -0,0 +1,62 @@
/*******************************************************************************
* Copyright (Zilliz) - All Rights Reserved
* Unauthorized copying of this file, via any medium is strictly prohibited.
* Proprietary and confidential.
******************************************************************************/
#pragma once
#include <chrono>
#include <iostream>
#include <memory>
#include <random>
#include <string>
#include <thread>
#include <grpc/grpc.h>
#include <grpcpp/channel.h>
#include <grpcpp/client_context.h>
#include <grpcpp/create_channel.h>
#include <grpcpp/security/credentials.h>
#include "Status.h"
#include "milvus.grpc.pb.h"
#include "status.grpc.pb.h"
#include <memory>
namespace zilliz {
namespace milvus {
class grpcClient {
public:
grpcClient(std::shared_ptr<::grpc::Channel> channel);
virtual ~grpcClient();
void CreateTable(const ::milvus::grpc::TableSchema table_schema);
bool HasTable(const ::milvus::grpc::TableName table_name);
void DropTable(const ::milvus::grpc::TableName table_name);
void BuildIndex(const ::milvus::grpc::TableName table_name);
void InsertVector(::milvus::grpc::VectorIds& vector_ids, const ::milvus::grpc::InsertInfos insert_infos);
void SearchVector(std::vector<::milvus::grpc::TopKQueryResult>& result_array,
const ::milvus::grpc::SearchVectorInfos search_vector_infos);
void DescribeTable(::milvus::grpc::TableSchema& grpc_schema, const std::string table_name);
int64_t GetTableRowCount(const std::string table_name);
void ShowTables(std::vector<std::string> &table_array);
void Ping(std::string &result, const std::string cmd);
void Disconnect();
private:
std::unique_ptr<::milvus::grpc::MilvusService::Stub> stub_;
};
}
}

View File

@ -0,0 +1,118 @@
/*******************************************************************************
* Copyright (Zilliz) - All Rights Reserved
* Unauthorized copying of this file, via any medium is strictly prohibited.
* Proprietary and confidential.
******************************************************************************/
#include "ConnectionImpl.h"
#include "version.h"
namespace zilliz {
namespace milvus {
std::shared_ptr<Connection>
Connection::Create() {
return std::shared_ptr<Connection>(new ConnectionImpl());
}
Status
Connection::Destroy(std::shared_ptr<milvus::Connection> connection_ptr) {
// if (connection_ptr != nullptr) {
// return connection_ptr->Disconnect();
// }
return Status::OK();
}
//////////////////////////////////////////////////////////////////////////////////////////////
ConnectionImpl::ConnectionImpl() {
client_proxy_ = std::make_shared<ClientProxy>();
}
Status
ConnectionImpl::Connect(const ConnectParam &param) {
return client_proxy_->Connect(param);
}
Status
ConnectionImpl::Connect(const std::string &uri) {
return client_proxy_->Connect(uri);
}
Status
ConnectionImpl::Connected() const {
return client_proxy_->Connected();
}
Status
ConnectionImpl::Disconnect() {
return client_proxy_->Disconnect();
}
std::string
ConnectionImpl::ClientVersion() const {
return MILVUS_VERSION;
}
Status
ConnectionImpl::CreateTable(const TableSchema &param) {
return client_proxy_->CreateTable(param);
}
bool
ConnectionImpl::HasTable(const std::string &table_name) {
return client_proxy_->HasTable(table_name);
}
Status
ConnectionImpl::DropTable(const std::string &table_name) {
return client_proxy_->DropTable(table_name);
}
Status
ConnectionImpl::BuildIndex(const std::string &table_name) {
return client_proxy_->BuildIndex(table_name);
}
Status
ConnectionImpl::InsertVector(const std::string &table_name,
const std::vector<RowRecord> &record_array,
std::vector<int64_t> &id_array) {
return client_proxy_->InsertVector(table_name, record_array, id_array);
}
Status
ConnectionImpl::SearchVector(const std::string &table_name,
const std::vector<RowRecord> &query_record_array,
const std::vector<Range> &query_range_array,
int64_t topk,
std::vector<TopKQueryResult> &topk_query_result_array) {
return client_proxy_->SearchVector(table_name, query_record_array, query_range_array, topk,
topk_query_result_array);
}
Status
ConnectionImpl::DescribeTable(const std::string &table_name, TableSchema &table_schema) {
return client_proxy_->DescribeTable(table_name, table_schema);
}
Status
ConnectionImpl::GetTableRowCount(const std::string &table_name, int64_t &row_count) {
return client_proxy_->GetTableRowCount(table_name, row_count);
}
Status
ConnectionImpl::ShowTables(std::vector<std::string> &table_array) {
return client_proxy_->ShowTables(table_array);
}
std::string
ConnectionImpl::ServerVersion() const {
return client_proxy_->ServerVersion();
}
std::string
ConnectionImpl::ServerStatus() const {
return client_proxy_->ServerStatus();
}
}
}

View File

@ -0,0 +1,62 @@
/*******************************************************************************
* Copyright (Zilliz) - All Rights Reserved
* Unauthorized copying of this file, via any medium is strictly prohibited.
* Proprietary and confidential.
******************************************************************************/
#pragma once
//#include "MilvusApi.h"
#include "client/ClientProxy.h"
namespace zilliz {
namespace milvus {
class ConnectionImpl : public Connection {
public:
ConnectionImpl();
// Implementations of the Connection interface
virtual Status Connect(const ConnectParam &param) override;
virtual Status Connect(const std::string &uri) override;
virtual Status Connected() const override;
virtual Status Disconnect() override;
virtual Status CreateTable(const TableSchema &param) override;
virtual bool HasTable(const std::string &table_name) override;
virtual Status DropTable(const std::string &table_name) override;
virtual Status BuildIndex(const std::string &table_name) override;
virtual Status InsertVector(const std::string &table_name,
const std::vector<RowRecord> &record_array,
std::vector<int64_t> &id_array) override;
virtual Status SearchVector(const std::string &table_name,
const std::vector<RowRecord> &query_record_array,
const std::vector<Range> &query_range_array,
int64_t topk,
std::vector<TopKQueryResult> &topk_query_result_array) override;
virtual Status DescribeTable(const std::string &table_name, TableSchema &table_schema) override;
virtual Status GetTableRowCount(const std::string &table_name, int64_t &row_count) override;
virtual Status ShowTables(std::vector<std::string> &table_array) override;
virtual std::string ClientVersion() const override;
virtual std::string ServerVersion() const override;
virtual std::string ServerStatus() const override;
private:
std::shared_ptr<ClientProxy> client_proxy_;
};
}
}

View File

@ -0,0 +1,129 @@
/*******************************************************************************
* Copyright (Zilliz) - All Rights Reserved
* Unauthorized copying of this file, via any medium is strictly prohibited.
* Proprietary and confidential.
******************************************************************************/
#include "Status.h"
namespace zilliz {
namespace milvus {
Status::~Status() noexcept {
if (state_ != nullptr) {
delete state_;
state_ = nullptr;
}
}
static inline std::ostream &operator<<(std::ostream &os, const Status &x) {
os << x.ToString();
return os;
}
void Status::MoveFrom(Status &s) {
delete state_;
state_ = s.state_;
s.state_ = nullptr;
}
Status::Status(const Status &s)
: state_((s.state_ == nullptr) ? nullptr : new State(*s.state_)) {}
Status &Status::operator=(const Status &s) {
if (state_ != s.state_) {
CopyFrom(s);
}
return *this;
}
Status &Status::operator=(Status &&s) noexcept {
MoveFrom(s);
return *this;
}
Status Status::operator&(const Status &status) const noexcept {
if (ok()) {
return status;
} else {
return *this;
}
}
Status Status::operator&(Status &&s) const noexcept {
if (ok()) {
return std::move(s);
} else {
return *this;
}
}
Status &Status::operator&=(const Status &s) noexcept {
if (ok() && !s.ok()) {
CopyFrom(s);
}
return *this;
}
Status &Status::operator&=(Status &&s) noexcept {
if (ok() && !s.ok()) {
MoveFrom(s);
}
return *this;
}
Status::Status(StatusCode code, const std::string &message) {
state_ = new State;
state_->code = code;
state_->message = message;
}
void Status::CopyFrom(const Status &status) {
delete state_;
if (status.state_ == nullptr) {
state_ = nullptr;
} else {
state_ = new State(*status.state_);
}
}
std::string Status::CodeAsString() const {
if (state_ == nullptr) {
return "OK";
}
const char *type = nullptr;
switch (code()) {
case StatusCode::OK:
type = "OK";
break;
case StatusCode::InvalidAgument:
type = "Invalid agument";
break;
case StatusCode::UnknownError:
type = "Unknown error";
break;
case StatusCode::NotSupported:
type = "Not Supported";
break;
case StatusCode::NotConnected:
type = "Not Connected";
break;
default:
type = "Unknown";
break;
}
return std::string(type);
}
std::string Status::ToString() const {
std::string result(CodeAsString());
if (state_ == nullptr) {
return result;
}
result += ": ";
result += state_->message;
return result;
}
}
}

View File

@ -0,0 +1,34 @@
/*******************************************************************************
* Copyright (Zilliz) - All Rights Reserved
* Unauthorized copying of this file, via any medium is strictly prohibited.
* Proprietary and confidential.
******************************************************************************/
#pragma once
#include "Status.h"
#include <exception>
namespace zilliz {
namespace milvus {
class Exception : public std::exception {
public:
Exception(StatusCode error_code,
const std::string &message = std::string())
: error_code_(error_code), message_(message) {}
public:
StatusCode error_code() const {
return error_code_;
}
virtual const char *what() const noexcept {
return message_.c_str();
}
private:
StatusCode error_code_;
std::string message_;
};
}
}

View File

@ -0,0 +1,102 @@
/*******************************************************************************
* Copyright (Zilliz) - All Rights Reserved
* Unauthorized copying of this file, via any medium is strictly prohibited.
* Proprietary and confidential.
******************************************************************************/
#include "DBWrapper.h"
#include "ServerConfig.h"
#include "utils/CommonUtil.h"
#include "utils/Log.h"
#include "utils/StringHelpFunctions.h"
namespace zilliz {
namespace milvus {
namespace server {
DBWrapper::DBWrapper() {
zilliz::milvus::engine::Options opt;
ConfigNode& config = ServerConfig::GetInstance().GetConfig(CONFIG_DB);
opt.meta.backend_uri = config.GetValue(CONFIG_DB_URL);
std::string db_path = config.GetValue(CONFIG_DB_PATH);
opt.meta.path = db_path + "/db";
std::string db_slave_path = config.GetValue(CONFIG_DB_SLAVE_PATH);
StringHelpFunctions::SplitStringByDelimeter(db_slave_path, ";", opt.meta.slave_paths);
int64_t index_size = config.GetInt64Value(CONFIG_DB_INDEX_TRIGGER_SIZE);
if(index_size > 0) {//ensure larger than zero, unit is MB
opt.index_trigger_size = (size_t)index_size * engine::ONE_MB;
}
int64_t insert_buffer_size = config.GetInt64Value(CONFIG_DB_INSERT_BUFFER_SIZE, 4);
if (insert_buffer_size >= 1) {
opt.insert_buffer_size = insert_buffer_size * engine::ONE_GB;
}
else {
std::cout << "ERROR: insert_buffer_size should be at least 1 GB" << std::endl;
kill(0, SIGUSR1);
}
ConfigNode& serverConfig = ServerConfig::GetInstance().GetConfig(CONFIG_SERVER);
std::string mode = serverConfig.GetValue(CONFIG_CLUSTER_MODE, "single");
if (mode == "single") {
opt.mode = zilliz::milvus::engine::Options::MODE::SINGLE;
}
else if (mode == "cluster") {
opt.mode = zilliz::milvus::engine::Options::MODE::CLUSTER;
}
else if (mode == "read_only") {
opt.mode = zilliz::milvus::engine::Options::MODE::READ_ONLY;
}
else {
std::cout << "ERROR: mode specified in server_config is not one of ['single', 'cluster', 'read_only']" << std::endl;
kill(0, SIGUSR1);
}
//set archive config
engine::ArchiveConf::CriteriaT criterial;
int64_t disk = config.GetInt64Value(CONFIG_DB_ARCHIVE_DISK, 0);
int64_t days = config.GetInt64Value(CONFIG_DB_ARCHIVE_DAYS, 0);
if(disk > 0) {
criterial[engine::ARCHIVE_CONF_DISK] = disk;
}
if(days > 0) {
criterial[engine::ARCHIVE_CONF_DAYS] = days;
}
opt.meta.archive_conf.SetCriterias(criterial);
//create db root folder
ServerError err = CommonUtil::CreateDirectory(opt.meta.path);
if(err != SERVER_SUCCESS) {
std::cout << "ERROR! Failed to create database root path: " << opt.meta.path << std::endl;
kill(0, SIGUSR1);
}
for(auto& path : opt.meta.slave_paths) {
err = CommonUtil::CreateDirectory(path);
if(err != SERVER_SUCCESS) {
std::cout << "ERROR! Failed to create database slave path: " << path << std::endl;
kill(0, SIGUSR1);
}
}
std::string msg = opt.meta.path;
try {
zilliz::milvus::engine::DB::Open(opt, &db_);
} catch(std::exception& ex) {
msg = ex.what();
}
if(db_ == nullptr) {
std::cout << "ERROR! Failed to open database: " << msg << std::endl;
kill(0, SIGUSR1);
}
}
DBWrapper::~DBWrapper() {
delete db_;
}
}
}
}

View File

@ -0,0 +1,34 @@
/*******************************************************************************
* Copyright (Zilliz) - All Rights Reserved
* Unauthorized copying of this file, via any medium is strictly prohibited.
* Proprietary and confidential.
******************************************************************************/
#pragma once
#include "db/DB.h"
#include "db/Meta.h"
namespace zilliz {
namespace milvus {
namespace server {
class DBWrapper {
private:
DBWrapper();
~DBWrapper();
public:
static zilliz::milvus::engine::DB* DB() {
static DBWrapper db_wrapper;
return db_wrapper.db();
}
zilliz::milvus::engine::DB* db() { return db_; }
private:
zilliz::milvus::engine::DB* db_ = nullptr;
};
}
}
}

View File

@ -0,0 +1,101 @@
/*******************************************************************************
* Copyright (Zilliz) - All Rights Reserved
* Unauthorized copying of this file, via any medium is strictly prohibited.
* Proprietary and confidential.
******************************************************************************/
#include "milvus.grpc.pb.h"
#include "MilvusServer.h"
#include "ServerConfig.h"
#include "DBWrapper.h"
#include "utils/Log.h"
#include "faiss/utils.h"
#include "RequestHandler.h"
#include <chrono>
#include <iostream>
#include <memory>
#include <random>
#include <string>
#include <thread>
#include <grpc/grpc.h>
#include <grpcpp/channel.h>
#include <grpcpp/client_context.h>
#include <grpcpp/create_channel.h>
#include <grpcpp/security/credentials.h>
#include <grpcpp/grpcpp.h>
namespace zilliz {
namespace milvus {
namespace server {
using grpc::Server;
using grpc::ServerBuilder;
using grpc::ServerContext;
using grpc::ServerReader;
using grpc::ServerReaderWriter;
using grpc::ServerWriter;
using grpc::Status;
static std::unique_ptr<Server> server;
void
MilvusServer::StartService() {
if (server != nullptr){
std::cout << "stopservice!\n";
StopService();
}
ServerConfig &config = ServerConfig::GetInstance();
ConfigNode server_config = config.GetConfig(CONFIG_SERVER);
ConfigNode engine_config = config.GetConfig(CONFIG_ENGINE);
std::string address = server_config.GetValue(CONFIG_SERVER_ADDRESS, "127.0.0.1");
int32_t port = server_config.GetInt32Value(CONFIG_SERVER_PORT, 19530);
faiss::distance_compute_blas_threshold = engine_config.GetInt32Value(CONFIG_DCBT, 20);
//TODO:add exception handle
DBWrapper::DB();//initialize db
std::string server_address("127.0.0.1:19530");
ServerBuilder builder;
builder.SetMaxReceiveMessageSize(400 * 1024 * 1024); //default 4 * 1024 * 1024
builder.SetMaxSendMessageSize(400 * 1024 * 1024);
builder.SetCompressionAlgorithmSupportStatus(GRPC_COMPRESS_STREAM_GZIP, true);
// builder.SetDefaultCompressionAlgorithm(GRPC_COMPRESS_DEFLATE);
// builder.SetDefaultCompressionAlgorithm(GRPC_COMPRESS_GZIP);
builder.SetDefaultCompressionAlgorithm(GRPC_COMPRESS_STREAM_GZIP);
// builder.SetDefaultCompressionAlgorithm(GRPC_COMPRESS_ALGORITHMS_COUNT);
builder.SetDefaultCompressionLevel(GRPC_COMPRESS_LEVEL_HIGH);
RequestHandler service;
builder.AddListeningPort(server_address, grpc::InsecureServerCredentials());
builder.RegisterService(&service);
builder.SetSyncServerOption(builder.MIN_POLLERS, 10);
builder.SetSyncServerOption(builder.MAX_POLLERS, 10);
server = builder.BuildAndStart();
server->Wait();
}
void
MilvusServer::StopService() {
auto stop_server_worker = [&]{
if (server != nullptr) {
server->Shutdown();
}
};
std::shared_ptr<std::thread> stop_thread = std::make_shared<std::thread>(stop_server_worker);
stop_thread->join();
}
}
}
}

View File

@ -0,0 +1,33 @@
/*******************************************************************************
* Copyright (Zilliz) - All Rights Reserved
* Unauthorized copying of this file, via any medium is strictly prohibited.
* Proprietary and confidential.
******************************************************************************/
#pragma once
#include <cstdint>
#include <string>
#include <grpc/grpc.h>
#include <grpcpp/server.h>
#include <grpcpp/server_builder.h>
#include <grpcpp/server_context.h>
#include <grpcpp/security/server_credentials.h>
using grpc::Channel;
using grpc::ClientContext;
using grpc::ClientReader;
using grpc::ClientReaderWriter;
using grpc::ClientWriter;
namespace zilliz {
namespace milvus {
namespace server {
class MilvusServer {
public:
static void StartService();
static void StopService();
};
}
}
}

View File

@ -0,0 +1,120 @@
/*******************************************************************************
* Copyright (Zilliz) - All Rights Reserved
* Unauthorized copying of this file, via any medium is strictly prohibited.
* Proprietary and confidential.
******************************************************************************/
#include "RequestHandler.h"
#include "RequestTask.h"
#include "utils/TimeRecorder.h"
namespace zilliz {
namespace milvus {
namespace server {
::grpc::Status
RequestHandler::CreateTable(::grpc::ServerContext *context, const ::milvus::grpc::TableSchema *request, ::milvus::Status *response) {
BaseTaskPtr task_ptr = CreateTableTask::Create(*request);
RequestScheduler::ExecTask(task_ptr, response);
return ::grpc::Status::OK;
}
//TODO: handle Response
::grpc::Status
RequestHandler::HasTable(::grpc::ServerContext *context, const ::milvus::grpc::TableName *request, ::milvus::grpc::BoolReply *response) {
bool has_table = false;
BaseTaskPtr task_ptr = HasTableTask::Create(request->table_name(), has_table);
::milvus::Status grpc_status;
RequestScheduler::ExecTask(task_ptr, &grpc_status);
response->set_bool_reply(has_table);
response->mutable_status()->set_reason(grpc_status.reason());
response->mutable_status()->set_error_code(grpc_status.error_code());
return ::grpc::Status::OK;
}
::grpc::Status
RequestHandler::DropTable(::grpc::ServerContext* context, const ::milvus::grpc::TableName* request, ::milvus::Status* response) {
BaseTaskPtr task_ptr = DropTableTask::Create(request->table_name());
RequestScheduler::ExecTask(task_ptr, response);
return ::grpc::Status::OK;
}
::grpc::Status
RequestHandler::BuildIndex(::grpc::ServerContext* context, const ::milvus::grpc::TableName* request, ::milvus::Status* response) {
BaseTaskPtr task_ptr = BuildIndexTask::Create(request->table_name());
RequestScheduler::ExecTask(task_ptr, response);
return ::grpc::Status::OK;
}
::grpc::Status
RequestHandler::InsertVector(::grpc::ServerContext* context, const ::milvus::grpc::InsertInfos* request, ::milvus::grpc::VectorIds* response) {
BaseTaskPtr task_ptr = InsertVectorTask::Create(*request, *response);
::milvus::Status grpc_status;
RequestScheduler::ExecTask(task_ptr, &grpc_status);
response->mutable_status()->set_reason(grpc_status.reason());
response->mutable_status()->set_error_code(grpc_status.error_code());
return ::grpc::Status::OK;
}
::grpc::Status
RequestHandler::SearchVector(::grpc::ServerContext* context, const ::milvus::grpc::SearchVectorInfos* request, ::grpc::ServerWriter<::milvus::grpc::TopKQueryResult>* writer) {
std::vector<std::string> file_id_array;
//TODO: handle status
BaseTaskPtr task_ptr = SearchVectorTask::Create(*request, file_id_array, *writer);
RequestScheduler::ExecTask(task_ptr, nullptr);
return ::grpc::Status::OK;
}
::grpc::Status
RequestHandler::SearchVectorInFiles(::grpc::ServerContext* context, const ::milvus::grpc::SearchVectorInFilesInfos* request, ::grpc::ServerWriter<::milvus::grpc::TopKQueryResult>* writer) {
std::vector<std::string> file_id_array;
BaseTaskPtr task_ptr = SearchVectorTask::Create(request->search_vector_infos(), file_id_array, *writer);
RequestScheduler::ExecTask(task_ptr, nullptr);
return ::grpc::Status::OK;
}
::grpc::Status
RequestHandler::DescribeTable(::grpc::ServerContext* context, const ::milvus::grpc::TableName* request, ::milvus::grpc::TableSchema* response) {
BaseTaskPtr task_ptr = DescribeTableTask::Create(request->table_name(), response);
::milvus::Status grpc_status;
RequestScheduler::ExecTask(task_ptr, &grpc_status);
response->mutable_table_name()->mutable_status()->set_error_code(grpc_status.error_code());
response->mutable_table_name()->mutable_status()->set_reason(grpc_status.reason());
return ::grpc::Status::OK;
}
::grpc::Status
RequestHandler::GetTableRowCount(::grpc::ServerContext* context, const ::milvus::grpc::TableName* request, ::milvus::grpc::TableRowCount* response) {
int64_t row_count = 0;
BaseTaskPtr task_ptr = GetTableRowCountTask::Create(request->table_name(), row_count);
::milvus::Status grpc_status;
RequestScheduler::ExecTask(task_ptr, &grpc_status);
response->set_table_row_count(row_count);
response->mutable_status()->set_reason(grpc_status.reason());
response->mutable_status()->set_error_code(grpc_status.error_code());
return ::grpc::Status::OK;
}
::grpc::Status
RequestHandler::ShowTables(::grpc::ServerContext* context, const ::milvus::grpc::Command* request, ::grpc::ServerWriter< ::milvus::grpc::TableName>* writer) {
BaseTaskPtr task_ptr = ShowTablesTask::Create(*writer);
RequestScheduler::ExecTask(task_ptr, nullptr);
return ::grpc::Status::OK;
}
::grpc::Status
RequestHandler::Ping(::grpc::ServerContext* context, const ::milvus::grpc::Command* request, ::milvus::grpc::ServerStatus* response) {
std::string result;
BaseTaskPtr task_ptr = PingTask::Create(request->cmd(), result);
::milvus::Status grpc_status;
RequestScheduler::ExecTask(task_ptr, &grpc_status);
response->set_info(result);
response->mutable_status()->set_reason(grpc_status.reason());
response->mutable_status()->set_error_code(grpc_status.error_code());
return ::grpc::Status::OK;
}
}
}
}

View File

@ -0,0 +1,175 @@
/*******************************************************************************
* Copyright (Zilliz) - All Rights Reserved
* Unauthorized copying of this file, via any medium is strictly prohibited.
* Proprietary and confidential.
******************************************************************************/
#pragma once
#include <cstdint>
#include <string>
#include "milvus.grpc.pb.h"
#include "status.pb.h"
namespace zilliz {
namespace milvus {
namespace server {
class RequestHandler final : public ::milvus::grpc::MilvusService::Service {
public:
/**
* @brief Create table method
*
* This method is used to create table
*
* @param param, use to provide table information to be created.
*
*
* @param param
*/
::grpc::Status CreateTable(::grpc::ServerContext* context, const ::milvus::grpc::TableSchema* request, ::milvus::Status* response);
/**
* @brief Test table existence method
*
* This method is used to test table existence.
*
* @param table_name, table name is going to be tested.
*
*
* @param table_name
*/
::grpc::Status HasTable(::grpc::ServerContext* context, const ::milvus::grpc::TableName* request, ::milvus::grpc::BoolReply* response);
/**
* @brief Drop table method
*
* This method is used to drop table.
*
* @param table_name, table name is going to be deleted.
*
*
* @param table_name
*/
::grpc::Status DropTable(::grpc::ServerContext* context, const ::milvus::grpc::TableName* request, ::milvus::Status* response);
/**
* @brief build index by table method
*
* This method is used to build index by table in sync.
*
* @param table_name, table name is going to be built index.
*
*
* @param table_name
*/
::grpc::Status BuildIndex(::grpc::ServerContext* context, const ::milvus::grpc::TableName* request, ::milvus::Status* response);
/**
* @brief Insert vector array to table
*
* This method is used to insert vector array to table.
*
* @param table_name, table_name is inserted.
* @param record_array, vector array is inserted.
*
* @return vector id array
*
* @param table_name
* @param record_array
*/
::grpc::Status InsertVector(::grpc::ServerContext* context, const ::milvus::grpc::InsertInfos* request, ::milvus::grpc::VectorIds* response);
/**
* @brief Query vector
*
* This method is used to query vector in table.
*
* @param table_name, table_name is queried.
* @param query_record_array, all vector are going to be queried.
* @param query_range_array, optional ranges for conditional search. If not specified, search whole table
* @param topk, how many similarity vectors will be searched.
*
* @return query result array.
*
* @param table_name
* @param query_record_array
* @param query_range_array
* @param topk
*/
::grpc::Status SearchVector(::grpc::ServerContext* context, const ::milvus::grpc::SearchVectorInfos* request, ::grpc::ServerWriter<::milvus::grpc::TopKQueryResult>* writer);
/**
* @brief Internal use query interface
*
* This method is used to query vector in specified files.
*
* @param file_id_array, specified files id array, queried.
* @param query_record_array, all vector are going to be queried.
* @param query_range_array, optional ranges for conditional search. If not specified, search whole table
* @param topk, how many similarity vectors will be searched.
*
* @return query result array.
*
* @param file_id_array
* @param query_record_array
* @param query_range_array
* @param topk
*/
::grpc::Status SearchVectorInFiles(::grpc::ServerContext* context, const ::milvus::grpc::SearchVectorInFilesInfos* request, ::grpc::ServerWriter<::milvus::grpc::TopKQueryResult>* writer);
/**
* @brief Get table schema
*
* This method is used to get table schema.
*
* @param table_name, target table name.
*
* @return table schema
*
* @param table_name
*/
::grpc::Status DescribeTable(::grpc::ServerContext* context, const ::milvus::grpc::TableName* request, ::milvus::grpc::TableSchema* response);
/**
* @brief Get table row count
*
* This method is used to get table row count.
*
* @param table_name, target table name.
*
* @return table row count
*
* @param table_name
*/
::grpc::Status GetTableRowCount(::grpc::ServerContext* context, const ::milvus::grpc::TableName* request, ::milvus::grpc::TableRowCount* response);
/**
* @brief List all tables in database
*
* This method is used to list all tables.
*
*
* @return table names.
*/
::grpc::Status ShowTables(::grpc::ServerContext* context, const ::milvus::grpc::Command* request, ::grpc::ServerWriter< ::milvus::grpc::TableName>* writer);
/**
* @brief Give the server status
*
*
* This method is used to give the server status.
*
* @return Server status.
*
* @param cmd
*/
::grpc::Status Ping(::grpc::ServerContext* context, const ::milvus::grpc::Command* request, ::milvus::grpc::ServerStatus* response);
};
}
}
}

View File

@ -0,0 +1,221 @@
/*******************************************************************************
* Copyright (Zilliz) - All Rights Reserved
* Unauthorized copying of this file, via any medium is strictly prohibited.
* Proprietary and confidential.
******************************************************************************/
#include "RequestScheduler.h"
#include "utils/Log.h"
#include "../grpc/gen-status/status.pb.h"
namespace zilliz {
namespace milvus {
namespace server {
using namespace ::milvus;
namespace {
const std::map<ServerError, ::milvus::ErrorCode > &ErrorMap() {
static const std::map<ServerError, ::milvus::ErrorCode> code_map = {
{SERVER_UNEXPECTED_ERROR, ErrorCode::UNEXPECTED_ERROR},
{SERVER_UNSUPPORTED_ERROR, ErrorCode::UNEXPECTED_ERROR},
{SERVER_NULL_POINTER, ErrorCode::UNEXPECTED_ERROR},
{SERVER_INVALID_ARGUMENT, ErrorCode::ILLEGAL_ARGUMENT},
{SERVER_FILE_NOT_FOUND, ErrorCode::FILE_NOT_FOUND},
{SERVER_NOT_IMPLEMENT, ErrorCode::UNEXPECTED_ERROR},
{SERVER_BLOCKING_QUEUE_EMPTY, ErrorCode::UNEXPECTED_ERROR},
{SERVER_CANNOT_CREATE_FOLDER, ErrorCode::CANNOT_CREATE_FOLDER},
{SERVER_CANNOT_CREATE_FILE, ErrorCode::CANNOT_CREATE_FILE},
{SERVER_CANNOT_DELETE_FOLDER, ErrorCode::CANNOT_DELETE_FOLDER},
{SERVER_CANNOT_DELETE_FILE, ErrorCode::CANNOT_DELETE_FILE},
{SERVER_TABLE_NOT_EXIST, ErrorCode::TABLE_NOT_EXISTS},
{SERVER_INVALID_TABLE_NAME, ErrorCode::ILLEGAL_TABLE_NAME},
{SERVER_INVALID_TABLE_DIMENSION, ErrorCode::ILLEGAL_DIMENSION},
{SERVER_INVALID_TIME_RANGE, ErrorCode::ILLEGAL_RANGE},
{SERVER_INVALID_VECTOR_DIMENSION, ErrorCode::ILLEGAL_DIMENSION},
{SERVER_INVALID_INDEX_TYPE, ErrorCode::ILLEGAL_INDEX_TYPE},
{SERVER_INVALID_ROWRECORD, ErrorCode::ILLEGAL_ROWRECORD},
{SERVER_INVALID_ROWRECORD_ARRAY, ErrorCode::ILLEGAL_ROWRECORD},
{SERVER_INVALID_TOPK, ErrorCode::ILLEGAL_TOPK},
{SERVER_ILLEGAL_VECTOR_ID, ErrorCode::ILLEGAL_VECTOR_ID},
{SERVER_ILLEGAL_SEARCH_RESULT, ErrorCode::ILLEGAL_SEARCH_RESULT},
{SERVER_CACHE_ERROR, ErrorCode::CACHE_FAILED},
{DB_META_TRANSACTION_FAILED, ErrorCode::META_FAILED},
{SERVER_BUILD_INDEX_ERROR, ErrorCode::BUILD_INDEX_ERROR},
};
return code_map;
}
}
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
BaseTask::BaseTask(const std::string& task_group, bool async)
: task_group_(task_group),
async_(async),
done_(false),
error_code_(SERVER_SUCCESS) {
}
BaseTask::~BaseTask() {
WaitToFinish();
}
ServerError BaseTask::Execute() {
error_code_ = OnExecute();
done_ = true;
finish_cond_.notify_all();
return error_code_;
}
ServerError BaseTask::SetError(ServerError error_code, const std::string& error_msg) {
error_code_ = error_code;
error_msg_ = error_msg;
SERVER_LOG_ERROR << error_msg_;
return error_code_;
}
ServerError BaseTask::WaitToFinish() {
std::unique_lock <std::mutex> lock(finish_mtx_);
finish_cond_.wait(lock, [this] { return done_; });
return error_code_;
}
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
RequestScheduler::RequestScheduler()
: stopped_(false) {
Start();
}
RequestScheduler::~RequestScheduler() {
Stop();
}
void RequestScheduler::ExecTask(BaseTaskPtr& task_ptr, ::milvus::Status *grpc_status) {
if(task_ptr == nullptr) {
return;
}
RequestScheduler& scheduler = RequestScheduler::GetInstance();
scheduler.ExecuteTask(task_ptr);
if(!task_ptr->IsAsync()) {
task_ptr->WaitToFinish();
ServerError err = task_ptr->ErrorCode();
if (err != SERVER_SUCCESS) {
//TODO:Handle exception
grpc_status->set_reason(task_ptr->ErrorMsg());
grpc_status->set_error_code(::milvus::ErrorCode((int) err));
// grpc_status.set_error_code(err);
// Exception ex;
// ex.__set_code(ErrorMap().at(err));
// std::string msg = task_ptr->ErrorMsg();
// if(msg.empty()){
// msg = "Error message not set";
// }
// ex.__set_reason(msg);
// throw ex;
}
}
}
void RequestScheduler::Start() {
if(!stopped_) {
return;
}
stopped_ = false;
}
void RequestScheduler::Stop() {
if(stopped_) {
return;
}
SERVER_LOG_INFO << "Scheduler gonna stop...";
{
std::lock_guard<std::mutex> lock(queue_mtx_);
for(auto iter : task_groups_) {
if(iter.second != nullptr) {
iter.second->Put(nullptr);
}
}
}
for(auto iter : execute_threads_) {
if(iter == nullptr)
continue;
iter->join();
}
stopped_ = true;
SERVER_LOG_INFO << "Scheduler stopped";
}
ServerError RequestScheduler::ExecuteTask(const BaseTaskPtr& task_ptr) {
if(task_ptr == nullptr) {
return SERVER_NULL_POINTER;
}
ServerError err = PutTaskToQueue(task_ptr);
if(err != SERVER_SUCCESS) {
return err;
}
if(task_ptr->IsAsync()) {
return SERVER_SUCCESS;//async execution, caller need to call WaitToFinish at somewhere
}
return task_ptr->WaitToFinish();//sync execution
}
namespace {
void TakeTaskToExecute(TaskQueuePtr task_queue) {
if(task_queue == nullptr) {
return;
}
while(true) {
BaseTaskPtr task = task_queue->Take();
if (task == nullptr) {
break;//stop the thread
}
try {
ServerError err = task->Execute();
if(err != SERVER_SUCCESS) {
SERVER_LOG_ERROR << "Task failed with code: " << err;
}
} catch (std::exception& ex) {
SERVER_LOG_ERROR << "Task failed to execute: " << ex.what();
}
}
}
}
ServerError RequestScheduler::PutTaskToQueue(const BaseTaskPtr& task_ptr) {
std::lock_guard<std::mutex> lock(queue_mtx_);
std::string group_name = task_ptr->TaskGroup();
if(task_groups_.count(group_name) > 0) {
task_groups_[group_name]->Put(task_ptr);
} else {
TaskQueuePtr queue = std::make_shared<TaskQueue>();
queue->Put(task_ptr);
task_groups_.insert(std::make_pair(group_name, queue));
//start a thread
ThreadPtr thread = std::make_shared<std::thread>(&TakeTaskToExecute, queue);
execute_threads_.push_back(thread);
SERVER_LOG_INFO << "Create new thread for task group: " << group_name;
}
return SERVER_SUCCESS;
}
}
}
}

View File

@ -0,0 +1,90 @@
/*******************************************************************************
* Copyright (Zilliz) - All Rights Reserved
* Unauthorized copying of this file, via any medium is strictly prohibited.
* Proprietary and confidential.
******************************************************************************/
#pragma once
#include "utils/BlockingQueue.h"
#include "status.grpc.pb.h"
#include "status.pb.h"
#include <map>
#include <vector>
#include <thread>
namespace zilliz {
namespace milvus {
namespace server {
class BaseTask {
protected:
BaseTask(const std::string& task_group, bool async = false);
virtual ~BaseTask();
public:
ServerError Execute();
ServerError WaitToFinish();
std::string TaskGroup() const { return task_group_; }
ServerError ErrorCode() const { return error_code_; }
std::string ErrorMsg() const { return error_msg_; }
bool IsAsync() const { return async_; }
protected:
virtual ServerError OnExecute() = 0;
ServerError SetError(ServerError error_code, const std::string& msg);
protected:
mutable std::mutex finish_mtx_;
std::condition_variable finish_cond_;
std::string task_group_;
bool async_;
bool done_;
ServerError error_code_;
std::string error_msg_;
};
using BaseTaskPtr = std::shared_ptr<BaseTask>;
using TaskQueue = BlockingQueue<BaseTaskPtr>;
using TaskQueuePtr = std::shared_ptr<TaskQueue>;
using ThreadPtr = std::shared_ptr<std::thread>;
class RequestScheduler {
public:
static RequestScheduler& GetInstance() {
static RequestScheduler scheduler;
return scheduler;
}
void Start();
void Stop();
ServerError ExecuteTask(const BaseTaskPtr& task_ptr);
static void ExecTask(BaseTaskPtr& task_ptr, ::milvus::Status* grpc_status);
protected:
RequestScheduler();
virtual ~RequestScheduler();
ServerError PutTaskToQueue(const BaseTaskPtr& task_ptr);
private:
mutable std::mutex queue_mtx_;
std::map<std::string, TaskQueuePtr> task_groups_;
std::vector<ThreadPtr> execute_threads_;
bool stopped_;
};
}
}
}

View File

@ -0,0 +1,649 @@
/*******************************************************************************
* Copyright (Zilliz) - All Rights Reserved
* Unauthorized copying of this file, via any medium is strictly prohibited.
* Proprietary and confidential.
******************************************************************************/
#include "RequestTask.h"
#include "ServerConfig.h"
#include "utils/CommonUtil.h"
#include "utils/Log.h"
#include "utils/TimeRecorder.h"
#include "utils/ValidationUtil.h"
#include "DBWrapper.h"
#include "version.h"
#include "MilvusServer.h"
#include "Server.h"
namespace zilliz {
namespace milvus {
namespace server {
static const std::string DQL_TASK_GROUP = "dql";
static const std::string DDL_DML_TASK_GROUP = "ddl_dml";
static const std::string PING_TASK_GROUP = "ping";
using DB_META = zilliz::milvus::engine::meta::Meta;
using DB_DATE = zilliz::milvus::engine::meta::DateT;
namespace {
engine::EngineType EngineType(int type) {
static std::map<int, engine::EngineType> map_type = {
{0, engine::EngineType::INVALID},
{1, engine::EngineType::FAISS_IDMAP},
{2, engine::EngineType::FAISS_IVFFLAT},
{3, engine::EngineType::FAISS_IVFSQ8},
};
if(map_type.find(type) == map_type.end()) {
return engine::EngineType::INVALID;
}
return map_type[type];
}
int IndexType(engine::EngineType type) {
static std::map<engine::EngineType, int> map_type = {
{engine::EngineType::INVALID, 0},
{engine::EngineType::FAISS_IDMAP, 1},
{engine::EngineType::FAISS_IVFFLAT, 2},
{engine::EngineType::FAISS_IVFSQ8, 3},
};
if(map_type.find(type) == map_type.end()) {
return 0;
}
return map_type[type];
}
void
ConvertRowRecordToFloatArray(const ::milvus::grpc::InsertInfos insert_infos,
uint64_t dimension,
std::vector<float>& float_array) {
uint64_t vec_count = insert_infos.row_record_array_size();
float_array.resize(vec_count * dimension);//allocate enough memory
for(uint64_t i = 0; i < vec_count; i++) {
for (uint64_t j = 0; j < dimension; j++) {
float_array[i * dimension + j] = insert_infos.row_record_array(i).vector_data(j);
}
}
}
static constexpr long DAY_SECONDS = 86400;
void
ConvertTimeRangeToDBDates(const std::vector<::milvus::grpc::Range> &range_array,
std::vector<DB_DATE>& dates,
ServerError& error_code,
std::string& error_msg) {
dates.clear();
for(auto& range : range_array) {
time_t tt_start, tt_end;
tm tm_start, tm_end;
if(!CommonUtil::TimeStrToTime(range.start_value(), tt_start, tm_start)){
error_code = SERVER_INVALID_TIME_RANGE;
error_msg = "Invalid time range: " + range.start_value();
return;
}
if(!CommonUtil::TimeStrToTime(range.end_value(), tt_end, tm_end)){
error_code = SERVER_INVALID_TIME_RANGE;
error_msg = "Invalid time range: " + range.start_value();
return;
}
long days = (tt_end > tt_start) ? (tt_end - tt_start)/DAY_SECONDS : (tt_start - tt_end)/DAY_SECONDS;
if(days == 0) {
error_code = SERVER_INVALID_TIME_RANGE;
error_msg = "Invalid time range: " + range.start_value() + " to " + range.end_value();
return ;
}
for(long i = 0; i < days; i++) {
time_t tt_day = tt_start + DAY_SECONDS*i;
tm tm_day;
CommonUtil::ConvertTime(tt_day, tm_day);
long date = tm_day.tm_year*10000 + tm_day.tm_mon*100 + tm_day.tm_mday;//according to db logic
dates.push_back(date);
}
}
}
}
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
CreateTableTask::CreateTableTask(const ::milvus::grpc::TableSchema schema)
: BaseTask(DDL_DML_TASK_GROUP),
schema_(schema) {
}
BaseTaskPtr CreateTableTask::Create(const ::milvus::grpc::TableSchema schema) {
return std::shared_ptr<BaseTask>(new CreateTableTask(schema));
}
ServerError CreateTableTask::OnExecute() {
TimeRecorder rc("CreateTableTask");
try {
//step 1: check arguments
ServerError res = SERVER_SUCCESS;
res = ValidateTableName(schema_.table_name().table_name());
if(res != SERVER_SUCCESS) {
return SetError(res, "Invalid table name: " + schema_.table_name().table_name());
}
res = ValidateTableDimension(schema_.dimension());
if(res != SERVER_SUCCESS) {
return SetError(res, "Invalid table dimension: " + std::to_string(schema_.dimension()));
}
res = ValidateTableIndexType(schema_.index_type());
if(res != SERVER_SUCCESS) {
return SetError(res, "Invalid index type: " + std::to_string(schema_.index_type()));
}
//step 2: construct table schema
engine::meta::TableSchema table_info;
table_info.dimension_ = (uint16_t)schema_.dimension();
table_info.table_id_ = schema_.table_name().table_name();
table_info.engine_type_ = (int)EngineType(schema_.index_type());
table_info.store_raw_data_ = schema_.store_raw_vector();
//step 3: create table
engine::Status stat = DBWrapper::DB()->CreateTable(table_info);
if(!stat.ok()) {//table could exist
return SetError(DB_META_TRANSACTION_FAILED, "Engine failed: " + stat.ToString());
}
} catch (std::exception& ex) {
return SetError(SERVER_UNEXPECTED_ERROR, ex.what());
}
rc.Record("done");
return SERVER_SUCCESS;
}
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
DescribeTableTask::DescribeTableTask(const std::string &table_name, ::milvus::grpc::TableSchema *schema)
: BaseTask(DDL_DML_TASK_GROUP),
table_name_(table_name),
schema_(schema) {
}
BaseTaskPtr DescribeTableTask::Create(const std::string& table_name, ::milvus::grpc::TableSchema *schema) {
return std::shared_ptr<BaseTask>(new DescribeTableTask(table_name, schema));
}
ServerError DescribeTableTask::OnExecute() {
TimeRecorder rc("DescribeTableTask");
try {
//step 1: check arguments
ServerError res = SERVER_SUCCESS;
res = ValidateTableName(table_name_);
if(res != SERVER_SUCCESS) {
return SetError(res, "Invalid table name: " + table_name_);
}
//step 2: get table info
engine::meta::TableSchema table_info;
table_info.table_id_ = table_name_;
engine::Status stat = DBWrapper::DB()->DescribeTable(table_info);
if(!stat.ok()) {
return SetError(DB_META_TRANSACTION_FAILED, "Engine failed: " + stat.ToString());
}
schema_->mutable_table_name()->set_table_name(table_info.table_id_);
schema_->set_index_type(IndexType((engine::EngineType)table_info.engine_type_));
schema_->set_dimension(table_info.dimension_);
schema_->set_store_raw_vector(table_info.store_raw_data_);
} catch (std::exception& ex) {
return SetError(SERVER_UNEXPECTED_ERROR, ex.what());
}
rc.Record("done");
return SERVER_SUCCESS;
}
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
BuildIndexTask::BuildIndexTask(const std::string& table_name)
: BaseTask(DDL_DML_TASK_GROUP),
table_name_(table_name) {
}
BaseTaskPtr BuildIndexTask::Create(const std::string& table_name) {
return std::shared_ptr<BaseTask>(new BuildIndexTask(table_name));
}
ServerError BuildIndexTask::OnExecute() {
try {
TimeRecorder rc("BuildIndexTask");
//step 1: check arguments
ServerError res = SERVER_SUCCESS;
res = ValidateTableName(table_name_);
if(res != SERVER_SUCCESS) {
return SetError(res, "Invalid table name: " + table_name_);
}
bool has_table = false;
engine::Status stat = DBWrapper::DB()->HasTable(table_name_, has_table);
if(!has_table) {
return SetError(SERVER_TABLE_NOT_EXIST, "Table " + table_name_ + " not exists");
}
//step 2: check table existence
stat = DBWrapper::DB()->BuildIndex(table_name_);
if(!stat.ok()) {
return SetError(SERVER_BUILD_INDEX_ERROR, "Engine failed: " + stat.ToString());
}
rc.Elapse("totally cost");
} catch (std::exception& ex) {
return SetError(SERVER_UNEXPECTED_ERROR, ex.what());
}
return SERVER_SUCCESS;
}
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
HasTableTask::HasTableTask(const std::string table_name, bool& has_table)
: BaseTask(DDL_DML_TASK_GROUP),
table_name_(table_name),
has_table_(has_table) {
}
BaseTaskPtr HasTableTask::Create(const std::string table_name, bool& has_table) {
return std::shared_ptr<BaseTask>(new HasTableTask(table_name, has_table));
}
ServerError HasTableTask::OnExecute() {
try {
TimeRecorder rc("HasTableTask");
//step 1: check arguments
ServerError res = SERVER_SUCCESS;
res = ValidateTableName(table_name_);
if(res != SERVER_SUCCESS) {
return SetError(res, "Invalid table name: " + table_name_);
}
//step 2: check table existence
engine::Status stat = DBWrapper::DB()->HasTable(table_name_, has_table_);
if(!stat.ok()) {
return SetError(DB_META_TRANSACTION_FAILED, "Engine failed: " + stat.ToString());
}
rc.Elapse("totally cost");
} catch (std::exception& ex) {
return SetError(SERVER_UNEXPECTED_ERROR, ex.what());
}
return SERVER_SUCCESS;
}
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
DropTableTask::DropTableTask(const std::string& table_name)
: BaseTask(DDL_DML_TASK_GROUP),
table_name_(table_name) {
}
BaseTaskPtr DropTableTask::Create(const std::string& table_name) {
return std::shared_ptr<BaseTask>(new DropTableTask(table_name));
}
ServerError DropTableTask::OnExecute() {
try {
TimeRecorder rc("DropTableTask");
//step 1: check arguments
ServerError res = SERVER_SUCCESS;
res = ValidateTableName(table_name_);
if(res != SERVER_SUCCESS) {
return SetError(res, "Invalid table name: " + table_name_);
}
//step 2: check table existence
engine::meta::TableSchema table_info;
table_info.table_id_ = table_name_;
engine::Status stat = DBWrapper::DB()->DescribeTable(table_info);
if(!stat.ok()) {
if(stat.IsNotFound()) {
return SetError(SERVER_TABLE_NOT_EXIST, "Table " + table_name_ + " not exists");
} else {
return SetError(DB_META_TRANSACTION_FAILED, "Engine failed: " + stat.ToString());
}
}
rc.Record("check validation");
//step 3: Drop table
std::vector<DB_DATE> dates;
stat = DBWrapper::DB()->DeleteTable(table_name_, dates);
if(!stat.ok()) {
return SetError(DB_META_TRANSACTION_FAILED, "Engine failed: " + stat.ToString());
}
rc.Record("drop table");
rc.Elapse("total cost");
} catch (std::exception& ex) {
return SetError(SERVER_UNEXPECTED_ERROR, ex.what());
}
return SERVER_SUCCESS;
}
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
ShowTablesTask::ShowTablesTask(::grpc::ServerWriter< ::milvus::grpc::TableName> writer)
: BaseTask(DDL_DML_TASK_GROUP),
writer_(writer) {
}
BaseTaskPtr ShowTablesTask::Create(::grpc::ServerWriter< ::milvus::grpc::TableName> writer) {
return std::shared_ptr<BaseTask>(new ShowTablesTask(writer));
}
ServerError ShowTablesTask::OnExecute() {
std::vector<engine::meta::TableSchema> schema_array;
engine::Status stat = DBWrapper::DB()->AllTables(schema_array);
if(!stat.ok()) {
return SetError(DB_META_TRANSACTION_FAILED, "Engine failed: " + stat.ToString());
}
for(auto& schema : schema_array) {
::milvus::grpc::TableName tableName;
tableName.set_table_name(schema.table_id_);
writer_.Write(tableName);
}
return SERVER_SUCCESS;
}
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
InsertVectorTask::InsertVectorTask(const ::milvus::grpc::InsertInfos insert_infos,
::milvus::grpc::VectorIds& record_ids)
: BaseTask(DDL_DML_TASK_GROUP),
insert_infos_(insert_infos),
record_ids_(record_ids) {
record_ids_.Clear();
}
BaseTaskPtr InsertVectorTask::Create(const ::milvus::grpc::InsertInfos insert_infos,
::milvus::grpc::VectorIds& record_ids) {
return std::shared_ptr<BaseTask>(new InsertVectorTask(insert_infos, record_ids));
}
ServerError InsertVectorTask::OnExecute() {
try {
TimeRecorder rc("InsertVectorTask");
//step 1: check arguments
ServerError res = SERVER_SUCCESS;
res = ValidateTableName(insert_infos_.table_name());
if(res != SERVER_SUCCESS) {
return SetError(res, "Invalid table name: " + insert_infos_.table_name());
}
if(insert_infos_.row_record_array().empty()) {
return SetError(SERVER_INVALID_ROWRECORD_ARRAY, "Row record array is empty");
}
//step 2: check table existence
engine::meta::TableSchema table_info;
table_info.table_id_ = insert_infos_.table_name();
engine::Status stat = DBWrapper::DB()->DescribeTable(table_info);
if(!stat.ok()) {
if(stat.IsNotFound()) {
return SetError(SERVER_TABLE_NOT_EXIST, "Table " + insert_infos_.table_name() + " not exists");
} else {
return SetError(DB_META_TRANSACTION_FAILED, "Engine failed: " + stat.ToString());
}
}
rc.Record("check validation");
//step 3: prepare float data
std::vector<float> vec_f;
vec_f.resize(insert_infos_.row_record_array_size() * table_info.dimension_);
for (size_t i = 0; i < insert_infos_.row_record_array_size(); i++) {
for (size_t j = 0; j < table_info.dimension_; j++) {
vec_f[i * table_info.dimension_ + j] = insert_infos_.row_record_array(i).vector_data(j);
}
}
rc.Record("prepare vectors data");
//step 4: insert vectors
uint64_t vec_count = (uint64_t)insert_infos_.row_record_array_size();
std::vector<int64_t> vec_ids;
vec_ids.resize(record_ids_.vector_id_array_size());
vec_ids.clear();
stat = DBWrapper::DB()->InsertVectors(insert_infos_.table_name(), vec_count, vec_f.data(), vec_ids);
rc.Record("add vectors to engine");
if(!stat.ok()) {
return SetError(SERVER_CACHE_ERROR, "Cache error: " + stat.ToString());
}
for (size_t i = 0; i < vec_ids.size(); i++) {
record_ids_.add_vector_id_array(vec_ids.at(i));
}
if(record_ids_.vector_id_array_size() != vec_count) {
std::string msg = "Add " + std::to_string(vec_count) + " vectors but only return "
+ std::to_string(record_ids_.vector_id_array_size()) + " id";
return SetError(SERVER_ILLEGAL_VECTOR_ID, msg);
}
rc.Record("do insert");
rc.Elapse("total cost");
} catch (std::exception& ex) {
return SetError(SERVER_UNEXPECTED_ERROR, ex.what());
}
return SERVER_SUCCESS;
}
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
SearchVectorTask::SearchVectorTask(const ::milvus::grpc::SearchVectorInfos searchVectorInfos,
const std::vector<std::string> file_id_array,
::grpc::ServerWriter<::milvus::grpc::TopKQueryResult> writer)
: BaseTask(DQL_TASK_GROUP),
searchVector_infos_(searchVectorInfos),
file_id_array_(file_id_array),
writer_(writer) {
}
BaseTaskPtr SearchVectorTask::Create(const ::milvus::grpc::SearchVectorInfos searchVectorInfos,
const std::vector<std::string> file_id_array,
::grpc::ServerWriter<::milvus::grpc::TopKQueryResult> writer) {
return std::shared_ptr<BaseTask>(new SearchVectorTask(searchVectorInfos, file_id_array,
writer));
}
ServerError SearchVectorTask::OnExecute() {
try {
TimeRecorder rc("SearchVectorTask");
//step 1: check arguments
ServerError res = SERVER_SUCCESS;
std::string table_name_ = searchVector_infos_.table_name();
res = ValidateTableName(table_name_);
if(res != SERVER_SUCCESS) {
return SetError(res, "Invalid table name: " + table_name_);
}
int top_k_ = searchVector_infos_.topk();
if(top_k_ <= 0) {
return SetError(SERVER_INVALID_TOPK, "Invalid topk: " + std::to_string(
top_k_));
}
if(searchVector_infos_.query_record_array().empty()) {
return SetError(SERVER_INVALID_ROWRECORD_ARRAY, "Row record array is empty");
}
//step 2: check table existence
engine::meta::TableSchema table_info;
table_info.table_id_ = table_name_;
engine::Status stat = DBWrapper::DB()->DescribeTable(table_info);
if(!stat.ok()) {
if(stat.IsNotFound()) {
return SetError(SERVER_TABLE_NOT_EXIST, "Table " + table_name_ + " not exists");
} else {
return SetError(DB_META_TRANSACTION_FAILED, "Engine failed: " + stat.ToString());
}
}
//step 3: check date range, and convert to db dates
std::vector<DB_DATE> dates;
ServerError error_code = SERVER_SUCCESS;
std::string error_msg;
std::vector<::milvus::grpc::Range> range_array;
for (size_t i = 0; i < searchVector_infos_.query_range_array_size(); i++) {
range_array.emplace_back(searchVector_infos_.query_range_array(i));
}
ConvertTimeRangeToDBDates(range_array, dates, error_code, error_msg);
if(error_code != SERVER_SUCCESS) {
return SetError(error_code, error_msg);
}
rc.Record("check validation");
//step 3: prepare float data
std::vector<float> vec_f;
vec_f.resize(searchVector_infos_.query_record_array_size() * table_info.dimension_);
// vec_f.resize(searchVector_infos_.query_range_array_size() * table_info.dimension_);
for (size_t i = 0; i < searchVector_infos_.query_record_array_size(); i++) {
for (size_t j = 0; j < table_info.dimension_; j++) {
vec_f[i * table_info.dimension_ + j] = searchVector_infos_.query_record_array(i).vector_data(j);
}
}
rc.Record("prepare vector data");
//step 4: search vectors
engine::QueryResults results;
uint64_t record_count = (uint64_t)searchVector_infos_.query_record_array().size();
if(file_id_array_.empty()) {
stat = DBWrapper::DB()->Query(table_name_, (size_t) top_k_, record_count, vec_f.data(), dates, results);
} else {
stat = DBWrapper::DB()->Query(table_name_, file_id_array_,
(size_t) top_k_, record_count, vec_f.data(), dates, results);
}
rc.Record("search vectors from engine");
if(!stat.ok()) {
return SetError(DB_META_TRANSACTION_FAILED, "Engine failed: " + stat.ToString());
}
if(results.empty()) {
return SERVER_SUCCESS; //empty table
}
if(results.size() != record_count) {
std::string msg = "Search " + std::to_string(record_count) + " vectors but only return "
+ std::to_string(results.size()) + " results";
return SetError(SERVER_ILLEGAL_SEARCH_RESULT, msg);
}
rc.Record("do search");
//step 5: construct result array
for(uint64_t i = 0; i < record_count; i++) {
auto& result = results[i];
const auto &record = searchVector_infos_.query_record_array(i);
::milvus::grpc::TopKQueryResult grpc_topk_result;
for(auto& pair : result) {
::milvus::grpc::QueryResult *grpc_result = grpc_topk_result.add_query_result_arrays();
grpc_result->set_id(pair.first);
grpc_result->set_distance(pair.second);
}
writer_.Write(grpc_topk_result);
}
rc.Record("construct result");
rc.Elapse("total cost");
} catch (std::exception& ex) {
return SetError(SERVER_UNEXPECTED_ERROR, ex.what());
}
return SERVER_SUCCESS;
}
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
GetTableRowCountTask::GetTableRowCountTask(const std::string table_name, int64_t& row_count)
: BaseTask(DDL_DML_TASK_GROUP),
table_name_(table_name),
row_count_(row_count) {
}
BaseTaskPtr GetTableRowCountTask::Create(const std::string table_name, int64_t& row_count) {
return std::shared_ptr<BaseTask>(new GetTableRowCountTask(table_name, row_count));
}
ServerError GetTableRowCountTask::OnExecute() {
try {
TimeRecorder rc("GetTableRowCountTask");
//step 1: check arguments
ServerError res = SERVER_SUCCESS;
res = ValidateTableName(table_name_);
if(res != SERVER_SUCCESS) {
return SetError(res, "Invalid table name: " + table_name_);
}
//step 2: get row count
uint64_t row_count = 0;
engine::Status stat = DBWrapper::DB()->GetTableRowCount(table_name_, row_count);
if (!stat.ok()) {
return SetError(DB_META_TRANSACTION_FAILED, "Engine failed: " + stat.ToString());
}
row_count_ = (int64_t) row_count;
rc.Elapse("total cost");
} catch (std::exception& ex) {
return SetError(SERVER_UNEXPECTED_ERROR, ex.what());
}
return SERVER_SUCCESS;
}
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
PingTask::PingTask(const std::string& cmd, std::string& result)
: BaseTask(PING_TASK_GROUP),
cmd_(cmd),
result_(result) {
}
BaseTaskPtr PingTask::Create(const std::string& cmd, std::string& result) {
return std::shared_ptr<BaseTask>(new PingTask(cmd, result));
}
ServerError PingTask::OnExecute() {
if(cmd_ == "version") {
result_ = MILVUS_VERSION;
} else if (cmd_ == "disconnect") {
//TODO stopservice
// MilvusServer::StopService();
}
return SERVER_SUCCESS;
}
}
}
}

View File

@ -0,0 +1,180 @@
/*******************************************************************************
* Copyright (Zilliz) - All Rights Reserved
* Unauthorized copying of this file, via any medium is strictly prohibited.
* Proprietary and confidential.
******************************************************************************/
#pragma once
#include "RequestScheduler.h"
#include "utils/Error.h"
#include "db/Types.h"
#include "milvus.grpc.pb.h"
#include "status.pb.h"
#include <condition_variable>
#include <memory>
namespace zilliz {
namespace milvus {
namespace server {
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
class CreateTableTask : public BaseTask {
public:
static BaseTaskPtr Create(const ::milvus::grpc::TableSchema schema);
protected:
CreateTableTask(const ::milvus::grpc::TableSchema request);
ServerError OnExecute() override;
private:
const ::milvus::grpc::TableSchema schema_;
};
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
class HasTableTask : public BaseTask {
public:
static BaseTaskPtr Create(const std::string table_name, bool& has_table);
protected:
HasTableTask(const std::string request, bool& has_table);
ServerError OnExecute() override;
private:
std::string table_name_;
bool has_table_;
};
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
class DescribeTableTask : public BaseTask {
public:
static BaseTaskPtr Create(const std::string& table_name, ::milvus::grpc::TableSchema* schema);
protected:
DescribeTableTask(const std::string& table_name, ::milvus::grpc::TableSchema* schema);
ServerError OnExecute() override;
private:
std::string table_name_;
::milvus::grpc::TableSchema* schema_;
};
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
class DropTableTask : public BaseTask {
public:
static BaseTaskPtr Create(const std::string& table_name);
protected:
DropTableTask(const std::string& table_name);
ServerError OnExecute() override;
private:
std::string table_name_;
};
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
class BuildIndexTask : public BaseTask {
public:
static BaseTaskPtr Create(const std::string& table_name);
protected:
BuildIndexTask(const std::string& table_name);
ServerError OnExecute() override;
private:
std::string table_name_;
};
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
class ShowTablesTask : public BaseTask {
public:
static BaseTaskPtr Create(::grpc::ServerWriter< ::milvus::grpc::TableName> writer);
protected:
ShowTablesTask(::grpc::ServerWriter< ::milvus::grpc::TableName> writer);
ServerError OnExecute() override;
private:
::grpc::ServerWriter< ::milvus::grpc::TableName> writer_;
};
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
class InsertVectorTask : public BaseTask {
public:
static BaseTaskPtr Create(const ::milvus::grpc::InsertInfos insert_infos,
::milvus::grpc::VectorIds& record_ids_);
protected:
InsertVectorTask(const ::milvus::grpc::InsertInfos insert_infos,
::milvus::grpc::VectorIds& record_ids_);
ServerError OnExecute() override;
private:
const ::milvus::grpc::InsertInfos insert_infos_;
::milvus::grpc::VectorIds& record_ids_;
};
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
class SearchVectorTask : public BaseTask {
public:
static BaseTaskPtr Create(const ::milvus::grpc::SearchVectorInfos searchVectorInfos,
const std::vector<std::string> file_id_array,
::grpc::ServerWriter<::milvus::grpc::TopKQueryResult> writer);
protected:
SearchVectorTask(const ::milvus::grpc::SearchVectorInfos searchVectorInfos,
const std::vector<std::string> file_id_array,
::grpc::ServerWriter<::milvus::grpc::TopKQueryResult> writer);
ServerError OnExecute() override;
private:
const ::milvus::grpc::SearchVectorInfos searchVector_infos_;
std::vector<std::string> file_id_array_;
::grpc::ServerWriter<::milvus::grpc::TopKQueryResult> writer_;
};
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
class GetTableRowCountTask : public BaseTask {
public:
static BaseTaskPtr Create(const std::string table_name, int64_t& row_count);
protected:
GetTableRowCountTask(const std::string table_name, int64_t& row_count);
ServerError OnExecute() override;
private:
std::string table_name_;
int64_t row_count_;
};
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
class PingTask : public BaseTask {
public:
static BaseTaskPtr Create(const std::string& cmd, std::string& result);
protected:
PingTask(const std::string& cmd, std::string& result);
ServerError OnExecute() override;
private:
std::string cmd_;
std::string& result_;
};
}
}
}

View File

@ -0,0 +1,245 @@
////////////////////////////////////////////////////////////////////////////////
// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved
// Unauthorized copying of this file, via any medium is strictly prohibited.
// Proprietary and confidential.
////////////////////////////////////////////////////////////////////////////////
#include "Server.h"
#include "ServerConfig.h"
#include "MilvusServer.h"
#include "utils/Log.h"
#include "utils/SignalUtil.h"
#include "utils/TimeRecorder.h"
#include "license/LicenseCheck.h"
#include "metrics/Metrics.h"
#include <fcntl.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <csignal>
//#include <numaif.h>
#include <unistd.h>
#include <string.h>
#include "metrics/Metrics.h"
namespace zilliz {
namespace milvus {
namespace server {
Server*
Server::Instance() {
static Server server;
return &server;
}
Server::Server() {
}
Server::~Server() {
}
void
Server::Init(int64_t daemonized, const std::string& pid_filename, const std::string& config_filename) {
daemonized_ = daemonized;
pid_filename_ = pid_filename;
config_filename_ = config_filename;
}
void
Server::Daemonize() {
if (daemonized_ == 0) {
return;
}
SERVER_LOG_INFO << "Milvus server run in daemonize mode";
// std::string log_path(GetLogDirFullPath());
// log_path += "zdb_server.(INFO/WARNNING/ERROR/CRITICAL)";
// SERVER_LOG_INFO << "Log will be exported to: " + log_path);
pid_t pid = 0;
// Fork off the parent process
pid = fork();
// An error occurred
if (pid < 0) {
exit(EXIT_FAILURE);
}
// Success: terminate parent
if (pid > 0) {
exit(EXIT_SUCCESS);
}
// On success: The child process becomes session leader
if (setsid() < 0) {
exit(EXIT_FAILURE);
}
// Ignore signal sent from child to parent process
signal(SIGCHLD, SIG_IGN);
// Fork off for the second time
pid = fork();
// An error occurred
if (pid < 0) {
exit(EXIT_FAILURE);
}
// Terminate the parent
if (pid > 0) {
exit(EXIT_SUCCESS);
}
// Set new file permissions
umask(0);
// Change the working directory to root
int ret = chdir("/");
if(ret != 0){
return;
}
// Close all open fd
for (long fd = sysconf(_SC_OPEN_MAX); fd > 0; fd--) {
close(fd);
}
SERVER_LOG_INFO << "Redirect stdin/stdout/stderr to /dev/null";
// Redirect stdin/stdout/stderr to /dev/null
stdin = fopen("/dev/null", "r");
stdout = fopen("/dev/null", "w+");
stderr = fopen("/dev/null", "w+");
// Try to write PID of daemon to lockfile
if (!pid_filename_.empty()) {
pid_fd = open(pid_filename_.c_str(), O_RDWR | O_CREAT, 0640);
if (pid_fd < 0) {
SERVER_LOG_INFO << "Can't open filename: " + pid_filename_ + ", Error: " + strerror(errno);
exit(EXIT_FAILURE);
}
if (lockf(pid_fd, F_TLOCK, 0) < 0) {
SERVER_LOG_INFO << "Can't lock filename: " + pid_filename_ + ", Error: " + strerror(errno);
exit(EXIT_FAILURE);
}
std::string pid_file_context = std::to_string(getpid());
ssize_t res = write(pid_fd, pid_file_context.c_str(), pid_file_context.size());
if(res != 0){
return;
}
}
}
int
Server::Start() {
if (daemonized_) {
Daemonize();
}
do {
try {
// Read config file
if(LoadConfig() != SERVER_SUCCESS) {
return 1;
}
//log path is defined by LoadConfig, so InitLog must be called after LoadConfig
ServerConfig &config = ServerConfig::GetInstance();
ConfigNode server_config = config.GetConfig(CONFIG_SERVER);
#ifdef ENABLE_LICENSE
ConfigNode license_config = config.GetConfig(CONFIG_LICENSE);
std::string license_file_path = license_config.GetValue(CONFIG_LICENSE_PATH);
SERVER_LOG_INFO << "License path: " << license_file_path;
if(server::LicenseCheck::LegalityCheck(license_file_path) != SERVER_SUCCESS) {
SERVER_LOG_ERROR << "License check failed";
exit(1);
}
server::LicenseCheck::GetInstance().StartCountingDown(license_file_path);
#endif
// Handle Signal
signal(SIGINT, SignalUtil::HandleSignal);
signal(SIGHUP, SignalUtil::HandleSignal);
signal(SIGTERM, SignalUtil::HandleSignal);
server::Metrics::GetInstance().Init();
server::SystemInfo::GetInstance().Init();
std::cout << "Milvus server start successfully." << std::endl;
StartService();
} catch(std::exception& ex){
SERVER_LOG_ERROR << "Milvus server encounter exception: " << std::string(ex.what())
<< "Is another server instance running?";
break;
}
} while(false);
Stop();
return 0;
}
void
Server::Stop() {
std::cout << "Milvus server is going to shutdown ..." << std::endl;
// Unlock and close lockfile
if (pid_fd != -1) {
int ret = lockf(pid_fd, F_ULOCK, 0);
if(ret != 0){
std::cout << "Can't lock file: " << strerror(errno) << std::endl;
exit(0);
}
ret = close(pid_fd);
if(ret != 0){
std::cout << "Can't close file: " << strerror(errno) << std::endl;
exit(0);
}
}
// Try to delete lockfile
if (!pid_filename_.empty()) {
int ret = unlink(pid_filename_.c_str());
if(ret != 0){
std::cout << "Can't unlink file: " << strerror(errno) << std::endl;
exit(0);
}
}
running_ = 0;
StopService();
#ifdef ENABLE_LICENSE
server::LicenseCheck::GetInstance().StopCountingDown();
#endif
std::cout << "Milvus server is closed!" << std::endl;
}
ServerError
Server::LoadConfig() {
ServerConfig::GetInstance().LoadConfigFile(config_filename_);
return SERVER_SUCCESS;
}
void
Server::StartService() {
MilvusServer::StartService();
}
void
Server::StopService() {
MilvusServer::StopService();
}
}
}
}

View File

@ -0,0 +1,47 @@
/*******************************************************************************
* Copyright (Zilliz) - All Rights Reserved
* Unauthorized copying of this file, via any medium is strictly prohibited.
* Proprietary and confidential.
******************************************************************************/
#pragma once
#include "utils/Error.h"
#include <cstdint>
#include <string>
namespace zilliz {
namespace milvus {
namespace server {
class Server {
public:
static Server* Instance();
void Init(int64_t daemonized, const std::string& pid_filename, const std::string& config_filename);
int Start();
void Stop();
private:
Server();
~Server();
void Daemonize();
static void HandleSignal(int signal);
ServerError LoadConfig();
void StartService();
void StopService();
private:
int64_t daemonized_ = 0;
int64_t running_ = 1;
int pid_fd = -1;
std::string pid_filename_;
std::string config_filename_;
}; // Server
} // server
} // sql
} // zilliz

View File

@ -0,0 +1,82 @@
////////////////////////////////////////////////////////////////////////////////
// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved
// Unauthorized copying of this file, via any medium is strictly prohibited.
// Proprietary and confidential.
////////////////////////////////////////////////////////////////////////////////
#include "ServerConfig.h"
#include <sys/stat.h>
#include <sys/types.h>
#include <unistd.h>
#include <stdlib.h>
#include <iostream>
#include "config/IConfigMgr.h"
namespace zilliz {
namespace milvus {
namespace server {
ServerConfig&
ServerConfig::GetInstance() {
static ServerConfig config;
return config;
}
ServerError
ServerConfig::LoadConfigFile(const std::string& config_filename) {
std::string filename = config_filename;
if(filename.empty()){
std::cout << "ERROR: a config file is required" << std::endl;
exit(1);//directly exit program if config file not specified
}
struct stat directoryStat;
int statOK = stat(filename.c_str(), &directoryStat);
if (statOK != 0) {
std::cout << "ERROR: " << filename << " not found!" << std::endl;
exit(1);//directly exit program if config file not found
}
try {
IConfigMgr* mgr = const_cast<IConfigMgr*>(IConfigMgr::GetInstance());
ServerError err = mgr->LoadConfigFile(filename);
if(err != 0) {
std::cout << "Server failed to load config file" << std::endl;
exit(1);//directly exit program if the config file is illegal
}
}
catch (YAML::Exception& e) {
std::cout << "Server failed to load config file: " << std::endl;
return SERVER_UNEXPECTED_ERROR;
}
return SERVER_SUCCESS;
}
void
ServerConfig::PrintAll() const {
if(const IConfigMgr* mgr = IConfigMgr::GetInstance()) {
std::string str = mgr->DumpString();
// SERVER_LOG_INFO << "\n" << str;
std::cout << "\n" << str << std::endl;
}
}
ConfigNode
ServerConfig::GetConfig(const std::string& name) const {
const IConfigMgr* mgr = IConfigMgr::GetInstance();
const ConfigNode& root_node = mgr->GetRootNode();
return root_node.GetChild(name);
}
ConfigNode&
ServerConfig::GetConfig(const std::string& name) {
IConfigMgr* mgr = IConfigMgr::GetInstance();
ConfigNode& root_node = mgr->GetRootNode();
return root_node.GetChild(name);
}
}
}
}

View File

@ -0,0 +1,67 @@
/*******************************************************************************
* Copyright (Zilliz) - All Rights Reserved
* Unauthorized copying of this file, via any medium is strictly prohibited.
* Proprietary and confidential.
******************************************************************************/
#pragma once
#include "utils/Error.h"
#include "config/ConfigNode.h"
#include "yaml-cpp/yaml.h"
namespace zilliz {
namespace milvus {
namespace server {
static const std::string CONFIG_SERVER = "server_config";
static const std::string CONFIG_SERVER_ADDRESS = "address";
static const std::string CONFIG_SERVER_PORT = "port";
static const std::string CONFIG_SERVER_PROTOCOL = "transfer_protocol";
static const std::string CONFIG_CLUSTER_MODE = "mode";
static const std::string CONFIG_DB = "db_config";
static const std::string CONFIG_DB_URL = "db_backend_url";
static const std::string CONFIG_DB_PATH = "db_path";
static const std::string CONFIG_DB_SLAVE_PATH = "db_slave_path";
static const std::string CONFIG_DB_INDEX_TRIGGER_SIZE = "index_building_threshold";
static const std::string CONFIG_DB_ARCHIVE_DISK = "archive_disk_threshold";
static const std::string CONFIG_DB_ARCHIVE_DAYS = "archive_days_threshold";
static const std::string CONFIG_DB_INSERT_BUFFER_SIZE = "insert_buffer_size";
static const std::string CONFIG_LOG = "log_config";
static const std::string CONFIG_CACHE = "cache_config";
static const std::string CONFIG_CPU_CACHE_CAPACITY = "cpu_cache_capacity";
static const std::string CONFIG_GPU_CACHE_CAPACITY = "gpu_cache_capacity";
static const std::string CONFIG_LICENSE = "license_config";
static const std::string CONFIG_LICENSE_PATH = "license_path";
static const std::string CONFIG_METRIC = "metric_config";
static const std::string CONFIG_METRIC_IS_STARTUP = "is_startup";
static const std::string CONFIG_METRIC_COLLECTOR = "collector";
static const std::string CONFIG_PROMETHEUS = "prometheus_config";
static const std::string CONFIG_METRIC_PROMETHEUS_PORT = "port";
static const std::string CONFIG_ENGINE = "engine_config";
static const std::string CONFIG_NPROBE = "nprobe";
static const std::string CONFIG_NLIST = "nlist";
static const std::string CONFIG_DCBT = "use_blas_threshold";
static const std::string CONFIG_METRICTYPE = "metric_type";
class ServerConfig {
public:
static ServerConfig &GetInstance();
ServerError LoadConfigFile(const std::string& config_filename);
void PrintAll() const;
ConfigNode GetConfig(const std::string& name) const;
ConfigNode& GetConfig(const std::string& name);
};
}
}
}

View File

@ -3,7 +3,8 @@
// Unauthorized copying of this file, via any medium is strictly prohibited.
// Proprietary and confidential.
////////////////////////////////////////////////////////////////////////////////
#include "server/Server.h"
//#include "server/Server.h"
#include "grpcserver/Server.h"
#include "version.h"
#include <getopt.h>

View File

@ -86,6 +86,7 @@ ClientProxy::CreateTable(const TableSchema &param) {
}
try {
thrift::TableSchema schema;
schema.__set_table_name(param.table_name);
schema.__set_index_type((int)param.index_type);

View File

@ -56,5 +56,5 @@ public:
private:
std::shared_ptr<ClientProxy> client_proxy_;
};
}