From 08a62e3081e9d8baf83ab4f6d631e9985219e9e3 Mon Sep 17 00:00:00 2001 From: "zhenshan.cao" Date: Sat, 19 Sep 2020 17:48:39 +0800 Subject: [PATCH] Refactor SDK Signed-off-by: zhenshan.cao --- sdk/CMakeLists.txt | 2 ++ sdk/examples/CMakeLists.txt | 11 ------ sdk/examples/simple/CMakeLists.txt | 40 +++++++++++---------- sdk/examples/simple/CreateCollection.cpp | 12 +++++-- sdk/examples/simple/delete.cpp | 13 ++++--- sdk/examples/simple/insert.cpp | 42 ++++++++++++++++------- sdk/examples/simple/ip.h | 16 --------- sdk/examples/simple/search.cpp | 22 ++++++------ sdk/grpc/ClientProxy.cpp | 36 ++++++++----------- sdk/grpc/GrpcClient.cpp | 26 +++++++------- sdk/{examples => }/utils/ThreadPool.h | 0 sdk/{examples => }/utils/TimeRecorder.cpp | 2 +- sdk/{examples => }/utils/TimeRecorder.h | 0 sdk/{examples => }/utils/Utils.cpp | 4 +-- sdk/{examples => }/utils/Utils.h | 0 15 files changed, 111 insertions(+), 115 deletions(-) delete mode 100644 sdk/examples/simple/ip.h rename sdk/{examples => }/utils/ThreadPool.h (100%) rename sdk/{examples => }/utils/TimeRecorder.cpp (96%) rename sdk/{examples => }/utils/TimeRecorder.h (100%) rename sdk/{examples => }/utils/Utils.cpp (99%) rename sdk/{examples => }/utils/Utils.h (100%) diff --git a/sdk/CMakeLists.txt b/sdk/CMakeLists.txt index 4938f5f36f..51edd9a9dd 100644 --- a/sdk/CMakeLists.txt +++ b/sdk/CMakeLists.txt @@ -66,6 +66,7 @@ endif () aux_source_directory(interface interface_files) aux_source_directory(grpc grpc_client_files) +aux_source_directory(utils utils_files) set(grpc_service_files grpc-gen/message.grpc.pb.cc @@ -84,6 +85,7 @@ add_library(milvus_sdk SHARED ${interface_files} ${grpc_client_files} ${grpc_service_files} + ${utils_files} ) target_link_libraries(milvus_sdk diff --git a/sdk/examples/CMakeLists.txt b/sdk/examples/CMakeLists.txt index 857750b0b6..0ac0a3ef80 100644 --- a/sdk/examples/CMakeLists.txt +++ b/sdk/examples/CMakeLists.txt @@ -12,14 +12,3 @@ #------------------------------------------------------------------------------- add_subdirectory(simple) -aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR}/utils UTIL_SRC_FILES) -aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR}/common COMMON_SRC_FILES) - -file( GLOB APP_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/*.cpp ) -foreach( sourcefile ${APP_SOURCES} ) - file(RELATIVE_PATH filename ${CMAKE_CURRENT_SOURCE_DIR} ${sourcefile}) - string( REPLACE ".cpp" "" program ${filename} ) - add_executable( ${program} ${sourcefile} ${COMMON_SRC_FILES} ${UTIL_SRC_FILES}) - target_link_libraries( ${program} milvus_sdk pthread ) - install(TARGETS ${program} DESTINATION bin) -endforeach( sourcefile ${APP_SOURCES} ) diff --git a/sdk/examples/simple/CMakeLists.txt b/sdk/examples/simple/CMakeLists.txt index 09307e1904..e46e47d621 100644 --- a/sdk/examples/simple/CMakeLists.txt +++ b/sdk/examples/simple/CMakeLists.txt @@ -1,21 +1,23 @@ -add_executable(search search.cpp) -target_link_libraries(search milvus_sdk pthread) -install(TARGETS search DESTINATION test) +#------------------------------------------------------------------------------- +# Copyright (C) 2019-2020 Zilliz. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software distributed under the License +# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +# or implied. See the License for the specific language governing permissions and limitations under the License. +#------------------------------------------------------------------------------- +aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR}/../common COMMON_SRC_FILES) - - -add_executable(insert insert.cpp) -target_link_libraries(insert milvus_sdk pthread) -install(TARGETS insert DESTINATION test) - - - - -add_executable(delete delete.cpp) -target_link_libraries(delete milvus_sdk pthread) -install(TARGETS delete DESTINATION test) - -add_executable(create CreateCollection.cpp) -target_link_libraries(create milvus_sdk pthread) -install(TARGETS create DESTINATION test) \ No newline at end of file +file( GLOB APP_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/*.cpp ) +foreach( sourcefile ${APP_SOURCES} ) + file(RELATIVE_PATH filename ${CMAKE_CURRENT_SOURCE_DIR} ${sourcefile}) + string( REPLACE ".cpp" "" program ${filename} ) + add_executable( ${program} ${sourcefile} ${COMMON_SRC_FILES}) + target_link_libraries( ${program} milvus_sdk pthread ) + install(TARGETS ${program} DESTINATION bin) +endforeach( sourcefile ${APP_SOURCES} ) diff --git a/sdk/examples/simple/CreateCollection.cpp b/sdk/examples/simple/CreateCollection.cpp index f66a2ee1d0..b3c9e7a0d4 100644 --- a/sdk/examples/simple/CreateCollection.cpp +++ b/sdk/examples/simple/CreateCollection.cpp @@ -3,12 +3,18 @@ #include #include #include +#include "utils/Utils.h" -int main() { +int main(int argc , char**argv) { + + TestParameters parameters = milvus_sdk::Utils::ParseTestParameters(argc, argv); + if (!parameters.is_valid){ + return 0; + } auto client = milvus::ConnectionImpl(); milvus::ConnectParam connect_param; - connect_param.ip_address = "localhost"; - connect_param.port = "19530"; + connect_param.ip_address = parameters.address_.empty() ? "127.0.0.1":parameters.address_; + connect_param.port = parameters.port_.empty() ? "19530":parameters.port_ ; client.Connect(connect_param); milvus::Status stat; diff --git a/sdk/examples/simple/delete.cpp b/sdk/examples/simple/delete.cpp index a05982e80e..f582586069 100644 --- a/sdk/examples/simple/delete.cpp +++ b/sdk/examples/simple/delete.cpp @@ -13,16 +13,19 @@ #include #include #include - #include "interface/ConnectionImpl.h" -#include "ip.h" - +#include "utils/Utils.h" int main(int argc, char *argv[]) { + TestParameters parameters = milvus_sdk::Utils::ParseTestParameters(argc, argv); + if (!parameters.is_valid){ + return 0; + } auto client = milvus::ConnectionImpl(); milvus::ConnectParam connect_param; - connect_param.ip_address = IP; - connect_param.port = "19530"; + connect_param.ip_address = parameters.address_.empty() ? "127.0.0.1":parameters.address_; + connect_param.port = parameters.port_.empty() ? "19530":parameters.port_ ; + client.Connect(connect_param); std::vector delete_ids; diff --git a/sdk/examples/simple/insert.cpp b/sdk/examples/simple/insert.cpp index 5d4261060c..87bb31a6a0 100644 --- a/sdk/examples/simple/insert.cpp +++ b/sdk/examples/simple/insert.cpp @@ -15,17 +15,19 @@ #include #include -#include "examples/utils/Utils.h" +#include "utils/Utils.h" #include "grpc/ClientProxy.h" #include "interface/ConnectionImpl.h" -#include "ip.h" +#include "utils/TimeRecorder.h" const int N = 100; const int DIM = 16; +const int LOOP = 10; const milvus::FieldValue GetData() { milvus::FieldValue value_map; + std::vector int32_data; for (int i = 0; i < N; i++) { int32_data.push_back(i); @@ -46,15 +48,29 @@ const milvus::FieldValue GetData() { int main(int argc, char* argv[]) { - auto client = milvus::ConnectionImpl(); - milvus::ConnectParam connect_param; - connect_param.ip_address = IP; - connect_param.port = "19530"; - client.Connect(connect_param); - std::vector ids_array; - auto data = GetData(); - for (int64_t i = 0; i < N; i++) { - ids_array.push_back(i); - } - auto status = client.Insert("collection1", "tag01", data, ids_array); + TestParameters parameters = milvus_sdk::Utils::ParseTestParameters(argc, argv); + if (!parameters.is_valid){ + return 0; + } + + auto client = milvus::ConnectionImpl(); + milvus::ConnectParam connect_param; + connect_param.ip_address = parameters.address_.empty() ? "127.0.0.1":parameters.address_; + connect_param.port = parameters.port_.empty() ? "19530":parameters.port_ ; + client.Connect(connect_param); + std::vector ids_array; + auto data = GetData(); + for (int64_t i = 0; i < N; i++) { + ids_array.push_back(i); + } + + milvus_sdk::TimeRecorder insert("insert"); + for (int j = 0; j < LOOP; ++j) { + auto status = client.Insert("collection1", "tag01", data, ids_array); + if (!status.ok()){ + return -1; + } + } + + return 0; } diff --git a/sdk/examples/simple/ip.h b/sdk/examples/simple/ip.h deleted file mode 100644 index 1558e70bea..0000000000 --- a/sdk/examples/simple/ip.h +++ /dev/null @@ -1,16 +0,0 @@ -// Copyright (C) 2019-2020 Zilliz. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software distributed under the License -// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -// or implied. See the License for the specific language governing permissions and limitations under the License. - - - -const std::string IP = "localhost"; -//const std::string IP = "192.168.2.9"; -//const std::string IP = "192.168.2.28"; \ No newline at end of file diff --git a/sdk/examples/simple/search.cpp b/sdk/examples/simple/search.cpp index 64222f54b5..c58d4a75b0 100644 --- a/sdk/examples/simple/search.cpp +++ b/sdk/examples/simple/search.cpp @@ -15,15 +15,21 @@ #include "include/MilvusApi.h" #include "grpc/ClientProxy.h" #include "interface/ConnectionImpl.h" -#include "ip.h" +#include "utils/TimeRecorder.h" +#include "utils/Utils.h" const int TOP_K = 10; int main(int argc , char**argv) { + TestParameters parameters = milvus_sdk::Utils::ParseTestParameters(argc, argv); + if (!parameters.is_valid){ + return 0; + } + auto client = milvus::ConnectionImpl(); milvus::ConnectParam connect_param; - connect_param.ip_address = IP; - connect_param.port = "19530"; + connect_param.ip_address = parameters.address_.empty() ? "127.0.0.1":parameters.address_; + connect_param.port = parameters.port_.empty() ? "19530":parameters.port_ ; client.Connect(connect_param); std::vector ids_array; std::vector partition_list; @@ -56,13 +62,9 @@ int main(int argc , char**argv) { milvus::TopKQueryResult result; - - auto t1 = std::chrono::high_resolution_clock::now(); + milvus_sdk::TimeRecorder test_search("search"); auto status = client.Search("collection1", partition_list, "dsl", vectorParam, result); - auto t2 = std::chrono::high_resolution_clock::now(); - auto duration = std::chrono::duration_cast(t2 - t1).count(); - - std::cout << "Query run time: " << duration/1000.0 << "ms" << std::endl; - return 0; + return 0; } + diff --git a/sdk/grpc/ClientProxy.cpp b/sdk/grpc/ClientProxy.cpp index c2fbc7de82..edf348e413 100644 --- a/sdk/grpc/ClientProxy.cpp +++ b/sdk/grpc/ClientProxy.cpp @@ -407,32 +407,24 @@ ClientProxy::Disconnect() { Status ClientProxy::CreateCollection(const Mapping& mapping, const std::string& extra_params) { try { -// ::milvus::grpc::Mapping grpc_mapping; -// grpc_mapping.set_collection_name(mapping.collection_name); -// for (auto& field : mapping.fields) { -// auto grpc_field = grpc_mapping.add_fields(); -// grpc_field->set_name(field->field_name); -// grpc_field->set_type((::milvus::grpc::DataType)field->field_type); -// JSON json_index_param = JSON::parse(field->index_params); -// for (auto& json_param : json_index_param.items()) { -// auto grpc_index_param = grpc_field->add_index_params(); -// grpc_index_param->set_key(json_param.key()); -// grpc_index_param->set_value(json_param.value()); -// } -// -// auto grpc_extra_param = grpc_field->add_extra_params(); -// grpc_extra_param->set_key(EXTRA_PARAM_KEY); -// grpc_extra_param->set_value(field->extra_params); -// } -// auto grpc_param = grpc_mapping.add_extra_params(); -// grpc_param->set_key(EXTRA_PARAM_KEY); -// grpc_param->set_value(extra_params); + ::milvus::grpc::Mapping grpc_mapping; + grpc_mapping.set_collection_name(mapping.collection_name); + for (auto& field : mapping.fields) { + auto grpc_field = grpc_mapping.mutable_schema()->add_field_metas(); + grpc_field->set_field_name(field->field_name); + grpc_field->set_type((::milvus::grpc::DataType)field->field_type); + grpc_field->set_dim(field->dim); -// return client_ptr_->CreateCollection(grpc_mapping); - return Status::OK(); + } + auto grpc_param = grpc_mapping.add_extra_params(); + grpc_param->set_key(EXTRA_PARAM_KEY); + grpc_param->set_value(extra_params); + + return client_ptr_->CreateCollection(grpc_mapping); } catch (std::exception& ex) { return Status(StatusCode::UnknownError, "Failed to create collection: " + std::string(ex.what())); } + return Status::OK(); } Status diff --git a/sdk/grpc/GrpcClient.cpp b/sdk/grpc/GrpcClient.cpp index 356a5118c5..00353535f6 100644 --- a/sdk/grpc/GrpcClient.cpp +++ b/sdk/grpc/GrpcClient.cpp @@ -37,19 +37,19 @@ GrpcClient::~GrpcClient() = default; Status GrpcClient::CreateCollection(const milvus::grpc::Mapping& mapping) { -// ClientContext context; -// ::milvus::grpc::Status response; -// ::grpc::Status grpc_status = stub_->CreateCollection(&context, mapping, &response); -// -// if (!grpc_status.ok()) { -// std::cerr << "CreateHybridCollection gRPC failed!" << std::endl; -// return Status(StatusCode::RPCFailed, grpc_status.error_message()); -// } -// -// if (response.error_code() != grpc::SUCCESS) { -// std::cerr << response.reason() << std::endl; -// return Status(StatusCode::ServerFailed, response.reason()); -// } + ClientContext context; + ::milvus::grpc::Status response; + ::grpc::Status grpc_status = stub_->CreateCollection(&context, mapping, &response); + + if (!grpc_status.ok()) { + std::cerr << "CreateHybridCollection gRPC failed!" << std::endl; + return Status(StatusCode::RPCFailed, grpc_status.error_message()); + } + + if (response.error_code() != grpc::SUCCESS) { + std::cerr << response.reason() << std::endl; + return Status(StatusCode::ServerFailed, response.reason()); + } return Status::OK(); } diff --git a/sdk/examples/utils/ThreadPool.h b/sdk/utils/ThreadPool.h similarity index 100% rename from sdk/examples/utils/ThreadPool.h rename to sdk/utils/ThreadPool.h diff --git a/sdk/examples/utils/TimeRecorder.cpp b/sdk/utils/TimeRecorder.cpp similarity index 96% rename from sdk/examples/utils/TimeRecorder.cpp rename to sdk/utils/TimeRecorder.cpp index 8d5b69c506..b290dbf6d8 100644 --- a/sdk/examples/utils/TimeRecorder.cpp +++ b/sdk/utils/TimeRecorder.cpp @@ -9,7 +9,7 @@ // is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express // or implied. See the License for the specific language governing permissions and limitations under the License. -#include "examples/utils/TimeRecorder.h" +#include "TimeRecorder.h" #include diff --git a/sdk/examples/utils/TimeRecorder.h b/sdk/utils/TimeRecorder.h similarity index 100% rename from sdk/examples/utils/TimeRecorder.h rename to sdk/utils/TimeRecorder.h diff --git a/sdk/examples/utils/Utils.cpp b/sdk/utils/Utils.cpp similarity index 99% rename from sdk/examples/utils/Utils.cpp rename to sdk/utils/Utils.cpp index 160ba42f29..f3954eb56a 100644 --- a/sdk/examples/utils/Utils.cpp +++ b/sdk/utils/Utils.cpp @@ -9,7 +9,7 @@ // is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express // or implied. See the License for the specific language governing permissions and limitations under the License. -#include "examples/utils/Utils.h" +#include "Utils.h" #include #include @@ -25,7 +25,7 @@ #include #include -#include "examples/utils/TimeRecorder.h" +#include "TimeRecorder.h" namespace { diff --git a/sdk/examples/utils/Utils.h b/sdk/utils/Utils.h similarity index 100% rename from sdk/examples/utils/Utils.h rename to sdk/utils/Utils.h