From 29586ba725686b1d9f3244ecc3ec2aac80a62db5 Mon Sep 17 00:00:00 2001 From: groot Date: Fri, 15 Nov 2019 19:06:14 +0800 Subject: [PATCH 1/4] #353 Rename config.h.in to version.h.in --- CHANGELOG.md | 1 + core/CMakeLists.txt | 2 +- core/src/db/meta/MetaTypes.h | 2 +- core/src/main.cpp | 2 +- core/src/sdk/grpc/ClientProxy.cpp | 2 +- core/src/server/Server.cpp | 2 +- core/src/server/grpc_impl/GrpcRequestTask.cpp | 2 +- core/src/{config.h.in => version.h.in} | 0 core/unittest/server/test_rpc.cpp | 2 +- 9 files changed, 8 insertions(+), 7 deletions(-) rename core/src/{config.h.in => version.h.in} (100%) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2aab00f8db..76c8a13c45 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ Please mark all change in change log and use the ticket from JIRA. - \#248 - Reside src/external in thirdparty - \#316 - Some files not merged after vectors added - \#340 - Test cases run failed on 0.6.0 +- \#353 - Rename config.h.in to version.h.in ## Feature - \#12 - Pure CPU version for Milvus diff --git a/core/CMakeLists.txt b/core/CMakeLists.txt index f24935f788..671330c192 100644 --- a/core/CMakeLists.txt +++ b/core/CMakeLists.txt @@ -70,7 +70,7 @@ if (MILVUS_VERSION_MAJOR STREQUAL "" endif() message(STATUS "Build version = ${MILVUS_VERSION}") -configure_file(${CMAKE_CURRENT_SOURCE_DIR}/src/config.h.in ${CMAKE_CURRENT_SOURCE_DIR}/src/config.h @ONLY) +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/src/version.h.in ${CMAKE_CURRENT_SOURCE_DIR}/src/version.h @ONLY) message(STATUS "Milvus version: " "${MILVUS_VERSION_MAJOR}.${MILVUS_VERSION_MINOR}.${MILVUS_VERSION_PATCH} " diff --git a/core/src/db/meta/MetaTypes.h b/core/src/db/meta/MetaTypes.h index 28f35e76fc..d98b74be7d 100644 --- a/core/src/db/meta/MetaTypes.h +++ b/core/src/db/meta/MetaTypes.h @@ -19,7 +19,7 @@ #include "db/Constants.h" #include "db/engine/ExecutionEngine.h" -#include "src/config.h" +#include "src/version.h" #include #include diff --git a/core/src/main.cpp b/core/src/main.cpp index 401736c34f..b39ba87997 100644 --- a/core/src/main.cpp +++ b/core/src/main.cpp @@ -25,7 +25,7 @@ #include "easyloggingpp/easylogging++.h" #include "metrics/Metrics.h" #include "server/Server.h" -#include "src/config.h" +#include "src/version.h" #include "utils/CommonUtil.h" #include "utils/SignalUtil.h" diff --git a/core/src/sdk/grpc/ClientProxy.cpp b/core/src/sdk/grpc/ClientProxy.cpp index 4ec94cfa98..4a84323d64 100644 --- a/core/src/sdk/grpc/ClientProxy.cpp +++ b/core/src/sdk/grpc/ClientProxy.cpp @@ -17,7 +17,7 @@ #include "sdk/grpc/ClientProxy.h" #include "grpc/gen-milvus/milvus.grpc.pb.h" -#include "src/config.h" +#include "src/version.h" #include #include diff --git a/core/src/server/Server.cpp b/core/src/server/Server.cpp index eb2b077829..5676504722 100644 --- a/core/src/server/Server.cpp +++ b/core/src/server/Server.cpp @@ -25,7 +25,7 @@ #include "server/DBWrapper.h" #include "server/Server.h" #include "server/grpc_impl/GrpcServer.h" -#include "src/config.h" +#include "src/version.h" #include "utils/Log.h" #include "utils/LogUtil.h" #include "utils/SignalUtil.h" diff --git a/core/src/server/grpc_impl/GrpcRequestTask.cpp b/core/src/server/grpc_impl/GrpcRequestTask.cpp index b8a9dbccbf..0d685bed4b 100644 --- a/core/src/server/grpc_impl/GrpcRequestTask.cpp +++ b/core/src/server/grpc_impl/GrpcRequestTask.cpp @@ -29,7 +29,7 @@ #include "scheduler/SchedInst.h" #include "server/DBWrapper.h" #include "server/Server.h" -#include "src/config.h" +#include "src/version.h" #include "utils/CommonUtil.h" #include "utils/Log.h" #include "utils/TimeRecorder.h" diff --git a/core/src/config.h.in b/core/src/version.h.in similarity index 100% rename from core/src/config.h.in rename to core/src/version.h.in diff --git a/core/unittest/server/test_rpc.cpp b/core/unittest/server/test_rpc.cpp index 76c10769a7..e491d5e7f3 100644 --- a/core/unittest/server/test_rpc.cpp +++ b/core/unittest/server/test_rpc.cpp @@ -23,7 +23,7 @@ #include "server/grpc_impl/GrpcRequestHandler.h" #include "server/grpc_impl/GrpcRequestScheduler.h" #include "server/grpc_impl/GrpcRequestTask.h" -#include "src/config.h" +#include "src/version.h" #include "grpc/gen-milvus/milvus.grpc.pb.h" #include "grpc/gen-status/status.pb.h" From a7cb907e44e44481e7391a0f08c2654359430e77 Mon Sep 17 00:00:00 2001 From: Zhiru Zhu Date: Fri, 15 Nov 2019 20:15:34 +0800 Subject: [PATCH 2/4] remove gpu in unittest --- core/unittest/db/utils.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/core/unittest/db/utils.cpp b/core/unittest/db/utils.cpp index f87371fe6b..bcf1462ecd 100644 --- a/core/unittest/db/utils.cpp +++ b/core/unittest/db/utils.cpp @@ -155,12 +155,10 @@ DBTest::SetUp() { res_mgr->Clear(); res_mgr->Add(milvus::scheduler::ResourceFactory::Create("disk", "DISK", 0, true, false)); res_mgr->Add(milvus::scheduler::ResourceFactory::Create("cpu", "CPU", 0, true, true)); - res_mgr->Add(milvus::scheduler::ResourceFactory::Create("gtx1660", "GPU", 0, true, true)); auto default_conn = milvus::scheduler::Connection("IO", 500.0); auto PCIE = milvus::scheduler::Connection("IO", 11000.0); res_mgr->Connect("disk", "cpu", default_conn); - res_mgr->Connect("cpu", "gtx1660", PCIE); res_mgr->Start(); milvus::scheduler::SchedInst::GetInstance()->Start(); From ead945870cd8d742f870862e6c60031d22841c0a Mon Sep 17 00:00:00 2001 From: zhenwu Date: Mon, 18 Nov 2019 15:49:37 +0800 Subject: [PATCH 3/4] Update nightly-build config --- ci/jenkins/Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/jenkins/Jenkinsfile b/ci/jenkins/Jenkinsfile index f0562ada64..8d3953b112 100644 --- a/ci/jenkins/Jenkinsfile +++ b/ci/jenkins/Jenkinsfile @@ -2,7 +2,7 @@ String cron_timezone = "TZ=Asia/Shanghai" String cron_string = BRANCH_NAME == "master" ? "H 0 * * * " : "" -cron_string = BRANCH_NAME == "0.5.1" ? "H 1 * * * " : cron_string +cron_string = BRANCH_NAME == "0.6.0" ? "H 1 * * * " : cron_string pipeline { agent none From 661a60374e2f9d10cd73b5a676c01cf7bbe181e6 Mon Sep 17 00:00:00 2001 From: groot Date: Mon, 18 Nov 2019 15:58:50 +0800 Subject: [PATCH 4/4] dont modify grpc generated code --- core/src/grpc/gen-milvus/milvus.pb.h | 57 ++++++------------- core/src/sdk/grpc/ClientProxy.cpp | 15 ++++- .../grpc_impl/request/SearchRequest.cpp | 2 +- 3 files changed, 29 insertions(+), 45 deletions(-) diff --git a/core/src/grpc/gen-milvus/milvus.pb.h b/core/src/grpc/gen-milvus/milvus.pb.h index 536365b23b..53ed2db22e 100644 --- a/core/src/grpc/gen-milvus/milvus.pb.h +++ b/core/src/grpc/gen-milvus/milvus.pb.h @@ -1321,8 +1321,7 @@ class RowRecord : void clear_vector_data(); float vector_data(int index) const; void set_vector_data(int index, float value); -// void add_vector_data(float value); - void add_vector_data(std::vector::const_iterator begin, std::vector::const_iterator end); + void add_vector_data(float value); const ::PROTOBUF_NAMESPACE_ID::RepeatedField< float >& vector_data() const; ::PROTOBUF_NAMESPACE_ID::RepeatedField< float >* @@ -1474,9 +1473,7 @@ class InsertParam : void clear_row_id_array(); ::PROTOBUF_NAMESPACE_ID::int64 row_id_array(int index) const; void set_row_id_array(int index, ::PROTOBUF_NAMESPACE_ID::int64 value); -// void add_row_id_array(::PROTOBUF_NAMESPACE_ID::int64 value); - void add_row_id_array(std::vector<::PROTOBUF_NAMESPACE_ID::int64>::const_iterator begin, - std::vector<::PROTOBUF_NAMESPACE_ID::int64>::const_iterator end); + void add_row_id_array(::PROTOBUF_NAMESPACE_ID::int64 value); const ::PROTOBUF_NAMESPACE_ID::RepeatedField< ::PROTOBUF_NAMESPACE_ID::int64 >& row_id_array() const; ::PROTOBUF_NAMESPACE_ID::RepeatedField< ::PROTOBUF_NAMESPACE_ID::int64 >* @@ -2139,9 +2136,7 @@ class TopKQueryResult : void clear_ids(); ::PROTOBUF_NAMESPACE_ID::int64 ids(int index) const; void set_ids(int index, ::PROTOBUF_NAMESPACE_ID::int64 value); -// void add_ids(::PROTOBUF_NAMESPACE_ID::int64 value); - void add_ids(std::vector<::PROTOBUF_NAMESPACE_ID::int64>::const_iterator begin, - std::vector<::PROTOBUF_NAMESPACE_ID::int64>::const_iterator end); + void add_ids(::PROTOBUF_NAMESPACE_ID::int64 value); const ::PROTOBUF_NAMESPACE_ID::RepeatedField< ::PROTOBUF_NAMESPACE_ID::int64 >& ids() const; ::PROTOBUF_NAMESPACE_ID::RepeatedField< ::PROTOBUF_NAMESPACE_ID::int64 >* @@ -2152,8 +2147,7 @@ class TopKQueryResult : void clear_distances(); float distances(int index) const; void set_distances(int index, float value); -// void add_distances(float value); - void add_distances(std::vector::const_iterator begin, std::vector::const_iterator end); + void add_distances(float value); const ::PROTOBUF_NAMESPACE_ID::RepeatedField< float >& distances() const; ::PROTOBUF_NAMESPACE_ID::RepeatedField< float >* @@ -3928,14 +3922,9 @@ inline void RowRecord::set_vector_data(int index, float value) { vector_data_.Set(index, value); // @@protoc_insertion_point(field_set:milvus.grpc.RowRecord.vector_data) } -//inline void RowRecord::add_vector_data(float value) { -// vector_data_.Add(value); -// // @@protoc_insertion_point(field_add:milvus.grpc.RowRecord.vector_data) -//} -inline void RowRecord::add_vector_data(std::vector::const_iterator begin, - std::vector::const_iterator end) { - vector_data_.Add(begin, end); - // @@protoc_insertion_point(field_add:milvus.grpc.RowRecord.vector_data) +inline void RowRecord::add_vector_data(float value) { + vector_data_.Add(value); + // @@protoc_insertion_point(field_add:milvus.grpc.RowRecord.vector_data) } inline const ::PROTOBUF_NAMESPACE_ID::RepeatedField< float >& RowRecord::vector_data() const { @@ -4048,14 +4037,9 @@ inline void InsertParam::set_row_id_array(int index, ::PROTOBUF_NAMESPACE_ID::in row_id_array_.Set(index, value); // @@protoc_insertion_point(field_set:milvus.grpc.InsertParam.row_id_array) } -//inline void InsertParam::add_row_id_array(::PROTOBUF_NAMESPACE_ID::int64 value) { -// row_id_array_.Add(value); -// // @@protoc_insertion_point(field_add:milvus.grpc.InsertParam.row_id_array) -//} -inline void InsertParam::add_row_id_array(std::vector<::PROTOBUF_NAMESPACE_ID::int64>::const_iterator begin, - std::vector<::PROTOBUF_NAMESPACE_ID::int64>::const_iterator end) { - row_id_array_.Add(begin, end); - // @@protoc_insertion_point(field_add:milvus.grpc.InsertParam.row_id_array) +inline void InsertParam::add_row_id_array(::PROTOBUF_NAMESPACE_ID::int64 value) { + row_id_array_.Add(value); + // @@protoc_insertion_point(field_add:milvus.grpc.InsertParam.row_id_array) } inline const ::PROTOBUF_NAMESPACE_ID::RepeatedField< ::PROTOBUF_NAMESPACE_ID::int64 >& InsertParam::row_id_array() const { @@ -4604,14 +4588,9 @@ inline void TopKQueryResult::set_ids(int index, ::PROTOBUF_NAMESPACE_ID::int64 v ids_.Set(index, value); // @@protoc_insertion_point(field_set:milvus.grpc.TopKQueryResult.ids) } -//inline void TopKQueryResult::add_ids(::PROTOBUF_NAMESPACE_ID::int64 value) { -// ids_.Add(value); -// // @@protoc_insertion_point(field_add:milvus.grpc.TopKQueryResult.ids) -//} -inline void TopKQueryResult::add_ids(std::vector<::PROTOBUF_NAMESPACE_ID::int64>::const_iterator begin, - std::vector<::PROTOBUF_NAMESPACE_ID::int64>::const_iterator end) { - ids_.Add(begin,end); - // @@protoc_insertion_point(field_add:milvus.grpc.TopKQueryResult.ids) +inline void TopKQueryResult::add_ids(::PROTOBUF_NAMESPACE_ID::int64 value) { + ids_.Add(value); + // @@protoc_insertion_point(field_add:milvus.grpc.TopKQueryResult.ids) } inline const ::PROTOBUF_NAMESPACE_ID::RepeatedField< ::PROTOBUF_NAMESPACE_ID::int64 >& TopKQueryResult::ids() const { @@ -4639,13 +4618,9 @@ inline void TopKQueryResult::set_distances(int index, float value) { distances_.Set(index, value); // @@protoc_insertion_point(field_set:milvus.grpc.TopKQueryResult.distances) } -//inline void TopKQueryResult::add_distances(float value) { -// distances_.Add(value); -// // @@protoc_insertion_point(field_add:milvus.grpc.TopKQueryResult.distances) -//} -inline void TopKQueryResult::add_distances(std::vector::const_iterator begin, std::vector::const_iterator end) { - distances_.Add(begin, end); - // @@protoc_insertion_point(field_add:milvus.grpc.TopKQueryResult.distances) +inline void TopKQueryResult::add_distances(float value) { + distances_.Add(value); + // @@protoc_insertion_point(field_add:milvus.grpc.TopKQueryResult.distances) } inline const ::PROTOBUF_NAMESPACE_ID::RepeatedField< float >& TopKQueryResult::distances() const { diff --git a/core/src/sdk/grpc/ClientProxy.cpp b/core/src/sdk/grpc/ClientProxy.cpp index 131d0bdc99..4a9c319b4d 100644 --- a/core/src/sdk/grpc/ClientProxy.cpp +++ b/core/src/sdk/grpc/ClientProxy.cpp @@ -32,6 +32,13 @@ UriCheck(const std::string& uri) { return (index != std::string::npos); } +void +CopyRowRecord(::milvus::grpc::RowRecord* target, const RowRecord& src) { + auto vector_data = target->mutable_vector_data(); + vector_data->Resize(static_cast(src.data.size()), 0.0); + memcpy(vector_data->mutable_data(), src.data.data(), src.data.size() * sizeof(float)); +} + Status ClientProxy::Connect(const ConnectParam& param) { std::string uri = param.ip_address + ":" + param.port; @@ -189,14 +196,16 @@ ClientProxy::Insert(const std::string& table_name, const std::string& partition_ for (auto& record : record_array) { ::milvus::grpc::RowRecord* grpc_record = insert_param.add_row_record_array(); - grpc_record->add_vector_data(record.data.begin(), record.data.end()); + CopyRowRecord(grpc_record, record); } // Single thread ::milvus::grpc::VectorIds vector_ids; if (!id_array.empty()) { /* set user's ids */ - insert_param.add_row_id_array(id_array.begin(), id_array.end()); + auto row_ids = insert_param.mutable_row_id_array(); + row_ids->Resize(static_cast(id_array.size()), -1); + memcpy(row_ids->mutable_data(), id_array.data(), id_array.size() * sizeof(int64_t)); client_ptr_->Insert(vector_ids, insert_param, status); } else { client_ptr_->Insert(vector_ids, insert_param, status); @@ -226,7 +235,7 @@ ClientProxy::Search(const std::string& table_name, const std::vectoradd_vector_data(record.data.begin(), record.data.end()); + CopyRowRecord(row_record, record); } // step 2: convert range array diff --git a/core/src/server/grpc_impl/request/SearchRequest.cpp b/core/src/server/grpc_impl/request/SearchRequest.cpp index 955576360a..28f4ff723e 100644 --- a/core/src/server/grpc_impl/request/SearchRequest.cpp +++ b/core/src/server/grpc_impl/request/SearchRequest.cpp @@ -167,7 +167,7 @@ SearchRequest::OnExecute() { // step 7: construct result array topk_result_->set_row_num(record_count); - topk_result_->mutable_ids()->Resize(static_cast(result_ids.size()), 0); + topk_result_->mutable_ids()->Resize(static_cast(result_ids.size()), -1); memcpy(topk_result_->mutable_ids()->mutable_data(), result_ids.data(), result_ids.size() * sizeof(int64_t)); topk_result_->mutable_distances()->Resize(static_cast(result_distances.size()), 0.0); memcpy(topk_result_->mutable_distances()->mutable_data(), result_distances.data(),