From 7812eed091952c37a384269cdb4e09ec4924fabb Mon Sep 17 00:00:00 2001 From: shengjh <1572099106@qq.com> Date: Tue, 22 Sep 2020 15:16:51 +0800 Subject: [PATCH] Add count row for proxy Signed-off-by: shengjh <1572099106@qq.com> --- conf/config.yaml | 2 +- proxy/src/server/MetaWrapper.cpp | 18 ++++++++++-- proxy/src/server/MetaWrapper.h | 3 ++ .../delivery/request/CountEntitiesReq.cpp | 3 +- sdk/examples/simple/CountCollection.cpp | 29 +++++++++++++++++++ sdk/examples/simple/CreateCollection.cpp | 2 +- sdk/examples/simple/CreateIndex.cpp | 2 +- sdk/examples/simple/delete.cpp | 2 +- 8 files changed, 53 insertions(+), 8 deletions(-) create mode 100644 sdk/examples/simple/CountCollection.cpp diff --git a/conf/config.yaml b/conf/config.yaml index 8684845959..505dbbb001 100644 --- a/conf/config.yaml +++ b/conf/config.yaml @@ -16,7 +16,7 @@ master: etcd: address: localhost port: 2379 - rootpath: by-dev/ + rootpath: by-dev segthreshold: 10000 timesync: diff --git a/proxy/src/server/MetaWrapper.cpp b/proxy/src/server/MetaWrapper.cpp index 515a058618..b570a5e671 100644 --- a/proxy/src/server/MetaWrapper.cpp +++ b/proxy/src/server/MetaWrapper.cpp @@ -3,6 +3,7 @@ #include "nlohmann/json.hpp" #include #include +#include using Collection = masterpb::Collection; using Schema = milvus::grpc::Schema; @@ -23,6 +24,7 @@ void ParseSegmentInfo(const std::string &json_str, SegmentInfo &segment_info) { segment_info.set_close_timestamp(json["close_timestamp"].get()); segment_info.set_collection_id(json["collection_id"].get()); segment_info.set_collection_name(json["collection_name"].get()); + segment_info.set_rows(json["rows"].get()); } void ParseCollectionSchema(const std::string &json_str, Collection &collection) { @@ -51,8 +53,8 @@ MetaWrapper &MetaWrapper::GetInstance() { Status MetaWrapper::Init() { try { etcd_root_path_ = config.etcd.rootpath(); - segment_path_ = etcd_root_path_ + "segment/"; - collection_path_ = etcd_root_path_ + "collection/"; + segment_path_ = (boost::filesystem::path(etcd_root_path_) / "segment/").string(); + collection_path_ = (boost::filesystem::path(etcd_root_path_) / "collection/").string(); auto master_addr = config.master.address() + ":" + std::to_string(config.master.port()); master_client_ = std::make_shared(master_addr); @@ -65,7 +67,6 @@ Status MetaWrapper::Init() { UpdateMeta(res); }; watcher_ = std::make_shared(etcd_addr, segment_path_, f, true); - SyncMeta(); } catch (const std::exception &e) { @@ -162,5 +163,16 @@ Status MetaWrapper::SyncMeta() { return status; } +int64_t MetaWrapper::CountCollection(const std::string &collection_name) { + uint64_t count = 0; + // TODO: index to speed up + for (const auto& segment_info : segment_infos_){ + if (segment_info.second.collection_name() == collection_name){ + count += segment_info.second.rows(); + } + } + return count; +} + } } \ No newline at end of file diff --git a/proxy/src/server/MetaWrapper.h b/proxy/src/server/MetaWrapper.h index ecd9667b1e..168820b0ff 100644 --- a/proxy/src/server/MetaWrapper.h +++ b/proxy/src/server/MetaWrapper.h @@ -31,6 +31,9 @@ class MetaWrapper { Status SyncMeta(); + int64_t + CountCollection(const std::string& collection_name); + private: bool IsCollectionMetaKey(const std::string &key); diff --git a/proxy/src/server/delivery/request/CountEntitiesReq.cpp b/proxy/src/server/delivery/request/CountEntitiesReq.cpp index 1023440009..a0ceb6990e 100644 --- a/proxy/src/server/delivery/request/CountEntitiesReq.cpp +++ b/proxy/src/server/delivery/request/CountEntitiesReq.cpp @@ -14,6 +14,7 @@ #include "server/ValidationUtil.h" #include "utils/Log.h" #include "utils/TimeRecorder.h" +#include "server/MetaWrapper.h" #include #include @@ -36,7 +37,7 @@ CountEntitiesReq::OnExecute() { try { std::string hdr = "CountEntitiesReq(collection=" + collection_name_ + ")"; TimeRecorderAuto rc(hdr); - + row_count_ = MetaWrapper::GetInstance().CountCollection(collection_name_); rc.ElapseFromBegin("done"); } catch (std::exception& ex) { diff --git a/sdk/examples/simple/CountCollection.cpp b/sdk/examples/simple/CountCollection.cpp new file mode 100644 index 0000000000..3c5a0d3beb --- /dev/null +++ b/sdk/examples/simple/CountCollection.cpp @@ -0,0 +1,29 @@ +#include +#include +#include +#include +#include "utils/Utils.h" + +int main(int argc , char**argv) { + + TestParameters parameters = milvus_sdk::Utils::ParseTestParameters(argc, argv); + if (!parameters.is_valid) { + return 0; + } + auto client = milvus::ConnectionImpl(); + milvus::ConnectParam connect_param; + connect_param.ip_address = parameters.address_.empty() ? "127.0.0.1" : parameters.address_; + connect_param.port = parameters.port_.empty() ? "19530" : parameters.port_; + client.Connect(connect_param); + + milvus::Status stat; + const std::string collectin_name = "collection1"; + + int64_t count = 0; + stat = client.CountEntities(collectin_name, count); + if (!stat.ok()){ + std::cerr << "Error: " << stat.message() << std::endl; + } + std::cout << "Collection " << collectin_name << " rows: " << count << std::endl; + +} \ No newline at end of file diff --git a/sdk/examples/simple/CreateCollection.cpp b/sdk/examples/simple/CreateCollection.cpp index b3c9e7a0d4..a3ec51b6d8 100644 --- a/sdk/examples/simple/CreateCollection.cpp +++ b/sdk/examples/simple/CreateCollection.cpp @@ -18,7 +18,7 @@ int main(int argc , char**argv) { client.Connect(connect_param); milvus::Status stat; - const std::string collectin_name = "collection0"; + const std::string collectin_name = "collection1"; // Create milvus::FieldPtr field_ptr1 = std::make_shared(); diff --git a/sdk/examples/simple/CreateIndex.cpp b/sdk/examples/simple/CreateIndex.cpp index 24a1dbb63e..7136526724 100644 --- a/sdk/examples/simple/CreateIndex.cpp +++ b/sdk/examples/simple/CreateIndex.cpp @@ -2,7 +2,7 @@ #include "interface/ConnectionImpl.h" #include "utils/Utils.h" -const std::string COLLECTION = "collection_0"; +const std::string COLLECTION = "collection1"; int main(int argc, char *argv[]) { TestParameters parameters = milvus_sdk::Utils::ParseTestParameters(argc, argv); diff --git a/sdk/examples/simple/delete.cpp b/sdk/examples/simple/delete.cpp index f582586069..125321450c 100644 --- a/sdk/examples/simple/delete.cpp +++ b/sdk/examples/simple/delete.cpp @@ -32,7 +32,7 @@ main(int argc, char *argv[]) { delete_ids.push_back(1); delete_ids.push_back(2); delete_ids.push_back(3); - client.DeleteEntityByID("collection0", delete_ids); + client.DeleteEntityByID("collection1", delete_ids); return 0; }