From ccec483e1e9e19d79f421c0a0a38a0b0ab91e78f Mon Sep 17 00:00:00 2001 From: Yu Kun Date: Wed, 14 Aug 2019 15:50:23 +0800 Subject: [PATCH 1/3] add preloadtable Former-commit-id: f052a555777fce2a1d27478f7b2bb51d1e536c7c --- cpp/src/db/DB.h | 1 + cpp/src/db/DBImpl.cpp | 31 +++++++++++++++++++++++++++ cpp/src/db/DBImpl.h | 3 +++ cpp/src/db/meta/Meta.h | 3 +++ cpp/src/db/meta/MySQLMetaImpl.cpp | 5 +++++ cpp/src/db/meta/MySQLMetaImpl.h | 1 + cpp/src/db/meta/SqliteMetaImpl.cpp | 34 ++++++++++++++++++++++++++++++ cpp/src/db/meta/SqliteMetaImpl.h | 3 +++ 8 files changed, 81 insertions(+) diff --git a/cpp/src/db/DB.h b/cpp/src/db/DB.h index f1d3a77a8a..0a84ffd7ff 100644 --- a/cpp/src/db/DB.h +++ b/cpp/src/db/DB.h @@ -28,6 +28,7 @@ public: virtual Status HasTable(const std::string& table_id, bool& has_or_not_) = 0; virtual Status AllTables(std::vector& table_schema_array) = 0; virtual Status GetTableRowCount(const std::string& table_id, uint64_t& row_count) = 0; + virtual Status PreloadTable(const std::string& table_id) = 0; virtual Status InsertVectors(const std::string& table_id_, uint64_t n, const float* vectors, IDNumbers& vector_ids_) = 0; diff --git a/cpp/src/db/DBImpl.cpp b/cpp/src/db/DBImpl.cpp index 636efc51bd..812438546c 100644 --- a/cpp/src/db/DBImpl.cpp +++ b/cpp/src/db/DBImpl.cpp @@ -32,6 +32,7 @@ namespace { constexpr uint64_t METRIC_ACTION_INTERVAL = 1; constexpr uint64_t COMPACT_ACTION_INTERVAL = 1; constexpr uint64_t INDEX_ACTION_INTERVAL = 1; +constexpr int64_t unit = 1024 * 1024 * 1024; void CollectInsertMetrics(double total_time, size_t n, bool succeed) { double avg_time = total_time / n; @@ -127,6 +128,36 @@ Status DBImpl::AllTables(std::vector& table_schema_array) { return meta_ptr_->AllTables(table_schema_array); } +Status DBImpl::PreloadTable(const std::string &table_id) { + meta::TableFilesSchema files; + auto status = meta_ptr_->PreloadTable(table_id, files); + + int64_t size = 0; + + server::ConfigNode& config = server::ServerConfig::GetInstance().GetConfig(server::CONFIG_CACHE); + int64_t cap = config.GetInt64Value(server::CONFIG_CPU_CACHE_CAPACITY, 16); + cap *= unit; + for(auto &file : files) { + ExecutionEnginePtr engine = EngineFactory::Build(file.dimension_, file.location_, (EngineType)file.engine_type_); + if(engine == nullptr) { + ENGINE_LOG_ERROR << "Invalid engine type"; + return Status::Error("Invalid engine type"); + } + + size += engine->PhysicalSize(); + if (size <= cap) { + try { + //step 1: load index + engine->Load(options_.insert_cache_immediately_); + } catch (std::exception &ex) { + std::string msg = "load to cache exception" + std::string(ex.what()); + ENGINE_LOG_ERROR << msg; + return Status::Error(msg); + } + } + } +} + Status DBImpl::GetTableRowCount(const std::string& table_id, uint64_t& row_count) { return meta_ptr_->Count(table_id, row_count); } diff --git a/cpp/src/db/DBImpl.h b/cpp/src/db/DBImpl.h index ccd7f541bd..969b7b0509 100644 --- a/cpp/src/db/DBImpl.h +++ b/cpp/src/db/DBImpl.h @@ -51,6 +51,9 @@ class DBImpl : public DB { Status AllTables(std::vector &table_schema_array) override; + Status + PreloadTable(const std::string &table_id) override; + Status GetTableRowCount(const std::string &table_id, uint64_t &row_count) override; diff --git a/cpp/src/db/meta/Meta.h b/cpp/src/db/meta/Meta.h index e88761b446..b6e867a819 100644 --- a/cpp/src/db/meta/Meta.h +++ b/cpp/src/db/meta/Meta.h @@ -38,6 +38,9 @@ class Meta { virtual Status AllTables(std::vector &table_schema_array) = 0; + virtual Status + PreloadTable(const std::string &table_id, meta::TableFilesSchema &files) = 0; + virtual Status DeleteTable(const std::string &table_id) = 0; diff --git a/cpp/src/db/meta/MySQLMetaImpl.cpp b/cpp/src/db/meta/MySQLMetaImpl.cpp index bf4589351e..800faefc7c 100644 --- a/cpp/src/db/meta/MySQLMetaImpl.cpp +++ b/cpp/src/db/meta/MySQLMetaImpl.cpp @@ -690,6 +690,11 @@ Status MySQLMetaImpl::AllTables(std::vector &table_schema_array) { return Status::OK(); } +Status +MySQLMetaImpl::PreloadTable(const std::string &table_id, meta::TableFilesSchema &files_schema) { + +} + Status MySQLMetaImpl::CreateTableFile(TableFileSchema &file_schema) { diff --git a/cpp/src/db/meta/MySQLMetaImpl.h b/cpp/src/db/meta/MySQLMetaImpl.h index 30695423dd..f673678aa3 100644 --- a/cpp/src/db/meta/MySQLMetaImpl.h +++ b/cpp/src/db/meta/MySQLMetaImpl.h @@ -29,6 +29,7 @@ class MySQLMetaImpl : public Meta { Status DescribeTable(TableSchema &group_info_) override; Status HasTable(const std::string &table_id, bool &has_or_not) override; Status AllTables(std::vector &table_schema_array) override; + Status PreloadTable(const std::string &table_id, meta::TableFilesSchema &files_schema) override; Status DeleteTable(const std::string &table_id) override; Status DeleteTableFiles(const std::string &table_id) override; diff --git a/cpp/src/db/meta/SqliteMetaImpl.cpp b/cpp/src/db/meta/SqliteMetaImpl.cpp index 9411a5e82b..738c31132b 100644 --- a/cpp/src/db/meta/SqliteMetaImpl.cpp +++ b/cpp/src/db/meta/SqliteMetaImpl.cpp @@ -404,6 +404,40 @@ Status SqliteMetaImpl::AllTables(std::vector& table_schema_array) { return Status::OK(); } +Status SqliteMetaImpl::PreloadTable(const std::string &table_id, TableFilesSchema &files_schema) { + try { + MetricCollector metric; + + auto selected = ConnectorPtr->select(columns(&TableFileSchema::size_, + &TableFileSchema::dimension_, + &TableFileSchema::location_, + &TableFileSchema::engine_type_), + where(c(&TableFileSchema::table_id_) == table_id)); + + TableSchema table_schema; + table_schema.table_id_ = table_id; + auto status = DescribeTable(table_schema); + if (!status.ok()) { + return status; + } + + int64_t size = 0; + for (auto &file : selected) { + TableFileSchema file_schema; + file_schema.size_ = std::get<0>(file); + file_schema.dimension_ = std::get<1>(file); + file_schema.location_ = std::get<2>(file); + file_schema.engine_type_ = std::get<3>(file); + files_schema.push_back(file_schema); + } + + + } catch (std::exception &e) { + return HandleException("Encounter exception when lookup all tables", e); + } + return Status::OK(); +} + Status SqliteMetaImpl::CreateTableFile(TableFileSchema &file_schema) { if (file_schema.date_ == EmptyDate) { file_schema.date_ = Meta::GetDate(); diff --git a/cpp/src/db/meta/SqliteMetaImpl.h b/cpp/src/db/meta/SqliteMetaImpl.h index 1525f27e4b..68e483b76f 100644 --- a/cpp/src/db/meta/SqliteMetaImpl.h +++ b/cpp/src/db/meta/SqliteMetaImpl.h @@ -33,6 +33,9 @@ class SqliteMetaImpl : public Meta { Status AllTables(std::vector &table_schema_array) override; + Status + PreloadTable(const std::string &table_id, meta::TableFilesSchema &files_schema) override; + Status DeleteTable(const std::string &table_id) override; From d7936a6c600c0ab01be5cd805fdda199a4be552e Mon Sep 17 00:00:00 2001 From: Yu Kun Date: Wed, 14 Aug 2019 19:06:38 +0800 Subject: [PATCH 2/3] add some parameters and merge upstream Former-commit-id: 680349208d62108ceda4ad822c466107a840327c --- cpp/src/db/DBImpl.cpp | 48 +-- cpp/src/db/meta/Meta.h | 3 - cpp/src/db/meta/MySQLMetaImpl.cpp | 5 - cpp/src/db/meta/MySQLMetaImpl.h | 1 - cpp/src/db/meta/SqliteMetaImpl.cpp | 34 -- cpp/src/db/meta/SqliteMetaImpl.h | 3 - cpp/src/grpc/gen-milvus/milvus.pb.cc | 309 +++++++++++++----- cpp/src/grpc/gen-milvus/milvus.pb.h | 86 +++++ cpp/src/grpc/milvus.proto | 3 + cpp/src/sdk/grpc/ClientProxy.cpp | 9 +- cpp/src/sdk/grpc/GrpcClient.cpp | 20 +- .../server/grpc_impl/GrpcRequestHandler.cpp | 7 +- cpp/src/server/grpc_impl/GrpcRequestTask.cpp | 39 +++ 13 files changed, 424 insertions(+), 143 deletions(-) diff --git a/cpp/src/db/DBImpl.cpp b/cpp/src/db/DBImpl.cpp index 812438546c..0275fcb31b 100644 --- a/cpp/src/db/DBImpl.cpp +++ b/cpp/src/db/DBImpl.cpp @@ -129,33 +129,41 @@ Status DBImpl::AllTables(std::vector& table_schema_array) { } Status DBImpl::PreloadTable(const std::string &table_id) { - meta::TableFilesSchema files; - auto status = meta_ptr_->PreloadTable(table_id, files); + meta::DatePartionedTableFilesSchema files; + + meta::DatesT dates; + auto status = meta_ptr_->FilesToSearch(table_id, dates, files); + if (!status.ok()) { + return status; + } int64_t size = 0; + int64_t cache_total = cache::CpuCacheMgr::GetInstance()->CacheCapacity(); - server::ConfigNode& config = server::ServerConfig::GetInstance().GetConfig(server::CONFIG_CACHE); - int64_t cap = config.GetInt64Value(server::CONFIG_CPU_CACHE_CAPACITY, 16); - cap *= unit; - for(auto &file : files) { - ExecutionEnginePtr engine = EngineFactory::Build(file.dimension_, file.location_, (EngineType)file.engine_type_); - if(engine == nullptr) { - ENGINE_LOG_ERROR << "Invalid engine type"; - return Status::Error("Invalid engine type"); - } + for(auto &day_files : files) { + for (auto &file : day_files.second) { + ExecutionEnginePtr engine = EngineFactory::Build(file.dimension_, file.location_, (EngineType)file.engine_type_); + if(engine == nullptr) { + ENGINE_LOG_ERROR << "Invalid engine type"; + return Status::Error("Invalid engine type"); + } - size += engine->PhysicalSize(); - if (size <= cap) { - try { - //step 1: load index - engine->Load(options_.insert_cache_immediately_); - } catch (std::exception &ex) { - std::string msg = "load to cache exception" + std::string(ex.what()); - ENGINE_LOG_ERROR << msg; - return Status::Error(msg); + size += engine->PhysicalSize(); + if (size > cache_total) { + break; + } else { + try { + //step 1: load index + engine->Load(options_.insert_cache_immediately_); + } catch (std::exception &ex) { + std::string msg = "load to cache exception" + std::string(ex.what()); + ENGINE_LOG_ERROR << msg; + return Status::Error(msg); + } } } } + return Status::OK(); } Status DBImpl::GetTableRowCount(const std::string& table_id, uint64_t& row_count) { diff --git a/cpp/src/db/meta/Meta.h b/cpp/src/db/meta/Meta.h index b6e867a819..e88761b446 100644 --- a/cpp/src/db/meta/Meta.h +++ b/cpp/src/db/meta/Meta.h @@ -38,9 +38,6 @@ class Meta { virtual Status AllTables(std::vector &table_schema_array) = 0; - virtual Status - PreloadTable(const std::string &table_id, meta::TableFilesSchema &files) = 0; - virtual Status DeleteTable(const std::string &table_id) = 0; diff --git a/cpp/src/db/meta/MySQLMetaImpl.cpp b/cpp/src/db/meta/MySQLMetaImpl.cpp index 800faefc7c..bf4589351e 100644 --- a/cpp/src/db/meta/MySQLMetaImpl.cpp +++ b/cpp/src/db/meta/MySQLMetaImpl.cpp @@ -690,11 +690,6 @@ Status MySQLMetaImpl::AllTables(std::vector &table_schema_array) { return Status::OK(); } -Status -MySQLMetaImpl::PreloadTable(const std::string &table_id, meta::TableFilesSchema &files_schema) { - -} - Status MySQLMetaImpl::CreateTableFile(TableFileSchema &file_schema) { diff --git a/cpp/src/db/meta/MySQLMetaImpl.h b/cpp/src/db/meta/MySQLMetaImpl.h index f673678aa3..30695423dd 100644 --- a/cpp/src/db/meta/MySQLMetaImpl.h +++ b/cpp/src/db/meta/MySQLMetaImpl.h @@ -29,7 +29,6 @@ class MySQLMetaImpl : public Meta { Status DescribeTable(TableSchema &group_info_) override; Status HasTable(const std::string &table_id, bool &has_or_not) override; Status AllTables(std::vector &table_schema_array) override; - Status PreloadTable(const std::string &table_id, meta::TableFilesSchema &files_schema) override; Status DeleteTable(const std::string &table_id) override; Status DeleteTableFiles(const std::string &table_id) override; diff --git a/cpp/src/db/meta/SqliteMetaImpl.cpp b/cpp/src/db/meta/SqliteMetaImpl.cpp index 738c31132b..9411a5e82b 100644 --- a/cpp/src/db/meta/SqliteMetaImpl.cpp +++ b/cpp/src/db/meta/SqliteMetaImpl.cpp @@ -404,40 +404,6 @@ Status SqliteMetaImpl::AllTables(std::vector& table_schema_array) { return Status::OK(); } -Status SqliteMetaImpl::PreloadTable(const std::string &table_id, TableFilesSchema &files_schema) { - try { - MetricCollector metric; - - auto selected = ConnectorPtr->select(columns(&TableFileSchema::size_, - &TableFileSchema::dimension_, - &TableFileSchema::location_, - &TableFileSchema::engine_type_), - where(c(&TableFileSchema::table_id_) == table_id)); - - TableSchema table_schema; - table_schema.table_id_ = table_id; - auto status = DescribeTable(table_schema); - if (!status.ok()) { - return status; - } - - int64_t size = 0; - for (auto &file : selected) { - TableFileSchema file_schema; - file_schema.size_ = std::get<0>(file); - file_schema.dimension_ = std::get<1>(file); - file_schema.location_ = std::get<2>(file); - file_schema.engine_type_ = std::get<3>(file); - files_schema.push_back(file_schema); - } - - - } catch (std::exception &e) { - return HandleException("Encounter exception when lookup all tables", e); - } - return Status::OK(); -} - Status SqliteMetaImpl::CreateTableFile(TableFileSchema &file_schema) { if (file_schema.date_ == EmptyDate) { file_schema.date_ = Meta::GetDate(); diff --git a/cpp/src/db/meta/SqliteMetaImpl.h b/cpp/src/db/meta/SqliteMetaImpl.h index 68e483b76f..1525f27e4b 100644 --- a/cpp/src/db/meta/SqliteMetaImpl.h +++ b/cpp/src/db/meta/SqliteMetaImpl.h @@ -33,9 +33,6 @@ class SqliteMetaImpl : public Meta { Status AllTables(std::vector &table_schema_array) override; - Status - PreloadTable(const std::string &table_id, meta::TableFilesSchema &files_schema) override; - Status DeleteTable(const std::string &table_id) override; diff --git a/cpp/src/grpc/gen-milvus/milvus.pb.cc b/cpp/src/grpc/gen-milvus/milvus.pb.cc index 9d3cb9604e..5ec8fddba0 100644 --- a/cpp/src/grpc/gen-milvus/milvus.pb.cc +++ b/cpp/src/grpc/gen-milvus/milvus.pb.cc @@ -388,6 +388,7 @@ const ::PROTOBUF_NAMESPACE_ID::uint32 TableStruct_milvus_2eproto::offsets[] PROT ~0u, // no _weak_field_map_ PROTOBUF_FIELD_OFFSET(::milvus::grpc::InsertParam, table_name_), PROTOBUF_FIELD_OFFSET(::milvus::grpc::InsertParam, row_record_array_), + PROTOBUF_FIELD_OFFSET(::milvus::grpc::InsertParam, row_id_array_), ~0u, // no _has_bits_ PROTOBUF_FIELD_OFFSET(::milvus::grpc::VectorIds, _internal_metadata_), ~0u, // no _extensions_ @@ -404,6 +405,7 @@ const ::PROTOBUF_NAMESPACE_ID::uint32 TableStruct_milvus_2eproto::offsets[] PROT PROTOBUF_FIELD_OFFSET(::milvus::grpc::SearchParam, query_record_array_), PROTOBUF_FIELD_OFFSET(::milvus::grpc::SearchParam, query_range_array_), PROTOBUF_FIELD_OFFSET(::milvus::grpc::SearchParam, topk_), + PROTOBUF_FIELD_OFFSET(::milvus::grpc::SearchParam, nprobe_), ~0u, // no _has_bits_ PROTOBUF_FIELD_OFFSET(::milvus::grpc::SearchInFilesParam, _internal_metadata_), ~0u, // no _extensions_ @@ -460,6 +462,7 @@ const ::PROTOBUF_NAMESPACE_ID::uint32 TableStruct_milvus_2eproto::offsets[] PROT PROTOBUF_FIELD_OFFSET(::milvus::grpc::Index, index_type_), PROTOBUF_FIELD_OFFSET(::milvus::grpc::Index, nlist_), PROTOBUF_FIELD_OFFSET(::milvus::grpc::Index, index_file_size_), + PROTOBUF_FIELD_OFFSET(::milvus::grpc::Index, metric_type_), ~0u, // no _has_bits_ PROTOBUF_FIELD_OFFSET(::milvus::grpc::IndexParam, _internal_metadata_), ~0u, // no _extensions_ @@ -481,18 +484,18 @@ static const ::PROTOBUF_NAMESPACE_ID::internal::MigrationSchema schemas[] PROTOB { 16, -1, sizeof(::milvus::grpc::Range)}, { 23, -1, sizeof(::milvus::grpc::RowRecord)}, { 29, -1, sizeof(::milvus::grpc::InsertParam)}, - { 36, -1, sizeof(::milvus::grpc::VectorIds)}, - { 43, -1, sizeof(::milvus::grpc::SearchParam)}, - { 52, -1, sizeof(::milvus::grpc::SearchInFilesParam)}, - { 59, -1, sizeof(::milvus::grpc::QueryResult)}, - { 66, -1, sizeof(::milvus::grpc::TopKQueryResult)}, - { 73, -1, sizeof(::milvus::grpc::StringReply)}, - { 80, -1, sizeof(::milvus::grpc::BoolReply)}, - { 87, -1, sizeof(::milvus::grpc::TableRowCount)}, - { 94, -1, sizeof(::milvus::grpc::Command)}, - { 100, -1, sizeof(::milvus::grpc::Index)}, - { 108, -1, sizeof(::milvus::grpc::IndexParam)}, - { 115, -1, sizeof(::milvus::grpc::DeleteByRangeParam)}, + { 37, -1, sizeof(::milvus::grpc::VectorIds)}, + { 44, -1, sizeof(::milvus::grpc::SearchParam)}, + { 54, -1, sizeof(::milvus::grpc::SearchInFilesParam)}, + { 61, -1, sizeof(::milvus::grpc::QueryResult)}, + { 68, -1, sizeof(::milvus::grpc::TopKQueryResult)}, + { 75, -1, sizeof(::milvus::grpc::StringReply)}, + { 82, -1, sizeof(::milvus::grpc::BoolReply)}, + { 89, -1, sizeof(::milvus::grpc::TableRowCount)}, + { 96, -1, sizeof(::milvus::grpc::Command)}, + { 102, -1, sizeof(::milvus::grpc::Index)}, + { 111, -1, sizeof(::milvus::grpc::IndexParam)}, + { 118, -1, sizeof(::milvus::grpc::DeleteByRangeParam)}, }; static ::PROTOBUF_NAMESPACE_ID::Message const * const file_default_instances[] = { @@ -523,59 +526,60 @@ const char descriptor_table_protodef_milvus_2eproto[] PROTOBUF_SECTION_VARIABLE( ".TableName\022\022\n\nindex_type\030\002 \001(\005\022\021\n\tdimens" "ion\030\003 \001(\003\022\030\n\020store_raw_vector\030\004 \001(\010\"/\n\005R" "ange\022\023\n\013start_value\030\001 \001(\t\022\021\n\tend_value\030\002" - " \001(\t\" \n\tRowRecord\022\023\n\013vector_data\030\001 \003(\002\"S" + " \001(\t\" \n\tRowRecord\022\023\n\013vector_data\030\001 \003(\002\"i" "\n\013InsertParam\022\022\n\ntable_name\030\001 \001(\t\0220\n\020row" "_record_array\030\002 \003(\0132\026.milvus.grpc.RowRec" - "ord\"I\n\tVectorIds\022#\n\006status\030\001 \001(\0132\023.milvu" - "s.grpc.Status\022\027\n\017vector_id_array\030\002 \003(\003\"\222" - "\001\n\013SearchParam\022\022\n\ntable_name\030\001 \001(\t\0222\n\022qu" - "ery_record_array\030\002 \003(\0132\026.milvus.grpc.Row" - "Record\022-\n\021query_range_array\030\003 \003(\0132\022.milv" - "us.grpc.Range\022\014\n\004topk\030\004 \001(\003\"[\n\022SearchInF" - "ilesParam\022\025\n\rfile_id_array\030\001 \003(\t\022.\n\014sear" - "ch_param\030\002 \001(\0132\030.milvus.grpc.SearchParam" - "\"+\n\013QueryResult\022\n\n\002id\030\001 \001(\003\022\020\n\010distance\030" - "\002 \001(\001\"m\n\017TopKQueryResult\022#\n\006status\030\001 \001(\013" - "2\023.milvus.grpc.Status\0225\n\023query_result_ar" - "rays\030\002 \003(\0132\030.milvus.grpc.QueryResult\"H\n\013" - "StringReply\022#\n\006status\030\001 \001(\0132\023.milvus.grp" - "c.Status\022\024\n\014string_reply\030\002 \001(\t\"D\n\tBoolRe" - "ply\022#\n\006status\030\001 \001(\0132\023.milvus.grpc.Status" - "\022\022\n\nbool_reply\030\002 \001(\010\"M\n\rTableRowCount\022#\n" - "\006status\030\001 \001(\0132\023.milvus.grpc.Status\022\027\n\017ta" - "ble_row_count\030\002 \001(\003\"\026\n\007Command\022\013\n\003cmd\030\001 " - "\001(\t\"C\n\005Index\022\022\n\nindex_type\030\001 \001(\005\022\r\n\005nlis" - "t\030\002 \001(\003\022\027\n\017index_file_size\030\003 \001(\005\"[\n\nInde" - "xParam\022*\n\ntable_name\030\001 \001(\0132\026.milvus.grpc" - ".TableName\022!\n\005index\030\002 \001(\0132\022.milvus.grpc." - "Index\"K\n\022DeleteByRangeParam\022!\n\005range\030\001 \001" - "(\0132\022.milvus.grpc.Range\022\022\n\ntable_name\030\002 \001" - "(\t2\352\007\n\rMilvusService\022>\n\013CreateTable\022\030.mi" - "lvus.grpc.TableSchema\032\023.milvus.grpc.Stat" - "us\"\000\022<\n\010HasTable\022\026.milvus.grpc.TableName" - "\032\026.milvus.grpc.BoolReply\"\000\022:\n\tDropTable\022" - "\026.milvus.grpc.TableName\032\023.milvus.grpc.St" - "atus\"\000\022=\n\013CreateIndex\022\027.milvus.grpc.Inde" - "xParam\032\023.milvus.grpc.Status\"\000\022<\n\006Insert\022" - "\030.milvus.grpc.InsertParam\032\026.milvus.grpc." - "VectorIds\"\000\022D\n\006Search\022\030.milvus.grpc.Sear" - "chParam\032\034.milvus.grpc.TopKQueryResult\"\0000" - "\001\022R\n\rSearchInFiles\022\037.milvus.grpc.SearchI" - "nFilesParam\032\034.milvus.grpc.TopKQueryResul" - "t\"\0000\001\022C\n\rDescribeTable\022\026.milvus.grpc.Tab" - "leName\032\030.milvus.grpc.TableSchema\"\000\022B\n\nCo" - "untTable\022\026.milvus.grpc.TableName\032\032.milvu" - "s.grpc.TableRowCount\"\000\022>\n\nShowTables\022\024.m" - "ilvus.grpc.Command\032\026.milvus.grpc.TableNa" - "me\"\0000\001\0227\n\003Cmd\022\024.milvus.grpc.Command\032\030.mi" - "lvus.grpc.StringReply\"\000\022G\n\rDeleteByRange" - "\022\037.milvus.grpc.DeleteByRangeParam\032\023.milv" - "us.grpc.Status\"\000\022=\n\014PreloadTable\022\026.milvu" - "s.grpc.TableName\032\023.milvus.grpc.Status\"\000\022" - "B\n\rDescribeIndex\022\026.milvus.grpc.TableName" - "\032\027.milvus.grpc.IndexParam\"\000\022:\n\tDropIndex" - "\022\026.milvus.grpc.TableName\032\023.milvus.grpc.S" - "tatus\"\000b\006proto3" + "ord\022\024\n\014row_id_array\030\003 \003(\003\"I\n\tVectorIds\022#" + "\n\006status\030\001 \001(\0132\023.milvus.grpc.Status\022\027\n\017v" + "ector_id_array\030\002 \003(\003\"\242\001\n\013SearchParam\022\022\n\n" + "table_name\030\001 \001(\t\0222\n\022query_record_array\030\002" + " \003(\0132\026.milvus.grpc.RowRecord\022-\n\021query_ra" + "nge_array\030\003 \003(\0132\022.milvus.grpc.Range\022\014\n\004t" + "opk\030\004 \001(\003\022\016\n\006nprobe\030\005 \001(\003\"[\n\022SearchInFil" + "esParam\022\025\n\rfile_id_array\030\001 \003(\t\022.\n\014search" + "_param\030\002 \001(\0132\030.milvus.grpc.SearchParam\"+" + "\n\013QueryResult\022\n\n\002id\030\001 \001(\003\022\020\n\010distance\030\002 " + "\001(\001\"m\n\017TopKQueryResult\022#\n\006status\030\001 \001(\0132\023" + ".milvus.grpc.Status\0225\n\023query_result_arra" + "ys\030\002 \003(\0132\030.milvus.grpc.QueryResult\"H\n\013St" + "ringReply\022#\n\006status\030\001 \001(\0132\023.milvus.grpc." + "Status\022\024\n\014string_reply\030\002 \001(\t\"D\n\tBoolRepl" + "y\022#\n\006status\030\001 \001(\0132\023.milvus.grpc.Status\022\022" + "\n\nbool_reply\030\002 \001(\010\"M\n\rTableRowCount\022#\n\006s" + "tatus\030\001 \001(\0132\023.milvus.grpc.Status\022\027\n\017tabl" + "e_row_count\030\002 \001(\003\"\026\n\007Command\022\013\n\003cmd\030\001 \001(" + "\t\"X\n\005Index\022\022\n\nindex_type\030\001 \001(\005\022\r\n\005nlist\030" + "\002 \001(\003\022\027\n\017index_file_size\030\003 \001(\005\022\023\n\013metric" + "_type\030\004 \001(\005\"[\n\nIndexParam\022*\n\ntable_name\030" + "\001 \001(\0132\026.milvus.grpc.TableName\022!\n\005index\030\002" + " \001(\0132\022.milvus.grpc.Index\"K\n\022DeleteByRang" + "eParam\022!\n\005range\030\001 \001(\0132\022.milvus.grpc.Rang" + "e\022\022\n\ntable_name\030\002 \001(\t2\352\007\n\rMilvusService\022" + ">\n\013CreateTable\022\030.milvus.grpc.TableSchema" + "\032\023.milvus.grpc.Status\"\000\022<\n\010HasTable\022\026.mi" + "lvus.grpc.TableName\032\026.milvus.grpc.BoolRe" + "ply\"\000\022:\n\tDropTable\022\026.milvus.grpc.TableNa" + "me\032\023.milvus.grpc.Status\"\000\022=\n\013CreateIndex" + "\022\027.milvus.grpc.IndexParam\032\023.milvus.grpc." + "Status\"\000\022<\n\006Insert\022\030.milvus.grpc.InsertP" + "aram\032\026.milvus.grpc.VectorIds\"\000\022D\n\006Search" + "\022\030.milvus.grpc.SearchParam\032\034.milvus.grpc" + ".TopKQueryResult\"\0000\001\022R\n\rSearchInFiles\022\037." + "milvus.grpc.SearchInFilesParam\032\034.milvus." + "grpc.TopKQueryResult\"\0000\001\022C\n\rDescribeTabl" + "e\022\026.milvus.grpc.TableName\032\030.milvus.grpc." + "TableSchema\"\000\022B\n\nCountTable\022\026.milvus.grp" + "c.TableName\032\032.milvus.grpc.TableRowCount\"" + "\000\022>\n\nShowTables\022\024.milvus.grpc.Command\032\026." + "milvus.grpc.TableName\"\0000\001\0227\n\003Cmd\022\024.milvu" + "s.grpc.Command\032\030.milvus.grpc.StringReply" + "\"\000\022G\n\rDeleteByRange\022\037.milvus.grpc.Delete" + "ByRangeParam\032\023.milvus.grpc.Status\"\000\022=\n\014P" + "reloadTable\022\026.milvus.grpc.TableName\032\023.mi" + "lvus.grpc.Status\"\000\022B\n\rDescribeIndex\022\026.mi" + "lvus.grpc.TableName\032\027.milvus.grpc.IndexP" + "aram\"\000\022:\n\tDropIndex\022\026.milvus.grpc.TableN" + "ame\032\023.milvus.grpc.Status\"\000b\006proto3" ; static const ::PROTOBUF_NAMESPACE_ID::internal::DescriptorTable*const descriptor_table_milvus_2eproto_deps[1] = { &::descriptor_table_status_2eproto, @@ -602,7 +606,7 @@ static ::PROTOBUF_NAMESPACE_ID::internal::SCCInfoBase*const descriptor_table_mil static ::PROTOBUF_NAMESPACE_ID::internal::once_flag descriptor_table_milvus_2eproto_once; static bool descriptor_table_milvus_2eproto_initialized = false; const ::PROTOBUF_NAMESPACE_ID::internal::DescriptorTable descriptor_table_milvus_2eproto = { - &descriptor_table_milvus_2eproto_initialized, descriptor_table_protodef_milvus_2eproto, "milvus.proto", 2375, + &descriptor_table_milvus_2eproto_initialized, descriptor_table_protodef_milvus_2eproto, "milvus.proto", 2434, &descriptor_table_milvus_2eproto_once, descriptor_table_milvus_2eproto_sccs, descriptor_table_milvus_2eproto_deps, 17, 1, schemas, file_default_instances, TableStruct_milvus_2eproto::offsets, file_level_metadata_milvus_2eproto, 17, file_level_enum_descriptors_milvus_2eproto, file_level_service_descriptors_milvus_2eproto, @@ -1963,7 +1967,8 @@ InsertParam::InsertParam() InsertParam::InsertParam(const InsertParam& from) : ::PROTOBUF_NAMESPACE_ID::Message(), _internal_metadata_(nullptr), - row_record_array_(from.row_record_array_) { + row_record_array_(from.row_record_array_), + row_id_array_(from.row_id_array_) { _internal_metadata_.MergeFrom(from._internal_metadata_); table_name_.UnsafeSetDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); if (!from.table_name().empty()) { @@ -2002,6 +2007,7 @@ void InsertParam::Clear() { (void) cached_has_bits; row_record_array_.Clear(); + row_id_array_.Clear(); table_name_.ClearToEmptyNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); _internal_metadata_.Clear(); } @@ -2033,6 +2039,16 @@ const char* InsertParam::_InternalParse(const char* ptr, ::PROTOBUF_NAMESPACE_ID } while (::PROTOBUF_NAMESPACE_ID::internal::UnalignedLoad<::PROTOBUF_NAMESPACE_ID::uint8>(ptr) == 18); } else goto handle_unusual; continue; + // repeated int64 row_id_array = 3; + case 3: + if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 26)) { + ptr = ::PROTOBUF_NAMESPACE_ID::internal::PackedInt64Parser(mutable_row_id_array(), ptr, ctx); + CHK_(ptr); + } else if (static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 24) { + add_row_id_array(::PROTOBUF_NAMESPACE_ID::internal::ReadVarint(&ptr)); + CHK_(ptr); + } else goto handle_unusual; + continue; default: { handle_unusual: if ((tag & 7) == 4 || tag == 0) { @@ -2089,6 +2105,22 @@ bool InsertParam::MergePartialFromCodedStream( break; } + // repeated int64 row_id_array = 3; + case 3: { + if (static_cast< ::PROTOBUF_NAMESPACE_ID::uint8>(tag) == (26 & 0xFF)) { + DO_((::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::ReadPackedPrimitive< + ::PROTOBUF_NAMESPACE_ID::int64, ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::TYPE_INT64>( + input, this->mutable_row_id_array()))); + } else if (static_cast< ::PROTOBUF_NAMESPACE_ID::uint8>(tag) == (24 & 0xFF)) { + DO_((::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::ReadRepeatedPrimitiveNoInline< + ::PROTOBUF_NAMESPACE_ID::int64, ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::TYPE_INT64>( + 1, 26u, input, this->mutable_row_id_array()))); + } else { + goto handle_unusual; + } + break; + } + default: { handle_unusual: if (tag == 0) { @@ -2135,6 +2167,17 @@ void InsertParam::SerializeWithCachedSizes( output); } + // repeated int64 row_id_array = 3; + if (this->row_id_array_size() > 0) { + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteTag(3, ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WIRETYPE_LENGTH_DELIMITED, output); + output->WriteVarint32(_row_id_array_cached_byte_size_.load( + std::memory_order_relaxed)); + } + for (int i = 0, n = this->row_id_array_size(); i < n; i++) { + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt64NoTag( + this->row_id_array(i), output); + } + if (_internal_metadata_.have_unknown_fields()) { ::PROTOBUF_NAMESPACE_ID::internal::WireFormat::SerializeUnknownFields( _internal_metadata_.unknown_fields(), output); @@ -2167,6 +2210,19 @@ void InsertParam::SerializeWithCachedSizes( 2, this->row_record_array(static_cast(i)), target); } + // repeated int64 row_id_array = 3; + if (this->row_id_array_size() > 0) { + target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteTagToArray( + 3, + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WIRETYPE_LENGTH_DELIMITED, + target); + target = ::PROTOBUF_NAMESPACE_ID::io::CodedOutputStream::WriteVarint32ToArray( + _row_id_array_cached_byte_size_.load(std::memory_order_relaxed), + target); + target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite:: + WriteInt64NoTagToArray(this->row_id_array_, target); + } + if (_internal_metadata_.have_unknown_fields()) { target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormat::SerializeUnknownFieldsToArray( _internal_metadata_.unknown_fields(), target); @@ -2199,6 +2255,21 @@ size_t InsertParam::ByteSizeLong() const { } } + // repeated int64 row_id_array = 3; + { + size_t data_size = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite:: + Int64Size(this->row_id_array_); + if (data_size > 0) { + total_size += 1 + + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::Int32Size( + static_cast<::PROTOBUF_NAMESPACE_ID::int32>(data_size)); + } + int cached_size = ::PROTOBUF_NAMESPACE_ID::internal::ToCachedSize(data_size); + _row_id_array_cached_byte_size_.store(cached_size, + std::memory_order_relaxed); + total_size += data_size; + } + // string table_name = 1; if (this->table_name().size() > 0) { total_size += 1 + @@ -2234,6 +2305,7 @@ void InsertParam::MergeFrom(const InsertParam& from) { (void) cached_has_bits; row_record_array_.MergeFrom(from.row_record_array_); + row_id_array_.MergeFrom(from.row_id_array_); if (from.table_name().size() > 0) { table_name_.AssignWithDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), from.table_name_); @@ -2262,6 +2334,7 @@ void InsertParam::InternalSwap(InsertParam* other) { using std::swap; _internal_metadata_.Swap(&other->_internal_metadata_); CastToBase(&row_record_array_)->InternalSwap(CastToBase(&other->row_record_array_)); + row_id_array_.InternalSwap(&other->row_id_array_); table_name_.Swap(&other->table_name_, &::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), GetArenaNoVirtual()); } @@ -2635,14 +2708,18 @@ SearchParam::SearchParam(const SearchParam& from) if (!from.table_name().empty()) { table_name_.AssignWithDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), from.table_name_); } - topk_ = from.topk_; + ::memcpy(&topk_, &from.topk_, + static_cast(reinterpret_cast(&nprobe_) - + reinterpret_cast(&topk_)) + sizeof(nprobe_)); // @@protoc_insertion_point(copy_constructor:milvus.grpc.SearchParam) } void SearchParam::SharedCtor() { ::PROTOBUF_NAMESPACE_ID::internal::InitSCC(&scc_info_SearchParam_milvus_2eproto.base); table_name_.UnsafeSetDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); - topk_ = PROTOBUF_LONGLONG(0); + ::memset(&topk_, 0, static_cast( + reinterpret_cast(&nprobe_) - + reinterpret_cast(&topk_)) + sizeof(nprobe_)); } SearchParam::~SearchParam() { @@ -2672,7 +2749,9 @@ void SearchParam::Clear() { query_record_array_.Clear(); query_range_array_.Clear(); table_name_.ClearToEmptyNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); - topk_ = PROTOBUF_LONGLONG(0); + ::memset(&topk_, 0, static_cast( + reinterpret_cast(&nprobe_) - + reinterpret_cast(&topk_)) + sizeof(nprobe_)); _internal_metadata_.Clear(); } @@ -2722,6 +2801,13 @@ const char* SearchParam::_InternalParse(const char* ptr, ::PROTOBUF_NAMESPACE_ID CHK_(ptr); } else goto handle_unusual; continue; + // int64 nprobe = 5; + case 5: + if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 40)) { + nprobe_ = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint(&ptr); + CHK_(ptr); + } else goto handle_unusual; + continue; default: { handle_unusual: if ((tag & 7) == 4 || tag == 0) { @@ -2802,6 +2888,19 @@ bool SearchParam::MergePartialFromCodedStream( break; } + // int64 nprobe = 5; + case 5: { + if (static_cast< ::PROTOBUF_NAMESPACE_ID::uint8>(tag) == (40 & 0xFF)) { + + DO_((::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::ReadPrimitive< + ::PROTOBUF_NAMESPACE_ID::int64, ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::TYPE_INT64>( + input, &nprobe_))); + } else { + goto handle_unusual; + } + break; + } + default: { handle_unusual: if (tag == 0) { @@ -2862,6 +2961,11 @@ void SearchParam::SerializeWithCachedSizes( ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt64(4, this->topk(), output); } + // int64 nprobe = 5; + if (this->nprobe() != 0) { + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt64(5, this->nprobe(), output); + } + if (_internal_metadata_.have_unknown_fields()) { ::PROTOBUF_NAMESPACE_ID::internal::WireFormat::SerializeUnknownFields( _internal_metadata_.unknown_fields(), output); @@ -2907,6 +3011,11 @@ void SearchParam::SerializeWithCachedSizes( target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt64ToArray(4, this->topk(), target); } + // int64 nprobe = 5; + if (this->nprobe() != 0) { + target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt64ToArray(5, this->nprobe(), target); + } + if (_internal_metadata_.have_unknown_fields()) { target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormat::SerializeUnknownFieldsToArray( _internal_metadata_.unknown_fields(), target); @@ -2964,6 +3073,13 @@ size_t SearchParam::ByteSizeLong() const { this->topk()); } + // int64 nprobe = 5; + if (this->nprobe() != 0) { + total_size += 1 + + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::Int64Size( + this->nprobe()); + } + int cached_size = ::PROTOBUF_NAMESPACE_ID::internal::ToCachedSize(total_size); SetCachedSize(cached_size); return total_size; @@ -3000,6 +3116,9 @@ void SearchParam::MergeFrom(const SearchParam& from) { if (from.topk() != 0) { set_topk(from.topk()); } + if (from.nprobe() != 0) { + set_nprobe(from.nprobe()); + } } void SearchParam::CopyFrom(const ::PROTOBUF_NAMESPACE_ID::Message& from) { @@ -3028,6 +3147,7 @@ void SearchParam::InternalSwap(SearchParam* other) { table_name_.Swap(&other->table_name_, &::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), GetArenaNoVirtual()); swap(topk_, other->topk_); + swap(nprobe_, other->nprobe_); } ::PROTOBUF_NAMESPACE_ID::Metadata SearchParam::GetMetadata() const { @@ -5236,15 +5356,15 @@ Index::Index(const Index& from) _internal_metadata_(nullptr) { _internal_metadata_.MergeFrom(from._internal_metadata_); ::memcpy(&nlist_, &from.nlist_, - static_cast(reinterpret_cast(&index_file_size_) - - reinterpret_cast(&nlist_)) + sizeof(index_file_size_)); + static_cast(reinterpret_cast(&metric_type_) - + reinterpret_cast(&nlist_)) + sizeof(metric_type_)); // @@protoc_insertion_point(copy_constructor:milvus.grpc.Index) } void Index::SharedCtor() { ::memset(&nlist_, 0, static_cast( - reinterpret_cast(&index_file_size_) - - reinterpret_cast(&nlist_)) + sizeof(index_file_size_)); + reinterpret_cast(&metric_type_) - + reinterpret_cast(&nlist_)) + sizeof(metric_type_)); } Index::~Index() { @@ -5271,8 +5391,8 @@ void Index::Clear() { (void) cached_has_bits; ::memset(&nlist_, 0, static_cast( - reinterpret_cast(&index_file_size_) - - reinterpret_cast(&nlist_)) + sizeof(index_file_size_)); + reinterpret_cast(&metric_type_) - + reinterpret_cast(&nlist_)) + sizeof(metric_type_)); _internal_metadata_.Clear(); } @@ -5305,6 +5425,13 @@ const char* Index::_InternalParse(const char* ptr, ::PROTOBUF_NAMESPACE_ID::inte CHK_(ptr); } else goto handle_unusual; continue; + // int32 metric_type = 4; + case 4: + if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 32)) { + metric_type_ = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint(&ptr); + CHK_(ptr); + } else goto handle_unusual; + continue; default: { handle_unusual: if ((tag & 7) == 4 || tag == 0) { @@ -5374,6 +5501,19 @@ bool Index::MergePartialFromCodedStream( break; } + // int32 metric_type = 4; + case 4: { + if (static_cast< ::PROTOBUF_NAMESPACE_ID::uint8>(tag) == (32 & 0xFF)) { + + DO_((::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::ReadPrimitive< + ::PROTOBUF_NAMESPACE_ID::int32, ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::TYPE_INT32>( + input, &metric_type_))); + } else { + goto handle_unusual; + } + break; + } + default: { handle_unusual: if (tag == 0) { @@ -5416,6 +5556,11 @@ void Index::SerializeWithCachedSizes( ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt32(3, this->index_file_size(), output); } + // int32 metric_type = 4; + if (this->metric_type() != 0) { + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt32(4, this->metric_type(), output); + } + if (_internal_metadata_.have_unknown_fields()) { ::PROTOBUF_NAMESPACE_ID::internal::WireFormat::SerializeUnknownFields( _internal_metadata_.unknown_fields(), output); @@ -5444,6 +5589,11 @@ void Index::SerializeWithCachedSizes( target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt32ToArray(3, this->index_file_size(), target); } + // int32 metric_type = 4; + if (this->metric_type() != 0) { + target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt32ToArray(4, this->metric_type(), target); + } + if (_internal_metadata_.have_unknown_fields()) { target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormat::SerializeUnknownFieldsToArray( _internal_metadata_.unknown_fields(), target); @@ -5486,6 +5636,13 @@ size_t Index::ByteSizeLong() const { this->index_file_size()); } + // int32 metric_type = 4; + if (this->metric_type() != 0) { + total_size += 1 + + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::Int32Size( + this->metric_type()); + } + int cached_size = ::PROTOBUF_NAMESPACE_ID::internal::ToCachedSize(total_size); SetCachedSize(cached_size); return total_size; @@ -5522,6 +5679,9 @@ void Index::MergeFrom(const Index& from) { if (from.index_file_size() != 0) { set_index_file_size(from.index_file_size()); } + if (from.metric_type() != 0) { + set_metric_type(from.metric_type()); + } } void Index::CopyFrom(const ::PROTOBUF_NAMESPACE_ID::Message& from) { @@ -5548,6 +5708,7 @@ void Index::InternalSwap(Index* other) { swap(nlist_, other->nlist_); swap(index_type_, other->index_type_); swap(index_file_size_, other->index_file_size_); + swap(metric_type_, other->metric_type_); } ::PROTOBUF_NAMESPACE_ID::Metadata Index::GetMetadata() const { diff --git a/cpp/src/grpc/gen-milvus/milvus.pb.h b/cpp/src/grpc/gen-milvus/milvus.pb.h index 92708e1852..d4c33b848a 100644 --- a/cpp/src/grpc/gen-milvus/milvus.pb.h +++ b/cpp/src/grpc/gen-milvus/milvus.pb.h @@ -838,6 +838,7 @@ class InsertParam : enum : int { kRowRecordArrayFieldNumber = 2, + kRowIdArrayFieldNumber = 3, kTableNameFieldNumber = 1, }; // repeated .milvus.grpc.RowRecord row_record_array = 2; @@ -851,6 +852,17 @@ class InsertParam : const ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::milvus::grpc::RowRecord >& row_record_array() const; + // repeated int64 row_id_array = 3; + int row_id_array_size() const; + void clear_row_id_array(); + ::PROTOBUF_NAMESPACE_ID::int64 row_id_array(int index) const; + void set_row_id_array(int index, ::PROTOBUF_NAMESPACE_ID::int64 value); + void add_row_id_array(::PROTOBUF_NAMESPACE_ID::int64 value); + const ::PROTOBUF_NAMESPACE_ID::RepeatedField< ::PROTOBUF_NAMESPACE_ID::int64 >& + row_id_array() const; + ::PROTOBUF_NAMESPACE_ID::RepeatedField< ::PROTOBUF_NAMESPACE_ID::int64 >* + mutable_row_id_array(); + // string table_name = 1; void clear_table_name(); const std::string& table_name() const; @@ -868,6 +880,8 @@ class InsertParam : ::PROTOBUF_NAMESPACE_ID::internal::InternalMetadataWithArena _internal_metadata_; ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::milvus::grpc::RowRecord > row_record_array_; + ::PROTOBUF_NAMESPACE_ID::RepeatedField< ::PROTOBUF_NAMESPACE_ID::int64 > row_id_array_; + mutable std::atomic _row_id_array_cached_byte_size_; ::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr table_name_; mutable ::PROTOBUF_NAMESPACE_ID::internal::CachedSize _cached_size_; friend struct ::TableStruct_milvus_2eproto; @@ -1139,6 +1153,7 @@ class SearchParam : kQueryRangeArrayFieldNumber = 3, kTableNameFieldNumber = 1, kTopkFieldNumber = 4, + kNprobeFieldNumber = 5, }; // repeated .milvus.grpc.RowRecord query_record_array = 2; int query_record_array_size() const; @@ -1178,6 +1193,11 @@ class SearchParam : ::PROTOBUF_NAMESPACE_ID::int64 topk() const; void set_topk(::PROTOBUF_NAMESPACE_ID::int64 value); + // int64 nprobe = 5; + void clear_nprobe(); + ::PROTOBUF_NAMESPACE_ID::int64 nprobe() const; + void set_nprobe(::PROTOBUF_NAMESPACE_ID::int64 value); + // @@protoc_insertion_point(class_scope:milvus.grpc.SearchParam) private: class _Internal; @@ -1187,6 +1207,7 @@ class SearchParam : ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::milvus::grpc::Range > query_range_array_; ::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr table_name_; ::PROTOBUF_NAMESPACE_ID::int64 topk_; + ::PROTOBUF_NAMESPACE_ID::int64 nprobe_; mutable ::PROTOBUF_NAMESPACE_ID::internal::CachedSize _cached_size_; friend struct ::TableStruct_milvus_2eproto; }; @@ -2312,6 +2333,7 @@ class Index : kNlistFieldNumber = 2, kIndexTypeFieldNumber = 1, kIndexFileSizeFieldNumber = 3, + kMetricTypeFieldNumber = 4, }; // int64 nlist = 2; void clear_nlist(); @@ -2328,6 +2350,11 @@ class Index : ::PROTOBUF_NAMESPACE_ID::int32 index_file_size() const; void set_index_file_size(::PROTOBUF_NAMESPACE_ID::int32 value); + // int32 metric_type = 4; + void clear_metric_type(); + ::PROTOBUF_NAMESPACE_ID::int32 metric_type() const; + void set_metric_type(::PROTOBUF_NAMESPACE_ID::int32 value); + // @@protoc_insertion_point(class_scope:milvus.grpc.Index) private: class _Internal; @@ -2336,6 +2363,7 @@ class Index : ::PROTOBUF_NAMESPACE_ID::int64 nlist_; ::PROTOBUF_NAMESPACE_ID::int32 index_type_; ::PROTOBUF_NAMESPACE_ID::int32 index_file_size_; + ::PROTOBUF_NAMESPACE_ID::int32 metric_type_; mutable ::PROTOBUF_NAMESPACE_ID::internal::CachedSize _cached_size_; friend struct ::TableStruct_milvus_2eproto; }; @@ -3059,6 +3087,36 @@ InsertParam::row_record_array() const { return row_record_array_; } +// repeated int64 row_id_array = 3; +inline int InsertParam::row_id_array_size() const { + return row_id_array_.size(); +} +inline void InsertParam::clear_row_id_array() { + row_id_array_.Clear(); +} +inline ::PROTOBUF_NAMESPACE_ID::int64 InsertParam::row_id_array(int index) const { + // @@protoc_insertion_point(field_get:milvus.grpc.InsertParam.row_id_array) + return row_id_array_.Get(index); +} +inline void InsertParam::set_row_id_array(int index, ::PROTOBUF_NAMESPACE_ID::int64 value) { + row_id_array_.Set(index, value); + // @@protoc_insertion_point(field_set:milvus.grpc.InsertParam.row_id_array) +} +inline void InsertParam::add_row_id_array(::PROTOBUF_NAMESPACE_ID::int64 value) { + row_id_array_.Add(value); + // @@protoc_insertion_point(field_add:milvus.grpc.InsertParam.row_id_array) +} +inline const ::PROTOBUF_NAMESPACE_ID::RepeatedField< ::PROTOBUF_NAMESPACE_ID::int64 >& +InsertParam::row_id_array() const { + // @@protoc_insertion_point(field_list:milvus.grpc.InsertParam.row_id_array) + return row_id_array_; +} +inline ::PROTOBUF_NAMESPACE_ID::RepeatedField< ::PROTOBUF_NAMESPACE_ID::int64 >* +InsertParam::mutable_row_id_array() { + // @@protoc_insertion_point(field_mutable_list:milvus.grpc.InsertParam.row_id_array) + return &row_id_array_; +} + // ------------------------------------------------------------------- // VectorIds @@ -3267,6 +3325,20 @@ inline void SearchParam::set_topk(::PROTOBUF_NAMESPACE_ID::int64 value) { // @@protoc_insertion_point(field_set:milvus.grpc.SearchParam.topk) } +// int64 nprobe = 5; +inline void SearchParam::clear_nprobe() { + nprobe_ = PROTOBUF_LONGLONG(0); +} +inline ::PROTOBUF_NAMESPACE_ID::int64 SearchParam::nprobe() const { + // @@protoc_insertion_point(field_get:milvus.grpc.SearchParam.nprobe) + return nprobe_; +} +inline void SearchParam::set_nprobe(::PROTOBUF_NAMESPACE_ID::int64 value) { + + nprobe_ = value; + // @@protoc_insertion_point(field_set:milvus.grpc.SearchParam.nprobe) +} + // ------------------------------------------------------------------- // SearchInFilesParam @@ -3825,6 +3897,20 @@ inline void Index::set_index_file_size(::PROTOBUF_NAMESPACE_ID::int32 value) { // @@protoc_insertion_point(field_set:milvus.grpc.Index.index_file_size) } +// int32 metric_type = 4; +inline void Index::clear_metric_type() { + metric_type_ = 0; +} +inline ::PROTOBUF_NAMESPACE_ID::int32 Index::metric_type() const { + // @@protoc_insertion_point(field_get:milvus.grpc.Index.metric_type) + return metric_type_; +} +inline void Index::set_metric_type(::PROTOBUF_NAMESPACE_ID::int32 value) { + + metric_type_ = value; + // @@protoc_insertion_point(field_set:milvus.grpc.Index.metric_type) +} + // ------------------------------------------------------------------- // IndexParam diff --git a/cpp/src/grpc/milvus.proto b/cpp/src/grpc/milvus.proto index 04317d45f9..59d74813d1 100644 --- a/cpp/src/grpc/milvus.proto +++ b/cpp/src/grpc/milvus.proto @@ -43,6 +43,7 @@ message RowRecord { message InsertParam { string table_name = 1; repeated RowRecord row_record_array = 2; + repeated int64 row_id_array = 3; //optional } /** @@ -61,6 +62,7 @@ message SearchParam { repeated RowRecord query_record_array = 2; repeated Range query_range_array = 3; int64 topk = 4; + int64 nprobe = 5; } /** @@ -125,6 +127,7 @@ message Index { int32 index_type = 1; int64 nlist = 2; int32 index_file_size = 3; + int32 metric_type = 4; } /** diff --git a/cpp/src/sdk/grpc/ClientProxy.cpp b/cpp/src/sdk/grpc/ClientProxy.cpp index 059bf5c852..f107a2694a 100644 --- a/cpp/src/sdk/grpc/ClientProxy.cpp +++ b/cpp/src/sdk/grpc/ClientProxy.cpp @@ -328,7 +328,14 @@ ClientProxy::DeleteByRange(milvus::Range &range, const std::string &table_name) Status ClientProxy::PreloadTable(const std::string &table_name) const { - + try { + ::milvus::grpc::TableName grpc_table_name; + grpc_table_name.set_table_name(table_name); + Status status = client_ptr_->PreloadTable(grpc_table_name); + return status; + } catch (std::exception &ex) { + return Status(StatusCode::UnknownError, "fail to show tables: " + std::string(ex.what())); + } } IndexParam diff --git a/cpp/src/sdk/grpc/GrpcClient.cpp b/cpp/src/sdk/grpc/GrpcClient.cpp index 1212c68847..00894ea529 100644 --- a/cpp/src/sdk/grpc/GrpcClient.cpp +++ b/cpp/src/sdk/grpc/GrpcClient.cpp @@ -234,7 +234,7 @@ GrpcClient::Cmd(std::string &result, result = response.string_reply(); if (!grpc_status.ok()) { - std::cerr << "Ping gRPC failed!" << std::endl; + std::cerr << "Cmd gRPC failed!" << std::endl; return Status(StatusCode::RPCFailed, grpc_status.error_message()); } @@ -246,6 +246,24 @@ GrpcClient::Cmd(std::string &result, return Status::OK(); } +Status +GrpcClient::PreloadTable(milvus::grpc::TableName &table_name) { + ClientContext context; + ::milvus::grpc::Status response; + ::grpc::Status grpc_status = stub_->PreloadTable(&context, table_name, &response); + + if (!grpc_status.ok()) { + std::cerr << "PreloadTable gRPC failed!" << std::endl; + return Status(StatusCode::RPCFailed, grpc_status.error_message()); + } + + if (response.error_code() != grpc::SUCCESS) { + std::cerr << response.reason() << std::endl; + return Status(StatusCode::ServerFailed, response.reason()); + } + return Status::OK(); +} + Status GrpcClient::Disconnect() { stub_.release(); diff --git a/cpp/src/server/grpc_impl/GrpcRequestHandler.cpp b/cpp/src/server/grpc_impl/GrpcRequestHandler.cpp index 0f7d5caf5c..584023e4fc 100644 --- a/cpp/src/server/grpc_impl/GrpcRequestHandler.cpp +++ b/cpp/src/server/grpc_impl/GrpcRequestHandler.cpp @@ -175,7 +175,12 @@ GrpcRequestHandler::DeleteByRange(::grpc::ServerContext *context, GrpcRequestHandler::PreloadTable(::grpc::ServerContext *context, const ::milvus::grpc::TableName *request, ::milvus::grpc::Status *response) { - + BaseTaskPtr task_ptr = PreloadTableTask::Create(request->table_name()); + ::milvus::grpc::Status grpc_status; + GrpcRequestScheduler::ExecTask(task_ptr, &grpc_status); + response->set_reason(grpc_status.reason()); + response->set_error_code(grpc_status.error_code()); + return ::grpc::Status::OK; } ::grpc::Status diff --git a/cpp/src/server/grpc_impl/GrpcRequestTask.cpp b/cpp/src/server/grpc_impl/GrpcRequestTask.cpp index 6e3d73edc6..545c3cde9b 100644 --- a/cpp/src/server/grpc_impl/GrpcRequestTask.cpp +++ b/cpp/src/server/grpc_impl/GrpcRequestTask.cpp @@ -706,6 +706,45 @@ CmdTask::OnExecute() { return SERVER_SUCCESS; } +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +PreloadTableTask::PreloadTableTask(const std::string &table_name) + : GrpcBaseTask(DDL_DML_TASK_GROUP), + table_name_(table_name) { + +} + +BaseTaskPtr +PreloadTableTask::Create(const std::string &table_name){ + return std::shared_ptr(new PreloadTableTask(table_name)); +} + +ServerError +PreloadTableTask::OnExecute() { + try { + TimeRecorder rc("PreloadTableTask"); + + //step 1: check arguments + ServerError res = ValidationUtil::ValidateTableName(table_name_); + if (res != SERVER_SUCCESS) { + return SetError(res, "Invalid table name: " + table_name_); + } + + //step 2: check table existence + engine::Status stat = DBWrapper::DB()->PreloadTable(table_name_); + if (!stat.ok()) { + return SetError(DB_META_TRANSACTION_FAILED, "Engine failed: " + stat.ToString()); + } + + rc.ElapseFromBegin("totally cost"); + } catch (std::exception &ex) { + return SetError(SERVER_UNEXPECTED_ERROR, ex.what()); + } + + return SERVER_SUCCESS; +} + + + } } } From b230f19ce56291cf3cff7692a2925e028039bc08 Mon Sep 17 00:00:00 2001 From: Yu Kun Date: Wed, 14 Aug 2019 19:23:56 +0800 Subject: [PATCH 3/3] fix bugs with discessions Former-commit-id: 54cd32977b70608898c6c4c61e5971d4716a8e08 --- cpp/src/db/DBImpl.cpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/cpp/src/db/DBImpl.cpp b/cpp/src/db/DBImpl.cpp index 0275fcb31b..1b6630ef84 100644 --- a/cpp/src/db/DBImpl.cpp +++ b/cpp/src/db/DBImpl.cpp @@ -32,7 +32,6 @@ namespace { constexpr uint64_t METRIC_ACTION_INTERVAL = 1; constexpr uint64_t COMPACT_ACTION_INTERVAL = 1; constexpr uint64_t INDEX_ACTION_INTERVAL = 1; -constexpr int64_t unit = 1024 * 1024 * 1024; void CollectInsertMetrics(double total_time, size_t n, bool succeed) { double avg_time = total_time / n; @@ -139,6 +138,8 @@ Status DBImpl::PreloadTable(const std::string &table_id) { int64_t size = 0; int64_t cache_total = cache::CpuCacheMgr::GetInstance()->CacheCapacity(); + int64_t cache_usage = cache::CpuCacheMgr::GetInstance()->CacheUsage(); + int64_t available_size = cache_total - cache_usage; for(auto &day_files : files) { for (auto &file : day_files.second) { @@ -149,12 +150,12 @@ Status DBImpl::PreloadTable(const std::string &table_id) { } size += engine->PhysicalSize(); - if (size > cache_total) { + if (size > available_size) { break; } else { try { //step 1: load index - engine->Load(options_.insert_cache_immediately_); + engine->Load(true); } catch (std::exception &ex) { std::string msg = "load to cache exception" + std::string(ex.what()); ENGINE_LOG_ERROR << msg;