From c600b71224de60c99cc8d17dd20f9e5a959c0303 Mon Sep 17 00:00:00 2001 From: starlord Date: Fri, 23 Aug 2019 16:07:47 +0800 Subject: [PATCH] MS-406 add table flag for meta Former-commit-id: acd9e3cbb26b7c2e74c1c566ba4ea8427a1da58d --- cpp/CHANGELOG.md | 1 + cpp/src/db/DB.h | 1 + cpp/src/db/DBImpl.cpp | 4 + cpp/src/db/DBImpl.h | 35 ++++----- cpp/src/db/Utils.cpp | 4 + cpp/src/db/Utils.h | 2 + cpp/src/db/meta/Meta.h | 3 + cpp/src/db/meta/MetaTypes.h | 3 + cpp/src/db/meta/MySQLMetaImpl.cpp | 43 ++++++++++- cpp/src/db/meta/MySQLMetaImpl.h | 7 ++ cpp/src/db/meta/SqliteMetaImpl.cpp | 51 ++++++++++--- cpp/src/db/meta/SqliteMetaImpl.h | 78 ++++++++------------ cpp/src/db/scheduler/task/SearchTask.cpp | 2 +- cpp/src/server/DBWrapper.cpp | 6 -- cpp/src/server/ServerConfig.h | 9 --- cpp/src/server/grpc_impl/GrpcRequestTask.cpp | 24 ++++++ cpp/unittest/metrics/metricbase_test.cpp | 3 +- cpp/unittest/metrics/prometheus_test.cpp | 3 +- 18 files changed, 179 insertions(+), 100 deletions(-) diff --git a/cpp/CHANGELOG.md b/cpp/CHANGELOG.md index 7fc573ad43..45bdad30f9 100644 --- a/cpp/CHANGELOG.md +++ b/cpp/CHANGELOG.md @@ -40,6 +40,7 @@ Please mark all change in change log and use the ticket from JIRA. - MS-394 - Update scheduler unittest - MS-400 - Add timestamp record in task state change function - MS-402 - Add dump implementation for TaskTableItem +- MS-406 - Add table flag for meta ## New Feature - MS-343 - Implement ResourceMgr diff --git a/cpp/src/db/DB.h b/cpp/src/db/DB.h index b143f0c233..f421541741 100644 --- a/cpp/src/db/DB.h +++ b/cpp/src/db/DB.h @@ -29,6 +29,7 @@ public: virtual Status AllTables(std::vector& table_schema_array) = 0; virtual Status GetTableRowCount(const std::string& table_id, uint64_t& row_count) = 0; virtual Status PreloadTable(const std::string& table_id) = 0; + virtual Status UpdateTableFlag(const std::string &table_id, int64_t flag) = 0; virtual Status InsertVectors(const std::string& table_id_, uint64_t n, const float* vectors, IDNumbers& vector_ids_) = 0; diff --git a/cpp/src/db/DBImpl.cpp b/cpp/src/db/DBImpl.cpp index 8a002f29b0..6b0b87889c 100644 --- a/cpp/src/db/DBImpl.cpp +++ b/cpp/src/db/DBImpl.cpp @@ -154,6 +154,10 @@ Status DBImpl::PreloadTable(const std::string &table_id) { return Status::OK(); } +Status DBImpl::UpdateTableFlag(const std::string &table_id, int64_t flag) { + return meta_ptr_->UpdateTableFlag(table_id, flag); +} + Status DBImpl::GetTableRowCount(const std::string& table_id, uint64_t& row_count) { return meta_ptr_->Count(table_id, row_count); } diff --git a/cpp/src/db/DBImpl.h b/cpp/src/db/DBImpl.h index 97c36fadbc..65ac489c49 100644 --- a/cpp/src/db/DBImpl.h +++ b/cpp/src/db/DBImpl.h @@ -36,40 +36,32 @@ class DBImpl : public DB { explicit DBImpl(const Options &options); - Status - CreateTable(meta::TableSchema &table_schema) override; + Status CreateTable(meta::TableSchema &table_schema) override; - Status - DeleteTable(const std::string &table_id, const meta::DatesT &dates) override; + Status DeleteTable(const std::string &table_id, const meta::DatesT &dates) override; - Status - DescribeTable(meta::TableSchema &table_schema) override; + Status DescribeTable(meta::TableSchema &table_schema) override; - Status - HasTable(const std::string &table_id, bool &has_or_not) override; + Status HasTable(const std::string &table_id, bool &has_or_not) override; - Status - AllTables(std::vector &table_schema_array) override; + Status AllTables(std::vector &table_schema_array) override; - Status - PreloadTable(const std::string &table_id) override; + Status PreloadTable(const std::string &table_id) override; - Status - GetTableRowCount(const std::string &table_id, uint64_t &row_count) override; + Status UpdateTableFlag(const std::string &table_id, int64_t flag); - Status - InsertVectors(const std::string &table_id, uint64_t n, const float *vectors, IDNumbers &vector_ids) override; + Status GetTableRowCount(const std::string &table_id, uint64_t &row_count) override; - Status - Query(const std::string &table_id, + Status InsertVectors(const std::string &table_id, uint64_t n, const float *vectors, IDNumbers &vector_ids) override; + + Status Query(const std::string &table_id, uint64_t k, uint64_t nq, uint64_t nprobe, const float *vectors, QueryResults &results) override; - Status - Query(const std::string &table_id, + Status Query(const std::string &table_id, uint64_t k, uint64_t nq, uint64_t nprobe, @@ -77,8 +69,7 @@ class DBImpl : public DB { const meta::DatesT &dates, QueryResults &results) override; - Status - Query(const std::string &table_id, + Status Query(const std::string &table_id, const std::vector &file_ids, uint64_t k, uint64_t nq, diff --git a/cpp/src/db/Utils.cpp b/cpp/src/db/Utils.cpp index 8dd12b0cdd..e227623e23 100644 --- a/cpp/src/db/Utils.cpp +++ b/cpp/src/db/Utils.cpp @@ -153,6 +153,10 @@ bool IsSameIndex(const TableIndex& index1, const TableIndex& index2) { && index1.metric_type_ == index2.metric_type_; } +bool UserDefinedId(int64_t flag) { + return flag & meta::FLAG_MASK_USERID; +} + } // namespace utils } // namespace engine } // namespace milvus diff --git a/cpp/src/db/Utils.h b/cpp/src/db/Utils.h index 101d849ca3..d6244ebc91 100644 --- a/cpp/src/db/Utils.h +++ b/cpp/src/db/Utils.h @@ -27,6 +27,8 @@ Status DeleteTableFilePath(const DBMetaOptions& options, meta::TableFileSchema& bool IsSameIndex(const TableIndex& index1, const TableIndex& index2); +bool UserDefinedId(int64_t flag); + } // namespace utils } // namespace engine } // namespace milvus diff --git a/cpp/src/db/meta/Meta.h b/cpp/src/db/meta/Meta.h index 80ae0fb22e..93ecb58807 100644 --- a/cpp/src/db/meta/Meta.h +++ b/cpp/src/db/meta/Meta.h @@ -42,6 +42,9 @@ class Meta { virtual Status UpdateTableIndexParam(const std::string &table_id, const TableIndex& index) = 0; + virtual Status + UpdateTableFlag(const std::string &table_id, int64_t flag) = 0; + virtual Status DeleteTable(const std::string &table_id) = 0; diff --git a/cpp/src/db/meta/MetaTypes.h b/cpp/src/db/meta/MetaTypes.h index b0c3376593..5fa4c0bcfc 100644 --- a/cpp/src/db/meta/MetaTypes.h +++ b/cpp/src/db/meta/MetaTypes.h @@ -22,6 +22,8 @@ constexpr int32_t DEFAULT_NLIST = 16384; constexpr int32_t DEFAULT_INDEX_FILE_SIZE = 1024*ONE_MB; constexpr int32_t DEFAULT_METRIC_TYPE = (int)MetricType::L2; +constexpr int64_t FLAG_MASK_USERID = 1; + typedef int DateT; const DateT EmptyDate = -1; typedef std::vector DatesT; @@ -37,6 +39,7 @@ struct TableSchema { int32_t state_ = (int)NORMAL; uint16_t dimension_ = 0; int64_t created_on_ = 0; + int64_t flag_ = 0; int32_t engine_type_ = DEFAULT_ENGINE_TYPE; int32_t nlist_ = DEFAULT_NLIST; int32_t index_file_size_ = DEFAULT_INDEX_FILE_SIZE; diff --git a/cpp/src/db/meta/MySQLMetaImpl.cpp b/cpp/src/db/meta/MySQLMetaImpl.cpp index 954c498f7f..3f85b3503f 100644 --- a/cpp/src/db/meta/MySQLMetaImpl.cpp +++ b/cpp/src/db/meta/MySQLMetaImpl.cpp @@ -155,6 +155,7 @@ Status MySQLMetaImpl::Initialize() { "state INT NOT NULL, " << "dimension SMALLINT NOT NULL, " << "created_on BIGINT NOT NULL, " << + "flag BIGINT DEFAULT 0 NOT NULL, " << "engine_type INT DEFAULT 1 NOT NULL, " << "nlist INT DEFAULT 16384 NOT NULL, " << "index_file_size INT DEFAULT 1024 NOT NULL, " << @@ -425,7 +426,7 @@ Status MySQLMetaImpl::UpdateTableIndexParam(const std::string &table_id, const T "engine_type_ = " << index.engine_type_ << ", " << "nlist = " << index.nlist_ << ", " << "index_file_size = " << index.index_file_size_*ONE_MB << ", " << - "metric_type = " << index.metric_type_ << ", " << + "metric_type = " << index.metric_type_ << " " << "WHERE id = " << quote << table_id << ";"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::UpdateTableIndexParam: " << updateTableIndexParamQuery.str(); @@ -455,6 +456,46 @@ Status MySQLMetaImpl::UpdateTableIndexParam(const std::string &table_id, const T return Status::OK(); } +Status MySQLMetaImpl::UpdateTableFlag(const std::string &table_id, int64_t flag) { + try { + MetricCollector metric; + + { + ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); + + if (connectionPtr == nullptr) { + return Status::Error("Failed to connect to database server"); + } + + Query updateTableFlagQuery = connectionPtr->query(); + updateTableFlagQuery << "UPDATE Tables " << + "SET flag = " << flag << " " << + "WHERE id = " << quote << table_id << ";"; + + ENGINE_LOG_DEBUG << "MySQLMetaImpl::UpdateTableFlag: " << updateTableFlagQuery.str(); + + + if (!updateTableFlagQuery.exec()) { + ENGINE_LOG_ERROR << "QUERY ERROR WHEN UPDATING TABLE FLAG"; + return Status::DBTransactionError("QUERY ERROR WHEN UPDATING TABLE FLAG", + updateTableFlagQuery.error()); + } + + } //Scoped Connection + + } catch (const BadQuery &er) { + // Handle any query errors + ENGINE_LOG_ERROR << "QUERY ERROR WHEN UPDATING TABLE FLAG" << ": " << er.what(); + return Status::DBTransactionError("QUERY ERROR WHEN UPDATING TABLE FLAG", er.what()); + } catch (const Exception &er) { + // Catch-all for any other MySQL++ exceptions + ENGINE_LOG_ERROR << "GENERAL ERROR WHEN UPDATING TABLE FLAG" << ": " << er.what(); + return Status::DBTransactionError("GENERAL ERROR WHEN UPDATING TABLE FLAG", er.what()); + } + + return Status::OK(); +} + Status MySQLMetaImpl::DescribeTableIndex(const std::string &table_id, TableIndex& index) { try { MetricCollector metric; diff --git a/cpp/src/db/meta/MySQLMetaImpl.h b/cpp/src/db/meta/MySQLMetaImpl.h index 3fdd80beed..fa9880942a 100644 --- a/cpp/src/db/meta/MySQLMetaImpl.h +++ b/cpp/src/db/meta/MySQLMetaImpl.h @@ -26,14 +26,19 @@ class MySQLMetaImpl : public Meta { MySQLMetaImpl(const DBMetaOptions &options_, const int &mode); Status CreateTable(TableSchema &table_schema) override; + Status DescribeTable(TableSchema &group_info_) override; + Status HasTable(const std::string &table_id, bool &has_or_not) override; + Status AllTables(std::vector &table_schema_array) override; Status DeleteTable(const std::string &table_id) override; + Status DeleteTableFiles(const std::string &table_id) override; Status CreateTableFile(TableFileSchema &file_schema) override; + Status DropPartitionsByDates(const std::string &table_id, const DatesT &dates) override; @@ -45,6 +50,8 @@ class MySQLMetaImpl : public Meta { Status UpdateTableIndexParam(const std::string &table_id, const TableIndex& index) override; + Status UpdateTableFlag(const std::string &table_id, int64_t flag); + Status DescribeTableIndex(const std::string &table_id, TableIndex& index) override; Status DropTableIndex(const std::string &table_id) override; diff --git a/cpp/src/db/meta/SqliteMetaImpl.cpp b/cpp/src/db/meta/SqliteMetaImpl.cpp index b4859473ef..0132fa87ce 100644 --- a/cpp/src/db/meta/SqliteMetaImpl.cpp +++ b/cpp/src/db/meta/SqliteMetaImpl.cpp @@ -62,6 +62,7 @@ inline auto StoragePrototype(const std::string &path) { make_column("state", &TableSchema::state_), make_column("dimension", &TableSchema::dimension_), make_column("created_on", &TableSchema::created_on_), + make_column("flag", &TableSchema::flag_, default_value(0)), make_column("engine_type", &TableSchema::engine_type_), make_column("nlist", &TableSchema::nlist_), make_column("index_file_size", &TableSchema::index_file_size_), @@ -267,6 +268,7 @@ Status SqliteMetaImpl::DescribeTable(TableSchema &table_schema) { &TableSchema::state_, &TableSchema::dimension_, &TableSchema::created_on_, + &TableSchema::flag_, &TableSchema::engine_type_, &TableSchema::nlist_, &TableSchema::index_file_size_, @@ -279,10 +281,11 @@ Status SqliteMetaImpl::DescribeTable(TableSchema &table_schema) { table_schema.state_ = std::get<1>(groups[0]); table_schema.dimension_ = std::get<2>(groups[0]); table_schema.created_on_ = std::get<3>(groups[0]); - table_schema.engine_type_ = std::get<4>(groups[0]); - table_schema.nlist_ = std::get<5>(groups[0]); - table_schema.index_file_size_ = std::get<6>(groups[0]); - table_schema.metric_type_ = std::get<7>(groups[0]); + table_schema.flag_ = std::get<4>(groups[0]); + table_schema.engine_type_ = std::get<5>(groups[0]); + table_schema.nlist_ = std::get<6>(groups[0]); + table_schema.index_file_size_ = std::get<7>(groups[0]); + table_schema.metric_type_ = std::get<8>(groups[0]); } else { return Status::NotFound("Table " + table_schema.table_id_ + " not found"); } @@ -358,7 +361,8 @@ Status SqliteMetaImpl::UpdateTableIndexParam(const std::string &table_id, const auto tables = ConnectorPtr->select(columns(&TableSchema::id_, &TableSchema::state_, &TableSchema::dimension_, - &TableSchema::created_on_), + &TableSchema::created_on_, + &TableSchema::flag_), where(c(&TableSchema::table_id_) == table_id and c(&TableSchema::state_) != (int) TableSchema::TO_DELETE)); @@ -369,6 +373,7 @@ Status SqliteMetaImpl::UpdateTableIndexParam(const std::string &table_id, const table_schema.state_ = std::get<1>(tables[0]); table_schema.dimension_ = std::get<2>(tables[0]); table_schema.created_on_ = std::get<3>(tables[0]); + table_schema.flag_ = std::get<4>(tables[0]); table_schema.engine_type_ = index.engine_type_; table_schema.nlist_ = index.nlist_; table_schema.index_file_size_ = index.index_file_size_*ONE_MB; @@ -394,6 +399,28 @@ Status SqliteMetaImpl::UpdateTableIndexParam(const std::string &table_id, const std::string msg = "Encounter exception when update table index: table_id = " + table_id; return HandleException(msg, e); } + + return Status::OK(); +} + +Status SqliteMetaImpl::UpdateTableFlag(const std::string &table_id, int64_t flag) { + try { + MetricCollector metric; + + //set all backup file to raw + ConnectorPtr->update_all( + set( + c(&TableSchema::flag_) = flag + ), + where( + c(&TableSchema::table_id_) == table_id + )); + + } catch (std::exception &e) { + std::string msg = "Encounter exception when update table flag: table_id = " + table_id; + return HandleException(msg, e); + } + return Status::OK(); } @@ -489,6 +516,7 @@ Status SqliteMetaImpl::AllTables(std::vector& table_schema_array) { &TableSchema::table_id_, &TableSchema::dimension_, &TableSchema::created_on_, + &TableSchema::flag_, &TableSchema::engine_type_, &TableSchema::nlist_, &TableSchema::index_file_size_, @@ -498,12 +526,13 @@ Status SqliteMetaImpl::AllTables(std::vector& table_schema_array) { TableSchema schema; schema.id_ = std::get<0>(table); schema.table_id_ = std::get<1>(table); - schema.created_on_ = std::get<2>(table); - schema.dimension_ = std::get<3>(table); - schema.engine_type_ = std::get<4>(table); - schema.nlist_ = std::get<5>(table); - schema.index_file_size_ = std::get<6>(table); - schema.metric_type_ = std::get<7>(table); + schema.dimension_ = std::get<2>(table); + schema.created_on_ = std::get<3>(table); + schema.flag_ = std::get<4>(table); + schema.engine_type_ = std::get<5>(table); + schema.nlist_ = std::get<6>(table); + schema.index_file_size_ = std::get<7>(table); + schema.metric_type_ = std::get<8>(table); table_schema_array.emplace_back(schema); } diff --git a/cpp/src/db/meta/SqliteMetaImpl.h b/cpp/src/db/meta/SqliteMetaImpl.h index 34808f202f..c11dce73cc 100644 --- a/cpp/src/db/meta/SqliteMetaImpl.h +++ b/cpp/src/db/meta/SqliteMetaImpl.h @@ -21,82 +21,64 @@ class SqliteMetaImpl : public Meta { public: explicit SqliteMetaImpl(const DBMetaOptions &options_); - Status - CreateTable(TableSchema &table_schema) override; + Status CreateTable(TableSchema &table_schema) override; - Status - DescribeTable(TableSchema &group_info_) override; + Status DescribeTable(TableSchema &group_info_) override; - Status - HasTable(const std::string &table_id, bool &has_or_not) override; + Status HasTable(const std::string &table_id, bool &has_or_not) override; - Status - AllTables(std::vector &table_schema_array) override; + Status AllTables(std::vector &table_schema_array) override; - Status - DeleteTable(const std::string &table_id) override; + Status DeleteTable(const std::string &table_id) override; - Status - DeleteTableFiles(const std::string &table_id) override; + Status DeleteTableFiles(const std::string &table_id) override; - Status - CreateTableFile(TableFileSchema &file_schema) override; + Status CreateTableFile(TableFileSchema &file_schema) override; - Status - DropPartitionsByDates(const std::string &table_id, const DatesT &dates) override; + Status DropPartitionsByDates(const std::string &table_id, const DatesT &dates) override; - Status - GetTableFiles(const std::string &table_id, const std::vector &ids, TableFilesSchema &table_files) override; + Status GetTableFiles(const std::string &table_id, + const std::vector &ids, + TableFilesSchema &table_files) override; - Status - HasNonIndexFiles(const std::string &table_id, bool &has) override; + Status HasNonIndexFiles(const std::string &table_id, bool &has) override; - Status - UpdateTableIndexParam(const std::string &table_id, const TableIndex& index) override; + Status UpdateTableIndexParam(const std::string &table_id, const TableIndex& index) override; - Status - DescribeTableIndex(const std::string &table_id, TableIndex& index) override; + Status UpdateTableFlag(const std::string &table_id, int64_t flag) override; - Status - DropTableIndex(const std::string &table_id) override; + Status DescribeTableIndex(const std::string &table_id, TableIndex& index) override; - Status - UpdateTableFilesToIndex(const std::string &table_id) override; + Status DropTableIndex(const std::string &table_id) override; - Status - UpdateTableFile(TableFileSchema &file_schema) override; + Status UpdateTableFilesToIndex(const std::string &table_id) override; - Status - UpdateTableFiles(TableFilesSchema &files) override; + Status UpdateTableFile(TableFileSchema &file_schema) override; - Status - FilesToSearch(const std::string &table_id, const DatesT &partition, DatePartionedTableFilesSchema &files) override; + Status UpdateTableFiles(TableFilesSchema &files) override; + + Status FilesToSearch(const std::string &table_id, + const DatesT &partition, + DatePartionedTableFilesSchema &files) override; Status FilesToSearch(const std::string &table_id, const std::vector &ids, const DatesT &partition, DatePartionedTableFilesSchema &files) override; - Status - FilesToMerge(const std::string &table_id, DatePartionedTableFilesSchema &files) override; + Status FilesToMerge(const std::string &table_id, DatePartionedTableFilesSchema &files) override; - Status - FilesToIndex(TableFilesSchema &) override; + Status FilesToIndex(TableFilesSchema &) override; - Status - Archive() override; + Status Archive() override; - Status - Size(uint64_t &result) override; + Status Size(uint64_t &result) override; - Status - CleanUp() override; + Status CleanUp() override; - Status - CleanUpFilesWithTTL(uint16_t seconds) override; + Status CleanUpFilesWithTTL(uint16_t seconds) override; - Status - DropAll() override; + Status DropAll() override; Status Count(const std::string &table_id, uint64_t &result) override; diff --git a/cpp/src/db/scheduler/task/SearchTask.cpp b/cpp/src/db/scheduler/task/SearchTask.cpp index 4e7c0f4611..70802e386d 100644 --- a/cpp/src/db/scheduler/task/SearchTask.cpp +++ b/cpp/src/db/scheduler/task/SearchTask.cpp @@ -16,7 +16,7 @@ namespace engine { namespace { -static constexpr size_t PARALLEL_REDUCE_THRESHOLD = 10000; +static constexpr size_t PARALLEL_REDUCE_THRESHOLD = 1000000; static constexpr size_t PARALLEL_REDUCE_BATCH = 1000; bool NeedParallelReduce(uint64_t nq, uint64_t topk) { diff --git a/cpp/src/server/DBWrapper.cpp b/cpp/src/server/DBWrapper.cpp index a84e9a2dd8..923f8f2861 100644 --- a/cpp/src/server/DBWrapper.cpp +++ b/cpp/src/server/DBWrapper.cpp @@ -74,12 +74,6 @@ DBWrapper::DBWrapper() { } } - std::string metric_type = engine_config.GetValue(CONFIG_METRICTYPE, "L2"); - if(metric_type != "L2" && metric_type != "IP") { - std::cout << "ERROR! Illegal metric type: " << metric_type << ", available options: L2 or IP" << std::endl; - kill(0, SIGUSR1); - } - //set archive config engine::ArchiveConf::CriteriaT criterial; int64_t disk = db_config.GetInt64Value(CONFIG_DB_ARCHIVE_DISK, 0); diff --git a/cpp/src/server/ServerConfig.h b/cpp/src/server/ServerConfig.h index 6a76399d42..a800664ebd 100644 --- a/cpp/src/server/ServerConfig.h +++ b/cpp/src/server/ServerConfig.h @@ -17,7 +17,6 @@ namespace server { static const char* CONFIG_SERVER = "server_config"; static const char* CONFIG_SERVER_ADDRESS = "address"; static const char* CONFIG_SERVER_PORT = "port"; -static const char* CONFIG_SERVER_PROTOCOL = "transfer_protocol"; static const char* CONFIG_CLUSTER_MODE = "mode"; static const char* CONFIG_GPU_INDEX = "gpu_index"; @@ -41,9 +40,6 @@ static const char* CONFIG_INSERT_CACHE_IMMEDIATELY = "insert_cache_immediately"; static const char* CONFIG_GPU_IDS = "gpu_ids"; static const char *GPU_CACHE_FREE_PERCENT = "gpu_cache_free_percent"; -static const char* CONFIG_LICENSE = "license_config"; -static const char* CONFIG_LICENSE_PATH = "license_path"; - static const char* CONFIG_METRIC = "metric_config"; static const char* CONFIG_METRIC_IS_STARTUP = "is_startup"; static const char* CONFIG_METRIC_COLLECTOR = "collector"; @@ -51,13 +47,8 @@ static const char* CONFIG_PROMETHEUS = "prometheus_config"; static const char* CONFIG_METRIC_PROMETHEUS_PORT = "port"; static const std::string CONFIG_ENGINE = "engine_config"; -static const std::string CONFIG_NPROBE = "nprobe"; -static const std::string CONFIG_NLIST = "nlist"; static const std::string CONFIG_DCBT = "use_blas_threshold"; -static const std::string CONFIG_METRICTYPE = "metric_type"; static const std::string CONFIG_OMP_THREAD_NUM = "omp_thread_num"; -static const std::string CONFIG_USE_HYBRID_INDEX = "use_hybrid_index"; -static const std::string CONFIG_HYBRID_INDEX_GPU = "hybrid_index_gpu"; class ServerConfig { public: diff --git a/cpp/src/server/grpc_impl/GrpcRequestTask.cpp b/cpp/src/server/grpc_impl/GrpcRequestTask.cpp index e10d2ae070..d62ab37d7e 100644 --- a/cpp/src/server/grpc_impl/GrpcRequestTask.cpp +++ b/cpp/src/server/grpc_impl/GrpcRequestTask.cpp @@ -12,6 +12,7 @@ #include "../DBWrapper.h" #include "version.h" #include "GrpcMilvusServer.h" +#include "db/Utils.h" #include "src/server/Server.h" @@ -435,6 +436,23 @@ InsertTask::OnExecute() { } } + //all user provide id, or all internal id + uint64_t row_count = 0; + DBWrapper::DB()->GetTableRowCount(table_info.table_id_, row_count); + bool empty_table = (row_count == 0); + bool user_provide_ids = !insert_param_.row_id_array().empty(); + if(!empty_table) { + //user already provided id before, all insert action require user id + if(engine::utils::UserDefinedId(table_info.flag_) && !user_provide_ids) { + return SetError(SERVER_INVALID_ARGUMENT, "Table vector ids are user defined, please provide id for this batch"); + } + + //user didn't provided id before, no need to provide user id + if(!engine::utils::UserDefinedId(table_info.flag_) && user_provide_ids) { + return SetError(SERVER_INVALID_ARGUMENT, "Table vector ids are auto generated, no need to provide id for this batch"); + } + } + rc.RecordSection("check validation"); #ifdef MILVUS_ENABLE_PROFILING @@ -490,6 +508,12 @@ InsertTask::OnExecute() { return SetError(SERVER_ILLEGAL_VECTOR_ID, msg); } + //step 5: update table flag + if(empty_table && user_provide_ids) { + stat = DBWrapper::DB()->UpdateTableFlag(insert_param_.table_name(), + table_info.flag_ | engine::meta::FLAG_MASK_USERID); + } + #ifdef MILVUS_ENABLE_PROFILING ProfilerStop(); #endif diff --git a/cpp/unittest/metrics/metricbase_test.cpp b/cpp/unittest/metrics/metricbase_test.cpp index 1997748fdd..feda6d1bf9 100644 --- a/cpp/unittest/metrics/metricbase_test.cpp +++ b/cpp/unittest/metrics/metricbase_test.cpp @@ -21,7 +21,8 @@ TEST(MetricbaseTest, METRICBASE_TEST){ instance.RawFileSizeHistogramObserve(1.0); instance.IndexFileSizeHistogramObserve(1.0); instance.BuildIndexDurationSecondsHistogramObserve(1.0); - instance.CacheUsageGaugeSet(1.0); + instance.CpuCacheUsageGaugeSet(1.0); + instance.GpuCacheUsageGaugeSet(1.0); instance.MetaAccessTotalIncrement(); instance.MetaAccessDurationSecondsHistogramObserve(1.0); instance.FaissDiskLoadDurationSecondsHistogramObserve(1.0); diff --git a/cpp/unittest/metrics/prometheus_test.cpp b/cpp/unittest/metrics/prometheus_test.cpp index 004e58a5fc..ece3cf9012 100644 --- a/cpp/unittest/metrics/prometheus_test.cpp +++ b/cpp/unittest/metrics/prometheus_test.cpp @@ -22,7 +22,8 @@ TEST(PrometheusTest, PROMETHEUS_TEST){ instance.RawFileSizeHistogramObserve(1.0); instance.IndexFileSizeHistogramObserve(1.0); instance.BuildIndexDurationSecondsHistogramObserve(1.0); - instance.CacheUsageGaugeSet(1.0); + instance.CpuCacheUsageGaugeSet(1.0); + instance.GpuCacheUsageGaugeSet(1.0); instance.MetaAccessTotalIncrement(); instance.MetaAccessDurationSecondsHistogramObserve(1.0); instance.FaissDiskLoadDurationSecondsHistogramObserve(1.0);