From 835715887c2b6a8fd8e8e7a0126d83949a1f67ca Mon Sep 17 00:00:00 2001 From: groot Date: Thu, 27 Jun 2019 11:39:27 +0800 Subject: [PATCH 1/6] archive config Former-commit-id: c8ee875b33d74f1a1a6f81351cecf21d0df6eb93 --- cpp/conf/server_config.yaml | 2 ++ cpp/src/db/DBMetaImpl.cpp | 6 +++--- cpp/src/db/Options.cpp | 6 ++++++ cpp/src/db/Options.h | 8 +++++++- cpp/src/server/DBWrapper.cpp | 13 +++++++++++++ cpp/src/server/ServerConfig.h | 2 ++ 6 files changed, 33 insertions(+), 4 deletions(-) diff --git a/cpp/conf/server_config.yaml b/cpp/conf/server_config.yaml index 8bc83dcebb..27a96960cf 100644 --- a/cpp/conf/server_config.yaml +++ b/cpp/conf/server_config.yaml @@ -8,6 +8,8 @@ db_config: db_path: /tmp/milvus # milvus data storage path db_backend_url: http://127.0.0.1 # meta database uri index_building_threshold: 1024 # index building trigger threshold, default: 1024, unit: MB + archive_disk_threshold: 512 # triger archive action if storage size exceed this value, unit: GB + archive_days_threshold: 30 # files older than x days will be archived, unit: day metric_config: is_startup: true # if monitoring start: on, off diff --git a/cpp/src/db/DBMetaImpl.cpp b/cpp/src/db/DBMetaImpl.cpp index 779307e8a6..1a8419eac2 100644 --- a/cpp/src/db/DBMetaImpl.cpp +++ b/cpp/src/db/DBMetaImpl.cpp @@ -655,7 +655,7 @@ Status DBMetaImpl::Archive() { for (auto kv : criterias) { auto &criteria = kv.first; auto &limit = kv.second; - if (criteria == "days") { + if (criteria == engine::ARCHIVE_CONF_DAYS) { long usecs = limit * D_SEC * US_PS; long now = utils::GetMicroSecTimeStamp(); try { @@ -671,11 +671,11 @@ Status DBMetaImpl::Archive() { return HandleException("Encounter exception when update table files", e); } } - if (criteria == "disk") { + if (criteria == engine::ARCHIVE_CONF_DISK) { uint64_t sum = 0; Size(sum); - auto to_delete = (sum - limit * G); + int64_t to_delete = (int64_t)sum - limit * G; DiscardFiles(to_delete); } } diff --git a/cpp/src/db/Options.cpp b/cpp/src/db/Options.cpp index ff6404751f..5f591dcb0f 100644 --- a/cpp/src/db/Options.cpp +++ b/cpp/src/db/Options.cpp @@ -24,6 +24,12 @@ ArchiveConf::ArchiveConf(const std::string& type, const std::string& criterias) ParseCritirias(criterias); } +void ArchiveConf::SetCriterias(const ArchiveConf::CriteriaT& criterial) { + for(auto& pair : criterial) { + criterias_[pair.first] = pair.second; + } +} + void ArchiveConf::ParseCritirias(const std::string& criterias) { std::stringstream ss(criterias); std::vector tokens; diff --git a/cpp/src/db/Options.h b/cpp/src/db/Options.h index 339044d9dd..919d21709c 100644 --- a/cpp/src/db/Options.h +++ b/cpp/src/db/Options.h @@ -19,14 +19,20 @@ static constexpr uint64_t ONE_KB = 1024; static constexpr uint64_t ONE_MB = ONE_KB*ONE_KB; static constexpr uint64_t ONE_GB = ONE_KB*ONE_MB; +static const std::string ARCHIVE_CONF_DISK = "disk"; +static const std::string ARCHIVE_CONF_DAYS = "days"; +static const std::string ARCHIVE_CONF_DEFAULT = ARCHIVE_CONF_DISK + ":512"; + struct ArchiveConf { using CriteriaT = std::map; - ArchiveConf(const std::string& type, const std::string& criterias = "disk:512"); + ArchiveConf(const std::string& type, const std::string& criterias = ARCHIVE_CONF_DEFAULT); const std::string& GetType() const { return type_; } const CriteriaT GetCriterias() const { return criterias_; } + void SetCriterias(const ArchiveConf::CriteriaT& criterial); + private: void ParseCritirias(const std::string& type); void ParseType(const std::string& criterias); diff --git a/cpp/src/server/DBWrapper.cpp b/cpp/src/server/DBWrapper.cpp index 4cae31ea6b..2a583a660d 100644 --- a/cpp/src/server/DBWrapper.cpp +++ b/cpp/src/server/DBWrapper.cpp @@ -24,6 +24,19 @@ DBWrapper::DBWrapper() { opt.index_trigger_size = (size_t)index_size * engine::ONE_MB; } + //set archive config + engine::ArchiveConf::CriteriaT criterial; + int64_t disk = config.GetInt64Value(CONFIG_DB_ARCHIVE_DISK, 0); + int64_t days = config.GetInt64Value(CONFIG_DB_ARCHIVE_DAYS, 0); + if(disk > 0) { + criterial[engine::ARCHIVE_CONF_DISK] = disk; + } + if(days > 0) { + criterial[engine::ARCHIVE_CONF_DAYS] = days; + } + opt.meta.archive_conf.SetCriterias(criterial); + + //create db root folder CommonUtil::CreateDirectory(opt.meta.path); zilliz::milvus::engine::DB::Open(opt, &db_); diff --git a/cpp/src/server/ServerConfig.h b/cpp/src/server/ServerConfig.h index dd7c9d2966..abf898bacf 100644 --- a/cpp/src/server/ServerConfig.h +++ b/cpp/src/server/ServerConfig.h @@ -24,6 +24,8 @@ static const std::string CONFIG_DB = "db_config"; static const std::string CONFIG_DB_URL = "db_backend_url"; static const std::string CONFIG_DB_PATH = "db_path"; static const std::string CONFIG_DB_INDEX_TRIGGER_SIZE = "index_building_threshold"; +static const std::string CONFIG_DB_ARCHIVE_DISK = "archive_disk_threshold"; +static const std::string CONFIG_DB_ARCHIVE_DAYS = "archive_days_threshold"; static const std::string CONFIG_LOG = "log_config"; From ac7e3beeba420d3fa04a9e2e875cc73f003e23d1 Mon Sep 17 00:00:00 2001 From: groot Date: Thu, 27 Jun 2019 11:56:43 +0800 Subject: [PATCH 2/6] archive config Former-commit-id: b15e120d31fae469af9171c5d3da33f7cdaeb2bd --- cpp/CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cpp/CHANGELOG.md b/cpp/CHANGELOG.md index ca4e466cf9..45c3f18d75 100644 --- a/cpp/CHANGELOG.md +++ b/cpp/CHANGELOG.md @@ -21,6 +21,8 @@ Please mark all change in change log and use the ticket from JIRA. - MS-92 - Unify behavior of debug and release build - MS-98 - Install all unit test to installation directory - MS-115 - Change is_startup of metric_config switch from true to on +- MS-122 - Archive criteria config + ## New Feature - MS-57 - Implement index load/search pipeline From 1d8d1d3e7783c3868ff872fa5111b7c1ab33bad6 Mon Sep 17 00:00:00 2001 From: groot Date: Thu, 27 Jun 2019 14:45:53 +0800 Subject: [PATCH 3/6] fix portential bug Former-commit-id: f0a5eed0de7e85c6db7b8b36e9751185b531d5ea --- cpp/src/db/MemManager.cpp | 24 +++++++++++++++++++++--- cpp/src/db/MemManager.h | 2 ++ 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/cpp/src/db/MemManager.cpp b/cpp/src/db/MemManager.cpp index fa7f3c54b0..e36b0c45ba 100644 --- a/cpp/src/db/MemManager.cpp +++ b/cpp/src/db/MemManager.cpp @@ -146,11 +146,16 @@ Status MemManager::InsertVectorsNoLock(const std::string& table_id, Status MemManager::ToImmutable() { std::unique_lock lock(mutex_); + MemIdMap temp_map; for (auto& kv: mem_id_map_) { + if(kv.second->RowCount() == 0) { + temp_map.insert(kv); + continue;//empty vector, no need to serialize + } immu_mem_list_.push_back(kv.second); } - mem_id_map_.clear(); + mem_id_map_.swap(temp_map); return Status::OK(); } @@ -168,8 +173,21 @@ Status MemManager::Serialize(std::set& table_ids) { } Status MemManager::EraseMemVector(const std::string& table_id) { - std::unique_lock lock(mutex_); - mem_id_map_.erase(table_id); + {//erase MemVector from rapid-insert cache + std::unique_lock lock(mutex_); + mem_id_map_.erase(table_id); + } + + {//erase MemVector from serialize cache + std::unique_lock lock(serialization_mtx_); + MemList temp_list; + for (auto& mem : immu_mem_list_) { + if(mem->TableId() != table_id) { + temp_list.push_back(mem); + } + } + immu_mem_list_.swap(temp_list); + } return Status::OK(); } diff --git a/cpp/src/db/MemManager.h b/cpp/src/db/MemManager.h index 2aa0183898..0ce88d504d 100644 --- a/cpp/src/db/MemManager.h +++ b/cpp/src/db/MemManager.h @@ -45,6 +45,8 @@ public: const std::string& Location() const { return schema_.location_; } + std::string TableId() const { return schema_.table_id_; } + private: MemVectors() = delete; MemVectors(const MemVectors&) = delete; From eb76842114e0ac20b64a1c38100bd9c07815ce55 Mon Sep 17 00:00:00 2001 From: groot Date: Thu, 27 Jun 2019 15:51:25 +0800 Subject: [PATCH 4/6] MS-124 HasTable interface Former-commit-id: 1f15f5d8bfbafbcbc3bfc0f3d9ee129150fd5186 --- cpp/CHANGELOG.md | 1 + .../sdk/examples/simple/src/ClientTest.cpp | 5 + cpp/src/sdk/include/MilvusApi.h | 12 + cpp/src/sdk/src/client/ClientProxy.cpp | 9 + cpp/src/sdk/src/client/ClientProxy.h | 2 + cpp/src/sdk/src/interface/ConnectionImpl.cpp | 5 + cpp/src/sdk/src/interface/ConnectionImpl.h | 2 + cpp/src/server/RequestHandler.cpp | 9 + cpp/src/server/RequestHandler.h | 32 +- cpp/src/server/RequestTask.cpp | 44 +- cpp/src/server/RequestTask.h | 16 + cpp/src/thrift/gen-cpp/MilvusService.cpp | 413 ++++++++++++++++++ cpp/src/thrift/gen-cpp/MilvusService.h | 145 ++++++ .../gen-cpp/MilvusService_server.skeleton.cpp | 15 + cpp/src/thrift/milvus.thrift | 10 + 15 files changed, 707 insertions(+), 13 deletions(-) diff --git a/cpp/CHANGELOG.md b/cpp/CHANGELOG.md index 45c3f18d75..2bb18211a6 100644 --- a/cpp/CHANGELOG.md +++ b/cpp/CHANGELOG.md @@ -22,6 +22,7 @@ Please mark all change in change log and use the ticket from JIRA. - MS-98 - Install all unit test to installation directory - MS-115 - Change is_startup of metric_config switch from true to on - MS-122 - Archive criteria config +- MS-124 - HasTable interface ## New Feature diff --git a/cpp/src/sdk/examples/simple/src/ClientTest.cpp b/cpp/src/sdk/examples/simple/src/ClientTest.cpp index 78145446a6..19c764fd0a 100644 --- a/cpp/src/sdk/examples/simple/src/ClientTest.cpp +++ b/cpp/src/sdk/examples/simple/src/ClientTest.cpp @@ -165,6 +165,11 @@ ClientTest::Test(const std::string& address, const std::string& port) { Status stat = conn->CreateTable(tb_schema); std::cout << "CreateTable function call status: " << stat.ToString() << std::endl; PrintTableSchema(tb_schema); + + bool has_table = conn->HasTable(tb_schema.table_name); + if(has_table) { + std::cout << "Table is created" << std::endl; + } } {//describe table diff --git a/cpp/src/sdk/include/MilvusApi.h b/cpp/src/sdk/include/MilvusApi.h index 2f4532b761..302871c48b 100644 --- a/cpp/src/sdk/include/MilvusApi.h +++ b/cpp/src/sdk/include/MilvusApi.h @@ -156,6 +156,18 @@ public: virtual Status CreateTable(const TableSchema ¶m) = 0; + /** + * @brief Test table existence method + * + * This method is used to create table + * + * @param table_name, table name is going to be tested. + * + * @return Indicate if table is cexist + */ + virtual bool HasTable(const std::string &table_name) = 0; + + /** * @brief Delete table method * diff --git a/cpp/src/sdk/src/client/ClientProxy.cpp b/cpp/src/sdk/src/client/ClientProxy.cpp index 6f68344fac..1185e4988c 100644 --- a/cpp/src/sdk/src/client/ClientProxy.cpp +++ b/cpp/src/sdk/src/client/ClientProxy.cpp @@ -102,6 +102,15 @@ ClientProxy::CreateTable(const TableSchema ¶m) { return Status::OK(); } +bool +ClientProxy::HasTable(const std::string &table_name) { + if(!IsConnected()) { + return false; + } + + return ClientPtr()->interface()->HasTable(table_name); +} + Status ClientProxy::DeleteTable(const std::string &table_name) { if(!IsConnected()) { diff --git a/cpp/src/sdk/src/client/ClientProxy.h b/cpp/src/sdk/src/client/ClientProxy.h index 601bcddd8a..a2ede77c40 100644 --- a/cpp/src/sdk/src/client/ClientProxy.h +++ b/cpp/src/sdk/src/client/ClientProxy.h @@ -23,6 +23,8 @@ public: virtual Status CreateTable(const TableSchema ¶m) override; + virtual bool HasTable(const std::string &table_name) override; + virtual Status DeleteTable(const std::string &table_name) override; virtual Status AddVector(const std::string &table_name, diff --git a/cpp/src/sdk/src/interface/ConnectionImpl.cpp b/cpp/src/sdk/src/interface/ConnectionImpl.cpp index e303cf0aa0..efb5f61b1b 100644 --- a/cpp/src/sdk/src/interface/ConnectionImpl.cpp +++ b/cpp/src/sdk/src/interface/ConnectionImpl.cpp @@ -56,6 +56,11 @@ ConnectionImpl::CreateTable(const TableSchema ¶m) { return client_proxy_->CreateTable(param); } +bool +ConnectionImpl::HasTable(const std::string &table_name) { + return client_proxy_->HasTable(table_name); +} + Status ConnectionImpl::DeleteTable(const std::string &table_name) { return client_proxy_->DeleteTable(table_name); diff --git a/cpp/src/sdk/src/interface/ConnectionImpl.h b/cpp/src/sdk/src/interface/ConnectionImpl.h index 61e11c9390..0f9cd14e39 100644 --- a/cpp/src/sdk/src/interface/ConnectionImpl.h +++ b/cpp/src/sdk/src/interface/ConnectionImpl.h @@ -25,6 +25,8 @@ public: virtual Status CreateTable(const TableSchema ¶m) override; + virtual bool HasTable(const std::string &table_name) override; + virtual Status DeleteTable(const std::string &table_name) override; virtual Status AddVector(const std::string &table_name, diff --git a/cpp/src/server/RequestHandler.cpp b/cpp/src/server/RequestHandler.cpp index 62ce0711b4..037f80e0db 100644 --- a/cpp/src/server/RequestHandler.cpp +++ b/cpp/src/server/RequestHandler.cpp @@ -24,6 +24,15 @@ RequestHandler::CreateTable(const thrift::TableSchema ¶m) { RequestScheduler::ExecTask(task_ptr); } +bool +RequestHandler::HasTable(const std::string &table_name) { + bool has_table = false; + BaseTaskPtr task_ptr = HasTableTask::Create(table_name, has_table); + RequestScheduler::ExecTask(task_ptr); + + return has_table; +} + void RequestHandler::DeleteTable(const std::string &table_name) { BaseTaskPtr task_ptr = DeleteTableTask::Create(table_name); diff --git a/cpp/src/server/RequestHandler.h b/cpp/src/server/RequestHandler.h index 8f3eca576b..e736b7593f 100644 --- a/cpp/src/server/RequestHandler.h +++ b/cpp/src/server/RequestHandler.h @@ -19,16 +19,28 @@ public: RequestHandler(); /** - * @brief Create table method - * - * This method is used to create table - * - * @param param, use to provide table information to be created. - * - * - * @param param - */ - void CreateTable(const ::milvus::thrift::TableSchema& param); + * @brief Create table method + * + * This method is used to create table + * + * @param param, use to provide table information to be created. + * + * + * @param param + */ + void CreateTable(const ::milvus::thrift::TableSchema ¶m); + + /** + * @brief Test table existence method + * + * This method is used to test table existence. + * + * @param table_name, table name is going to be tested. + * + * + * @param table_name + */ + bool HasTable(const std::string &table_name); /** * @brief Delete table method diff --git a/cpp/src/server/RequestTask.cpp b/cpp/src/server/RequestTask.cpp index dbe4e6a740..b645d15a4e 100644 --- a/cpp/src/server/RequestTask.cpp +++ b/cpp/src/server/RequestTask.cpp @@ -70,7 +70,7 @@ namespace { uint64_t vec_dim = record.vector_data.size()/sizeof(double);//how many double value? if(vec_dim != dimension) { SERVER_LOG_ERROR << "Invalid vector dimension: " << vec_dim - << " vs. group dimension:" << dimension; + << " vs. table dimension:" << dimension; error_code = SERVER_INVALID_VECTOR_DIMENSION; return error_code; } @@ -233,6 +233,44 @@ ServerError DescribeTableTask::OnExecute() { return SERVER_SUCCESS; } +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +HasTableTask::HasTableTask(const std::string& table_name, bool& has_table) + : BaseTask(DDL_DML_TASK_GROUP), + table_name_(table_name), + has_table_(has_table) { + +} + +BaseTaskPtr HasTableTask::Create(const std::string& table_name, bool& has_table) { + return std::shared_ptr(new HasTableTask(table_name, has_table)); +} + +ServerError HasTableTask::OnExecute() { + try { + TimeRecorder rc("HasTableTask"); + + //step 1: check arguments + if (table_name_.empty()) { + error_code_ = SERVER_INVALID_ARGUMENT; + error_msg_ = "Table name cannot be empty"; + SERVER_LOG_ERROR << error_msg_; + return error_code_; + } + + //step 2: check table existence + engine::Status stat = DBWrapper::DB()->HasTable(table_name_, has_table_); + + rc.Elapse("totally cost"); + } catch (std::exception& ex) { + error_code_ = SERVER_UNEXPECTED_ERROR; + error_msg_ = ex.what(); + SERVER_LOG_ERROR << error_msg_; + return error_code_; + } + + return SERVER_SUCCESS; +} + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// DeleteTableTask::DeleteTableTask(const std::string& table_name) : BaseTask(DDL_DML_TASK_GROUP), @@ -240,8 +278,8 @@ DeleteTableTask::DeleteTableTask(const std::string& table_name) } -BaseTaskPtr DeleteTableTask::Create(const std::string& group_id) { - return std::shared_ptr(new DeleteTableTask(group_id)); +BaseTaskPtr DeleteTableTask::Create(const std::string& table_name) { + return std::shared_ptr(new DeleteTableTask(table_name)); } ServerError DeleteTableTask::OnExecute() { diff --git a/cpp/src/server/RequestTask.h b/cpp/src/server/RequestTask.h index b4ddc69726..3061b3b75d 100644 --- a/cpp/src/server/RequestTask.h +++ b/cpp/src/server/RequestTask.h @@ -33,6 +33,22 @@ private: const ::milvus::thrift::TableSchema& schema_; }; +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +class HasTableTask : public BaseTask { +public: + static BaseTaskPtr Create(const std::string& table_name, bool& has_table); + +protected: + HasTableTask(const std::string& table_name, bool& has_table); + + ServerError OnExecute() override; + + +private: + std::string table_name_; + bool& has_table_; +}; + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// class DescribeTableTask : public BaseTask { public: diff --git a/cpp/src/thrift/gen-cpp/MilvusService.cpp b/cpp/src/thrift/gen-cpp/MilvusService.cpp index 7e0a120bc0..7a591276f6 100644 --- a/cpp/src/thrift/gen-cpp/MilvusService.cpp +++ b/cpp/src/thrift/gen-cpp/MilvusService.cpp @@ -196,6 +196,213 @@ uint32_t MilvusService_CreateTable_presult::read(::apache::thrift::protocol::TPr } +MilvusService_HasTable_args::~MilvusService_HasTable_args() throw() { +} + + +uint32_t MilvusService_HasTable_args::read(::apache::thrift::protocol::TProtocol* iprot) { + + ::apache::thrift::protocol::TInputRecursionTracker tracker(*iprot); + uint32_t xfer = 0; + std::string fname; + ::apache::thrift::protocol::TType ftype; + int16_t fid; + + xfer += iprot->readStructBegin(fname); + + using ::apache::thrift::protocol::TProtocolException; + + + while (true) + { + xfer += iprot->readFieldBegin(fname, ftype, fid); + if (ftype == ::apache::thrift::protocol::T_STOP) { + break; + } + switch (fid) + { + case 2: + if (ftype == ::apache::thrift::protocol::T_STRING) { + xfer += iprot->readString(this->table_name); + this->__isset.table_name = true; + } else { + xfer += iprot->skip(ftype); + } + break; + default: + xfer += iprot->skip(ftype); + break; + } + xfer += iprot->readFieldEnd(); + } + + xfer += iprot->readStructEnd(); + + return xfer; +} + +uint32_t MilvusService_HasTable_args::write(::apache::thrift::protocol::TProtocol* oprot) const { + uint32_t xfer = 0; + ::apache::thrift::protocol::TOutputRecursionTracker tracker(*oprot); + xfer += oprot->writeStructBegin("MilvusService_HasTable_args"); + + xfer += oprot->writeFieldBegin("table_name", ::apache::thrift::protocol::T_STRING, 2); + xfer += oprot->writeString(this->table_name); + xfer += oprot->writeFieldEnd(); + + xfer += oprot->writeFieldStop(); + xfer += oprot->writeStructEnd(); + return xfer; +} + + +MilvusService_HasTable_pargs::~MilvusService_HasTable_pargs() throw() { +} + + +uint32_t MilvusService_HasTable_pargs::write(::apache::thrift::protocol::TProtocol* oprot) const { + uint32_t xfer = 0; + ::apache::thrift::protocol::TOutputRecursionTracker tracker(*oprot); + xfer += oprot->writeStructBegin("MilvusService_HasTable_pargs"); + + xfer += oprot->writeFieldBegin("table_name", ::apache::thrift::protocol::T_STRING, 2); + xfer += oprot->writeString((*(this->table_name))); + xfer += oprot->writeFieldEnd(); + + xfer += oprot->writeFieldStop(); + xfer += oprot->writeStructEnd(); + return xfer; +} + + +MilvusService_HasTable_result::~MilvusService_HasTable_result() throw() { +} + + +uint32_t MilvusService_HasTable_result::read(::apache::thrift::protocol::TProtocol* iprot) { + + ::apache::thrift::protocol::TInputRecursionTracker tracker(*iprot); + uint32_t xfer = 0; + std::string fname; + ::apache::thrift::protocol::TType ftype; + int16_t fid; + + xfer += iprot->readStructBegin(fname); + + using ::apache::thrift::protocol::TProtocolException; + + + while (true) + { + xfer += iprot->readFieldBegin(fname, ftype, fid); + if (ftype == ::apache::thrift::protocol::T_STOP) { + break; + } + switch (fid) + { + case 0: + if (ftype == ::apache::thrift::protocol::T_BOOL) { + xfer += iprot->readBool(this->success); + this->__isset.success = true; + } else { + xfer += iprot->skip(ftype); + } + break; + case 1: + if (ftype == ::apache::thrift::protocol::T_STRUCT) { + xfer += this->e.read(iprot); + this->__isset.e = true; + } else { + xfer += iprot->skip(ftype); + } + break; + default: + xfer += iprot->skip(ftype); + break; + } + xfer += iprot->readFieldEnd(); + } + + xfer += iprot->readStructEnd(); + + return xfer; +} + +uint32_t MilvusService_HasTable_result::write(::apache::thrift::protocol::TProtocol* oprot) const { + + uint32_t xfer = 0; + + xfer += oprot->writeStructBegin("MilvusService_HasTable_result"); + + if (this->__isset.success) { + xfer += oprot->writeFieldBegin("success", ::apache::thrift::protocol::T_BOOL, 0); + xfer += oprot->writeBool(this->success); + xfer += oprot->writeFieldEnd(); + } else if (this->__isset.e) { + xfer += oprot->writeFieldBegin("e", ::apache::thrift::protocol::T_STRUCT, 1); + xfer += this->e.write(oprot); + xfer += oprot->writeFieldEnd(); + } + xfer += oprot->writeFieldStop(); + xfer += oprot->writeStructEnd(); + return xfer; +} + + +MilvusService_HasTable_presult::~MilvusService_HasTable_presult() throw() { +} + + +uint32_t MilvusService_HasTable_presult::read(::apache::thrift::protocol::TProtocol* iprot) { + + ::apache::thrift::protocol::TInputRecursionTracker tracker(*iprot); + uint32_t xfer = 0; + std::string fname; + ::apache::thrift::protocol::TType ftype; + int16_t fid; + + xfer += iprot->readStructBegin(fname); + + using ::apache::thrift::protocol::TProtocolException; + + + while (true) + { + xfer += iprot->readFieldBegin(fname, ftype, fid); + if (ftype == ::apache::thrift::protocol::T_STOP) { + break; + } + switch (fid) + { + case 0: + if (ftype == ::apache::thrift::protocol::T_BOOL) { + xfer += iprot->readBool((*(this->success))); + this->__isset.success = true; + } else { + xfer += iprot->skip(ftype); + } + break; + case 1: + if (ftype == ::apache::thrift::protocol::T_STRUCT) { + xfer += this->e.read(iprot); + this->__isset.e = true; + } else { + xfer += iprot->skip(ftype); + } + break; + default: + xfer += iprot->skip(ftype); + break; + } + xfer += iprot->readFieldEnd(); + } + + xfer += iprot->readStructEnd(); + + return xfer; +} + + MilvusService_DeleteTable_args::~MilvusService_DeleteTable_args() throw() { } @@ -2290,6 +2497,67 @@ void MilvusServiceClient::recv_CreateTable() return; } +bool MilvusServiceClient::HasTable(const std::string& table_name) +{ + send_HasTable(table_name); + return recv_HasTable(); +} + +void MilvusServiceClient::send_HasTable(const std::string& table_name) +{ + int32_t cseqid = 0; + oprot_->writeMessageBegin("HasTable", ::apache::thrift::protocol::T_CALL, cseqid); + + MilvusService_HasTable_pargs args; + args.table_name = &table_name; + args.write(oprot_); + + oprot_->writeMessageEnd(); + oprot_->getTransport()->writeEnd(); + oprot_->getTransport()->flush(); +} + +bool MilvusServiceClient::recv_HasTable() +{ + + int32_t rseqid = 0; + std::string fname; + ::apache::thrift::protocol::TMessageType mtype; + + iprot_->readMessageBegin(fname, mtype, rseqid); + if (mtype == ::apache::thrift::protocol::T_EXCEPTION) { + ::apache::thrift::TApplicationException x; + x.read(iprot_); + iprot_->readMessageEnd(); + iprot_->getTransport()->readEnd(); + throw x; + } + if (mtype != ::apache::thrift::protocol::T_REPLY) { + iprot_->skip(::apache::thrift::protocol::T_STRUCT); + iprot_->readMessageEnd(); + iprot_->getTransport()->readEnd(); + } + if (fname.compare("HasTable") != 0) { + iprot_->skip(::apache::thrift::protocol::T_STRUCT); + iprot_->readMessageEnd(); + iprot_->getTransport()->readEnd(); + } + bool _return; + MilvusService_HasTable_presult result; + result.success = &_return; + result.read(iprot_); + iprot_->readMessageEnd(); + iprot_->getTransport()->readEnd(); + + if (result.__isset.success) { + return _return; + } + if (result.__isset.e) { + throw result.e; + } + throw ::apache::thrift::TApplicationException(::apache::thrift::TApplicationException::MISSING_RESULT, "HasTable failed: unknown result"); +} + void MilvusServiceClient::DeleteTable(const std::string& table_name) { send_DeleteTable(table_name); @@ -2855,6 +3123,63 @@ void MilvusServiceProcessor::process_CreateTable(int32_t seqid, ::apache::thrift } } +void MilvusServiceProcessor::process_HasTable(int32_t seqid, ::apache::thrift::protocol::TProtocol* iprot, ::apache::thrift::protocol::TProtocol* oprot, void* callContext) +{ + void* ctx = NULL; + if (this->eventHandler_.get() != NULL) { + ctx = this->eventHandler_->getContext("MilvusService.HasTable", callContext); + } + ::apache::thrift::TProcessorContextFreer freer(this->eventHandler_.get(), ctx, "MilvusService.HasTable"); + + if (this->eventHandler_.get() != NULL) { + this->eventHandler_->preRead(ctx, "MilvusService.HasTable"); + } + + MilvusService_HasTable_args args; + args.read(iprot); + iprot->readMessageEnd(); + uint32_t bytes = iprot->getTransport()->readEnd(); + + if (this->eventHandler_.get() != NULL) { + this->eventHandler_->postRead(ctx, "MilvusService.HasTable", bytes); + } + + MilvusService_HasTable_result result; + try { + result.success = iface_->HasTable(args.table_name); + result.__isset.success = true; + } catch (Exception &e) { + result.e = e; + result.__isset.e = true; + } catch (const std::exception& e) { + if (this->eventHandler_.get() != NULL) { + this->eventHandler_->handlerError(ctx, "MilvusService.HasTable"); + } + + ::apache::thrift::TApplicationException x(e.what()); + oprot->writeMessageBegin("HasTable", ::apache::thrift::protocol::T_EXCEPTION, seqid); + x.write(oprot); + oprot->writeMessageEnd(); + oprot->getTransport()->writeEnd(); + oprot->getTransport()->flush(); + return; + } + + if (this->eventHandler_.get() != NULL) { + this->eventHandler_->preWrite(ctx, "MilvusService.HasTable"); + } + + oprot->writeMessageBegin("HasTable", ::apache::thrift::protocol::T_REPLY, seqid); + result.write(oprot); + oprot->writeMessageEnd(); + bytes = oprot->getTransport()->writeEnd(); + oprot->getTransport()->flush(); + + if (this->eventHandler_.get() != NULL) { + this->eventHandler_->postWrite(ctx, "MilvusService.HasTable", bytes); + } +} + void MilvusServiceProcessor::process_DeleteTable(int32_t seqid, ::apache::thrift::protocol::TProtocol* iprot, ::apache::thrift::protocol::TProtocol* oprot, void* callContext) { void* ctx = NULL; @@ -3399,6 +3724,94 @@ void MilvusServiceConcurrentClient::recv_CreateTable(const int32_t seqid) } // end while(true) } +bool MilvusServiceConcurrentClient::HasTable(const std::string& table_name) +{ + int32_t seqid = send_HasTable(table_name); + return recv_HasTable(seqid); +} + +int32_t MilvusServiceConcurrentClient::send_HasTable(const std::string& table_name) +{ + int32_t cseqid = this->sync_.generateSeqId(); + ::apache::thrift::async::TConcurrentSendSentry sentry(&this->sync_); + oprot_->writeMessageBegin("HasTable", ::apache::thrift::protocol::T_CALL, cseqid); + + MilvusService_HasTable_pargs args; + args.table_name = &table_name; + args.write(oprot_); + + oprot_->writeMessageEnd(); + oprot_->getTransport()->writeEnd(); + oprot_->getTransport()->flush(); + + sentry.commit(); + return cseqid; +} + +bool MilvusServiceConcurrentClient::recv_HasTable(const int32_t seqid) +{ + + int32_t rseqid = 0; + std::string fname; + ::apache::thrift::protocol::TMessageType mtype; + + // the read mutex gets dropped and reacquired as part of waitForWork() + // The destructor of this sentry wakes up other clients + ::apache::thrift::async::TConcurrentRecvSentry sentry(&this->sync_, seqid); + + while(true) { + if(!this->sync_.getPending(fname, mtype, rseqid)) { + iprot_->readMessageBegin(fname, mtype, rseqid); + } + if(seqid == rseqid) { + if (mtype == ::apache::thrift::protocol::T_EXCEPTION) { + ::apache::thrift::TApplicationException x; + x.read(iprot_); + iprot_->readMessageEnd(); + iprot_->getTransport()->readEnd(); + sentry.commit(); + throw x; + } + if (mtype != ::apache::thrift::protocol::T_REPLY) { + iprot_->skip(::apache::thrift::protocol::T_STRUCT); + iprot_->readMessageEnd(); + iprot_->getTransport()->readEnd(); + } + if (fname.compare("HasTable") != 0) { + iprot_->skip(::apache::thrift::protocol::T_STRUCT); + iprot_->readMessageEnd(); + iprot_->getTransport()->readEnd(); + + // in a bad state, don't commit + using ::apache::thrift::protocol::TProtocolException; + throw TProtocolException(TProtocolException::INVALID_DATA); + } + bool _return; + MilvusService_HasTable_presult result; + result.success = &_return; + result.read(iprot_); + iprot_->readMessageEnd(); + iprot_->getTransport()->readEnd(); + + if (result.__isset.success) { + sentry.commit(); + return _return; + } + if (result.__isset.e) { + sentry.commit(); + throw result.e; + } + // in a bad state, don't commit + throw ::apache::thrift::TApplicationException(::apache::thrift::TApplicationException::MISSING_RESULT, "HasTable failed: unknown result"); + } + // seqid != rseqid + this->sync_.updatePending(fname, mtype, rseqid); + + // this will temporarily unlock the readMutex, and let other clients get work done + this->sync_.waitForWork(seqid); + } // end while(true) +} + void MilvusServiceConcurrentClient::DeleteTable(const std::string& table_name) { int32_t seqid = send_DeleteTable(table_name); diff --git a/cpp/src/thrift/gen-cpp/MilvusService.h b/cpp/src/thrift/gen-cpp/MilvusService.h index 4c1183de60..3868681463 100644 --- a/cpp/src/thrift/gen-cpp/MilvusService.h +++ b/cpp/src/thrift/gen-cpp/MilvusService.h @@ -34,6 +34,18 @@ class MilvusServiceIf { */ virtual void CreateTable(const TableSchema& param) = 0; + /** + * @brief Test table existence method + * + * This method is used to test table existence. + * + * @param table_name, table name is going to be tested. + * + * + * @param table_name + */ + virtual bool HasTable(const std::string& table_name) = 0; + /** * @brief Delete table method * @@ -178,6 +190,10 @@ class MilvusServiceNull : virtual public MilvusServiceIf { void CreateTable(const TableSchema& /* param */) { return; } + bool HasTable(const std::string& /* table_name */) { + bool _return = false; + return _return; + } void DeleteTable(const std::string& /* table_name */) { return; } @@ -309,6 +325,118 @@ class MilvusService_CreateTable_presult { }; +typedef struct _MilvusService_HasTable_args__isset { + _MilvusService_HasTable_args__isset() : table_name(false) {} + bool table_name :1; +} _MilvusService_HasTable_args__isset; + +class MilvusService_HasTable_args { + public: + + MilvusService_HasTable_args(const MilvusService_HasTable_args&); + MilvusService_HasTable_args& operator=(const MilvusService_HasTable_args&); + MilvusService_HasTable_args() : table_name() { + } + + virtual ~MilvusService_HasTable_args() throw(); + std::string table_name; + + _MilvusService_HasTable_args__isset __isset; + + void __set_table_name(const std::string& val); + + bool operator == (const MilvusService_HasTable_args & rhs) const + { + if (!(table_name == rhs.table_name)) + return false; + return true; + } + bool operator != (const MilvusService_HasTable_args &rhs) const { + return !(*this == rhs); + } + + bool operator < (const MilvusService_HasTable_args & ) const; + + uint32_t read(::apache::thrift::protocol::TProtocol* iprot); + uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const; + +}; + + +class MilvusService_HasTable_pargs { + public: + + + virtual ~MilvusService_HasTable_pargs() throw(); + const std::string* table_name; + + uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const; + +}; + +typedef struct _MilvusService_HasTable_result__isset { + _MilvusService_HasTable_result__isset() : success(false), e(false) {} + bool success :1; + bool e :1; +} _MilvusService_HasTable_result__isset; + +class MilvusService_HasTable_result { + public: + + MilvusService_HasTable_result(const MilvusService_HasTable_result&); + MilvusService_HasTable_result& operator=(const MilvusService_HasTable_result&); + MilvusService_HasTable_result() : success(0) { + } + + virtual ~MilvusService_HasTable_result() throw(); + bool success; + Exception e; + + _MilvusService_HasTable_result__isset __isset; + + void __set_success(const bool val); + + void __set_e(const Exception& val); + + bool operator == (const MilvusService_HasTable_result & rhs) const + { + if (!(success == rhs.success)) + return false; + if (!(e == rhs.e)) + return false; + return true; + } + bool operator != (const MilvusService_HasTable_result &rhs) const { + return !(*this == rhs); + } + + bool operator < (const MilvusService_HasTable_result & ) const; + + uint32_t read(::apache::thrift::protocol::TProtocol* iprot); + uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const; + +}; + +typedef struct _MilvusService_HasTable_presult__isset { + _MilvusService_HasTable_presult__isset() : success(false), e(false) {} + bool success :1; + bool e :1; +} _MilvusService_HasTable_presult__isset; + +class MilvusService_HasTable_presult { + public: + + + virtual ~MilvusService_HasTable_presult() throw(); + bool* success; + Exception e; + + _MilvusService_HasTable_presult__isset __isset; + + uint32_t read(::apache::thrift::protocol::TProtocol* iprot); + +}; + typedef struct _MilvusService_DeleteTable_args__isset { _MilvusService_DeleteTable_args__isset() : table_name(false) {} bool table_name :1; @@ -1269,6 +1397,9 @@ class MilvusServiceClient : virtual public MilvusServiceIf { void CreateTable(const TableSchema& param); void send_CreateTable(const TableSchema& param); void recv_CreateTable(); + bool HasTable(const std::string& table_name); + void send_HasTable(const std::string& table_name); + bool recv_HasTable(); void DeleteTable(const std::string& table_name); void send_DeleteTable(const std::string& table_name); void recv_DeleteTable(); @@ -1309,6 +1440,7 @@ class MilvusServiceProcessor : public ::apache::thrift::TDispatchProcessor { typedef std::map ProcessMap; ProcessMap processMap_; void process_CreateTable(int32_t seqid, ::apache::thrift::protocol::TProtocol* iprot, ::apache::thrift::protocol::TProtocol* oprot, void* callContext); + void process_HasTable(int32_t seqid, ::apache::thrift::protocol::TProtocol* iprot, ::apache::thrift::protocol::TProtocol* oprot, void* callContext); void process_DeleteTable(int32_t seqid, ::apache::thrift::protocol::TProtocol* iprot, ::apache::thrift::protocol::TProtocol* oprot, void* callContext); void process_AddVector(int32_t seqid, ::apache::thrift::protocol::TProtocol* iprot, ::apache::thrift::protocol::TProtocol* oprot, void* callContext); void process_SearchVector(int32_t seqid, ::apache::thrift::protocol::TProtocol* iprot, ::apache::thrift::protocol::TProtocol* oprot, void* callContext); @@ -1321,6 +1453,7 @@ class MilvusServiceProcessor : public ::apache::thrift::TDispatchProcessor { MilvusServiceProcessor(::apache::thrift::stdcxx::shared_ptr iface) : iface_(iface) { processMap_["CreateTable"] = &MilvusServiceProcessor::process_CreateTable; + processMap_["HasTable"] = &MilvusServiceProcessor::process_HasTable; processMap_["DeleteTable"] = &MilvusServiceProcessor::process_DeleteTable; processMap_["AddVector"] = &MilvusServiceProcessor::process_AddVector; processMap_["SearchVector"] = &MilvusServiceProcessor::process_SearchVector; @@ -1366,6 +1499,15 @@ class MilvusServiceMultiface : virtual public MilvusServiceIf { ifaces_[i]->CreateTable(param); } + bool HasTable(const std::string& table_name) { + size_t sz = ifaces_.size(); + size_t i = 0; + for (; i < (sz - 1); ++i) { + ifaces_[i]->HasTable(table_name); + } + return ifaces_[i]->HasTable(table_name); + } + void DeleteTable(const std::string& table_name) { size_t sz = ifaces_.size(); size_t i = 0; @@ -1477,6 +1619,9 @@ class MilvusServiceConcurrentClient : virtual public MilvusServiceIf { void CreateTable(const TableSchema& param); int32_t send_CreateTable(const TableSchema& param); void recv_CreateTable(const int32_t seqid); + bool HasTable(const std::string& table_name); + int32_t send_HasTable(const std::string& table_name); + bool recv_HasTable(const int32_t seqid); void DeleteTable(const std::string& table_name); int32_t send_DeleteTable(const std::string& table_name); void recv_DeleteTable(const int32_t seqid); diff --git a/cpp/src/thrift/gen-cpp/MilvusService_server.skeleton.cpp b/cpp/src/thrift/gen-cpp/MilvusService_server.skeleton.cpp index 55d73b9642..ecb22c0b62 100644 --- a/cpp/src/thrift/gen-cpp/MilvusService_server.skeleton.cpp +++ b/cpp/src/thrift/gen-cpp/MilvusService_server.skeleton.cpp @@ -35,6 +35,21 @@ class MilvusServiceHandler : virtual public MilvusServiceIf { printf("CreateTable\n"); } + /** + * @brief Test table existence method + * + * This method is used to test table existence. + * + * @param table_name, table name is going to be tested. + * + * + * @param table_name + */ + bool HasTable(const std::string& table_name) { + // Your implementation goes here + printf("HasTable\n"); + } + /** * @brief Delete table method * diff --git a/cpp/src/thrift/milvus.thrift b/cpp/src/thrift/milvus.thrift index 2936b85a5c..48116256e8 100644 --- a/cpp/src/thrift/milvus.thrift +++ b/cpp/src/thrift/milvus.thrift @@ -80,6 +80,16 @@ service MilvusService { */ void CreateTable(2: TableSchema param) throws(1: Exception e); + /** + * @brief Test table existence method + * + * This method is used to test table existence. + * + * @param table_name, table name is going to be tested. + * + */ + bool HasTable(2: string table_name) throws(1: Exception e); + /** * @brief Delete table method From 8622b12aa6723340d34bbf4e573c1998ac775d44 Mon Sep 17 00:00:00 2001 From: groot Date: Thu, 27 Jun 2019 18:03:52 +0800 Subject: [PATCH 5/6] MS-126 Add more error code Former-commit-id: ab57dcf1a8449e9cf2fc6440b308647f4d866777 --- cpp/CHANGELOG.md | 1 + cpp/src/db/DBMetaImpl.cpp | 12 +- cpp/src/server/RequestScheduler.cpp | 52 +++--- cpp/src/server/RequestScheduler.h | 2 + cpp/src/server/RequestTask.cpp | 235 ++++++++++-------------- cpp/src/thrift/gen-cpp/milvus_types.cpp | 34 +++- cpp/src/thrift/gen-cpp/milvus_types.h | 26 ++- cpp/src/thrift/milvus.thrift | 14 ++ cpp/src/utils/Error.h | 41 +++-- cpp/unittest/db/meta_tests.cpp | 70 +++---- 10 files changed, 261 insertions(+), 226 deletions(-) diff --git a/cpp/CHANGELOG.md b/cpp/CHANGELOG.md index 2bb18211a6..6c33a9eb2f 100644 --- a/cpp/CHANGELOG.md +++ b/cpp/CHANGELOG.md @@ -23,6 +23,7 @@ Please mark all change in change log and use the ticket from JIRA. - MS-115 - Change is_startup of metric_config switch from true to on - MS-122 - Archive criteria config - MS-124 - HasTable interface +- MS-126 - Add more error code ## New Feature diff --git a/cpp/src/db/DBMetaImpl.cpp b/cpp/src/db/DBMetaImpl.cpp index 1a8419eac2..d834d6cff2 100644 --- a/cpp/src/db/DBMetaImpl.cpp +++ b/cpp/src/db/DBMetaImpl.cpp @@ -192,9 +192,11 @@ Status DBMetaImpl::CreateTable(TableSchema &table_schema) { auto table = ConnectorPtr->select(columns(&TableSchema::state_), where(c(&TableSchema::table_id_) == table_schema.table_id_)); if (table.size() == 1) { - std::string msg = (TableSchema::TO_DELETE == std::get<0>(table[0])) ? - "Table already exists and it is in delete state, please wait a second" : "Table already exists"; - return Status::Error(msg); + if(TableSchema::TO_DELETE == std::get<0>(table[0])) { + return Status::Error("Table already exists and it is in delete state, please wait a second"); + } else { + return Status::OK();//table already exists, no error + } } } @@ -328,7 +330,7 @@ Status DBMetaImpl::HasTable(const std::string &table_id, bool &has_or_not) { } } catch (std::exception &e) { - HandleException("Encounter exception when lookup table", e); + return HandleException("Encounter exception when lookup table", e); } return Status::OK(); @@ -358,7 +360,7 @@ Status DBMetaImpl::AllTables(std::vector& table_schema_array) { } } catch (std::exception &e) { - HandleException("Encounter exception when lookup all tables", e); + return HandleException("Encounter exception when lookup all tables", e); } return Status::OK(); diff --git a/cpp/src/server/RequestScheduler.cpp b/cpp/src/server/RequestScheduler.cpp index 99ae36701e..36df155b62 100644 --- a/cpp/src/server/RequestScheduler.cpp +++ b/cpp/src/server/RequestScheduler.cpp @@ -18,35 +18,35 @@ using namespace ::milvus; namespace { const std::map &ErrorMap() { static const std::map code_map = { - {SERVER_UNEXPECTED_ERROR, thrift::ErrorCode::ILLEGAL_ARGUMENT}, - {SERVER_NULL_POINTER, thrift::ErrorCode::ILLEGAL_ARGUMENT}, + {SERVER_UNEXPECTED_ERROR, thrift::ErrorCode::UNEXPECTED_ERROR}, + {SERVER_UNSUPPORTED_ERROR, thrift::ErrorCode::UNEXPECTED_ERROR}, + {SERVER_NULL_POINTER, thrift::ErrorCode::UNEXPECTED_ERROR}, {SERVER_INVALID_ARGUMENT, thrift::ErrorCode::ILLEGAL_ARGUMENT}, - {SERVER_FILE_NOT_FOUND, thrift::ErrorCode::ILLEGAL_ARGUMENT}, - {SERVER_NOT_IMPLEMENT, thrift::ErrorCode::ILLEGAL_ARGUMENT}, - {SERVER_BLOCKING_QUEUE_EMPTY, thrift::ErrorCode::ILLEGAL_ARGUMENT}, + {SERVER_FILE_NOT_FOUND, thrift::ErrorCode::FILE_NOT_FOUND}, + {SERVER_NOT_IMPLEMENT, thrift::ErrorCode::UNEXPECTED_ERROR}, + {SERVER_BLOCKING_QUEUE_EMPTY, thrift::ErrorCode::UNEXPECTED_ERROR}, + {SERVER_CANNOT_CREATE_FOLDER, thrift::ErrorCode::CANNOT_CREATE_FOLDER}, + {SERVER_CANNOT_CREATE_FILE, thrift::ErrorCode::CANNOT_CREATE_FILE}, + {SERVER_CANNOT_DELETE_FOLDER, thrift::ErrorCode::CANNOT_DELETE_FOLDER}, + {SERVER_CANNOT_DELETE_FILE, thrift::ErrorCode::CANNOT_DELETE_FILE}, {SERVER_TABLE_NOT_EXIST, thrift::ErrorCode::TABLE_NOT_EXISTS}, + {SERVER_INVALID_TABLE_NAME, thrift::ErrorCode::ILLEGAL_TABLE_NAME}, + {SERVER_INVALID_TABLE_DIMENSION, thrift::ErrorCode::ILLEGAL_DIMENSION}, {SERVER_INVALID_TIME_RANGE, thrift::ErrorCode::ILLEGAL_RANGE}, {SERVER_INVALID_VECTOR_DIMENSION, thrift::ErrorCode::ILLEGAL_DIMENSION}, + + {SERVER_INVALID_INDEX_TYPE, thrift::ErrorCode::ILLEGAL_INDEX_TYPE}, + {SERVER_INVALID_ROWRECORD, thrift::ErrorCode::ILLEGAL_ROWRECORD}, + {SERVER_INVALID_ROWRECORD_ARRAY, thrift::ErrorCode::ILLEGAL_ROWRECORD}, + {SERVER_INVALID_TOPK, thrift::ErrorCode::ILLEGAL_TOPK}, + {SERVER_ILLEGAL_VECTOR_ID, thrift::ErrorCode::ILLEGAL_VECTOR_ID}, + {SERVER_ILLEGAL_SEARCH_RESULT, thrift::ErrorCode::ILLEGAL_SEARCH_RESULT}, + {SERVER_CACHE_ERROR, thrift::ErrorCode::CACHE_FAILED}, + {DB_META_TRANSACTION_FAILED, thrift::ErrorCode::META_FAILED}, }; return code_map; } - - const std::map &ErrorMessage() { - static const std::map msg_map = { - {SERVER_UNEXPECTED_ERROR, "unexpected error occurs"}, - {SERVER_NULL_POINTER, "null pointer error"}, - {SERVER_INVALID_ARGUMENT, "invalid argument"}, - {SERVER_FILE_NOT_FOUND, "file not found"}, - {SERVER_NOT_IMPLEMENT, "not implemented"}, - {SERVER_BLOCKING_QUEUE_EMPTY, "queue empty"}, - {SERVER_TABLE_NOT_EXIST, "table not exist"}, - {SERVER_INVALID_TIME_RANGE, "invalid time range"}, - {SERVER_INVALID_VECTOR_DIMENSION, "invalid vector dimension"}, - }; - - return msg_map; - } } //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -69,6 +69,14 @@ ServerError BaseTask::Execute() { return error_code_; } +ServerError BaseTask::SetError(ServerError error_code, const std::string& error_msg) { + error_code_ = error_code; + error_msg_ = error_msg; + + SERVER_LOG_ERROR << error_msg_; + return error_code_; +} + ServerError BaseTask::WaitToFinish() { std::unique_lock lock(finish_mtx_); finish_cond_.wait(lock, [this] { return done_; }); @@ -102,7 +110,7 @@ void RequestScheduler::ExecTask(BaseTaskPtr& task_ptr) { ex.__set_code(ErrorMap().at(err)); std::string msg = task_ptr->ErrorMsg(); if(msg.empty()){ - msg = ErrorMessage().at(err); + msg = "Error message not set"; } ex.__set_reason(msg); throw ex; diff --git a/cpp/src/server/RequestScheduler.h b/cpp/src/server/RequestScheduler.h index b7562cc367..d4b1e1c826 100644 --- a/cpp/src/server/RequestScheduler.h +++ b/cpp/src/server/RequestScheduler.h @@ -34,6 +34,8 @@ public: protected: virtual ServerError OnExecute() = 0; + ServerError SetError(ServerError error_code, const std::string& msg); + protected: mutable std::mutex finish_mtx_; std::condition_variable finish_cond_; diff --git a/cpp/src/server/RequestTask.cpp b/cpp/src/server/RequestTask.cpp index b645d15a4e..5bf2165cfe 100644 --- a/cpp/src/server/RequestTask.cpp +++ b/cpp/src/server/RequestTask.cpp @@ -53,26 +53,27 @@ namespace { return map_type[type]; } - ServerError + void ConvertRowRecordToFloatArray(const std::vector& record_array, uint64_t dimension, - std::vector& float_array) { - ServerError error_code; + std::vector& float_array, + ServerError& error_code, + std::string& error_msg) { uint64_t vec_count = record_array.size(); float_array.resize(vec_count*dimension);//allocate enough memory for(uint64_t i = 0; i < vec_count; i++) { const auto& record = record_array[i]; if(record.vector_data.empty()) { - error_code = SERVER_INVALID_ARGUMENT; - SERVER_LOG_ERROR << "No vector provided in record"; - return error_code; + error_code = SERVER_INVALID_ROWRECORD; + error_msg = "Rowrecord float array is empty"; + return; } uint64_t vec_dim = record.vector_data.size()/sizeof(double);//how many double value? if(vec_dim != dimension) { - SERVER_LOG_ERROR << "Invalid vector dimension: " << vec_dim - << " vs. table dimension:" << dimension; error_code = SERVER_INVALID_VECTOR_DIMENSION; - return error_code; + error_msg = "Invalid rowrecord dimension: " + std::to_string(vec_dim) + + " vs. table dimension:" + std::to_string(dimension); + return; } //convert double array to float array(thrift has no float type) @@ -81,30 +82,29 @@ namespace { float_array[i*vec_dim + d] = (float)(d_p[d]); } } - - return SERVER_SUCCESS; } static constexpr long DAY_SECONDS = 86400; - ServerError + void ConvertTimeRangeToDBDates(const std::vector &range_array, - std::vector& dates) { + std::vector& dates, + ServerError& error_code, + std::string& error_msg) { dates.clear(); - ServerError error_code; for(auto& range : range_array) { time_t tt_start, tt_end; tm tm_start, tm_end; if(!CommonUtil::TimeStrToTime(range.start_value, tt_start, tm_start)){ error_code = SERVER_INVALID_TIME_RANGE; - SERVER_LOG_ERROR << "Invalid time range: " << range.start_value; - return error_code; + error_msg = "Invalid time range: " + range.start_value; + return; } if(!CommonUtil::TimeStrToTime(range.end_value, tt_end, tm_end)){ error_code = SERVER_INVALID_TIME_RANGE; - SERVER_LOG_ERROR << "Invalid time range: " << range.end_value; - return error_code; + error_msg = "Invalid time range: " + range.start_value; + return; } long days = (tt_end > tt_start) ? (tt_end - tt_start)/DAY_SECONDS : (tt_start - tt_end)/DAY_SECONDS; @@ -117,8 +117,6 @@ namespace { dates.push_back(date); } } - - return SERVER_SUCCESS; } } @@ -138,19 +136,16 @@ ServerError CreateTableTask::OnExecute() { try { //step 1: check arguments - if(schema_.table_name.empty() || schema_.dimension <= 0) { - error_code_ = SERVER_INVALID_ARGUMENT; - error_msg_ = "Invalid table name or dimension"; - SERVER_LOG_ERROR << error_msg_; - return error_code_; + if(schema_.table_name.empty()) { + return SetError(SERVER_INVALID_TABLE_NAME, "Empty table name"); + } + if(schema_.dimension <= 0) { + return SetError(SERVER_INVALID_TABLE_DIMENSION, "Invalid table dimension: " + std::to_string(schema_.dimension)); } engine::EngineType engine_type = EngineType(schema_.index_type); if(engine_type == engine::EngineType::INVALID) { - error_code_ = SERVER_INVALID_ARGUMENT; - error_msg_ = "Invalid index type"; - SERVER_LOG_ERROR << error_msg_; - return error_code_; + return SetError(SERVER_INVALID_INDEX_TYPE, "Invalid index type: " + std::to_string(schema_.index_type)); } //step 2: construct table schema @@ -163,17 +158,11 @@ ServerError CreateTableTask::OnExecute() { //step 3: create table engine::Status stat = DBWrapper::DB()->CreateTable(table_info); if(!stat.ok()) {//table could exist - error_code_ = SERVER_UNEXPECTED_ERROR; - error_msg_ = "Engine failed: " + stat.ToString(); - SERVER_LOG_ERROR << error_msg_; - return error_code_; + return SetError(DB_META_TRANSACTION_FAILED, "Engine failed: " + stat.ToString()); } } catch (std::exception& ex) { - error_code_ = SERVER_UNEXPECTED_ERROR; - error_msg_ = ex.what(); - SERVER_LOG_ERROR << error_msg_; - return error_code_; + return SetError(SERVER_UNEXPECTED_ERROR, ex.what()); } rc.Record("done"); @@ -199,10 +188,7 @@ ServerError DescribeTableTask::OnExecute() { try { //step 1: check arguments if(table_name_.empty()) { - error_code_ = SERVER_INVALID_ARGUMENT; - error_msg_ = "Table name cannot be empty"; - SERVER_LOG_ERROR << error_msg_; - return error_code_; + return SetError(SERVER_INVALID_TABLE_NAME, "Empty table name"); } //step 2: get table info @@ -210,10 +196,7 @@ ServerError DescribeTableTask::OnExecute() { table_info.table_id_ = table_name_; engine::Status stat = DBWrapper::DB()->DescribeTable(table_info); if(!stat.ok()) { - error_code_ = SERVER_TABLE_NOT_EXIST; - error_msg_ = "Engine failed: " + stat.ToString(); - SERVER_LOG_ERROR << error_msg_; - return error_code_; + return SetError(DB_META_TRANSACTION_FAILED, "Engine failed: " + stat.ToString()); } schema_.table_name = table_info.table_id_; @@ -222,10 +205,7 @@ ServerError DescribeTableTask::OnExecute() { schema_.store_raw_vector = table_info.store_raw_data_; } catch (std::exception& ex) { - error_code_ = SERVER_UNEXPECTED_ERROR; - error_msg_ = ex.what(); - SERVER_LOG_ERROR << error_msg_; - return SERVER_UNEXPECTED_ERROR; + return SetError(SERVER_UNEXPECTED_ERROR, ex.what()); } rc.Record("done"); @@ -250,22 +230,19 @@ ServerError HasTableTask::OnExecute() { TimeRecorder rc("HasTableTask"); //step 1: check arguments - if (table_name_.empty()) { - error_code_ = SERVER_INVALID_ARGUMENT; - error_msg_ = "Table name cannot be empty"; - SERVER_LOG_ERROR << error_msg_; - return error_code_; + if(table_name_.empty()) { + return SetError(SERVER_INVALID_TABLE_NAME, "Empty table name"); } //step 2: check table existence engine::Status stat = DBWrapper::DB()->HasTable(table_name_, has_table_); + if(!stat.ok()) { + return SetError(DB_META_TRANSACTION_FAILED, "Engine failed: " + stat.ToString()); + } rc.Elapse("totally cost"); } catch (std::exception& ex) { - error_code_ = SERVER_UNEXPECTED_ERROR; - error_msg_ = ex.what(); - SERVER_LOG_ERROR << error_msg_; - return error_code_; + return SetError(SERVER_UNEXPECTED_ERROR, ex.what()); } return SERVER_SUCCESS; @@ -288,10 +265,7 @@ ServerError DeleteTableTask::OnExecute() { //step 1: check arguments if (table_name_.empty()) { - error_code_ = SERVER_INVALID_ARGUMENT; - error_msg_ = "Table name cannot be empty"; - SERVER_LOG_ERROR << error_msg_; - return error_code_; + return SetError(SERVER_INVALID_TABLE_NAME, "Empty table name"); } //step 2: check table existence @@ -299,10 +273,11 @@ ServerError DeleteTableTask::OnExecute() { table_info.table_id_ = table_name_; engine::Status stat = DBWrapper::DB()->DescribeTable(table_info); if(!stat.ok()) { - error_code_ = SERVER_TABLE_NOT_EXIST; - error_msg_ = "Engine failed: " + stat.ToString(); - SERVER_LOG_ERROR << error_msg_; - return error_code_; + if(stat.IsNotFound()) { + return SetError(SERVER_TABLE_NOT_EXIST, "Table " + table_name_ + " not exists"); + } else { + return SetError(DB_META_TRANSACTION_FAILED, "Engine failed: " + stat.ToString()); + } } rc.Record("check validation"); @@ -311,17 +286,13 @@ ServerError DeleteTableTask::OnExecute() { std::vector dates; stat = DBWrapper::DB()->DeleteTable(table_name_, dates); if(!stat.ok()) { - SERVER_LOG_ERROR << "Engine failed: " << stat.ToString(); - return SERVER_UNEXPECTED_ERROR; + return SetError(DB_META_TRANSACTION_FAILED, "Engine failed: " + stat.ToString()); } rc.Record("deleta table"); rc.Elapse("totally cost"); } catch (std::exception& ex) { - error_code_ = SERVER_UNEXPECTED_ERROR; - error_msg_ = ex.what(); - SERVER_LOG_ERROR << error_msg_; - return error_code_; + return SetError(SERVER_UNEXPECTED_ERROR, ex.what()); } return SERVER_SUCCESS; @@ -342,10 +313,7 @@ ServerError ShowTablesTask::OnExecute() { std::vector schema_array; engine::Status stat = DBWrapper::DB()->AllTables(schema_array); if(!stat.ok()) { - error_code_ = SERVER_UNEXPECTED_ERROR; - error_msg_ = "Engine failed: " + stat.ToString(); - SERVER_LOG_ERROR << error_msg_; - return error_code_; + return SetError(DB_META_TRANSACTION_FAILED, "Engine failed: " + stat.ToString()); } tables_.clear(); @@ -379,17 +347,11 @@ ServerError AddVectorTask::OnExecute() { //step 1: check arguments if (table_name_.empty()) { - error_code_ = SERVER_INVALID_ARGUMENT; - error_msg_ = "Table name cannot be empty"; - SERVER_LOG_ERROR << error_msg_; - return error_code_; + return SetError(SERVER_INVALID_TABLE_NAME, "Empty table name"); } if(record_array_.empty()) { - error_code_ = SERVER_INVALID_ARGUMENT; - error_msg_ = "Row record array is empty"; - SERVER_LOG_ERROR << error_msg_; - return error_code_; + return SetError(SERVER_INVALID_ROWRECORD_ARRAY, "Row record array is empty"); } //step 2: check table existence @@ -397,20 +359,22 @@ ServerError AddVectorTask::OnExecute() { table_info.table_id_ = table_name_; engine::Status stat = DBWrapper::DB()->DescribeTable(table_info); if(!stat.ok()) { - error_code_ = SERVER_TABLE_NOT_EXIST; - error_msg_ = "Engine failed: " + stat.ToString(); - SERVER_LOG_ERROR << error_msg_; - return error_code_; + if(stat.IsNotFound()) { + return SetError(SERVER_TABLE_NOT_EXIST, "Table " + table_name_ + " not exists"); + } else { + return SetError(DB_META_TRANSACTION_FAILED, "Engine failed: " + stat.ToString()); + } } rc.Record("check validation"); //step 3: prepare float data std::vector vec_f; - error_code_ = ConvertRowRecordToFloatArray(record_array_, table_info.dimension_, vec_f); - if(error_code_ != SERVER_SUCCESS) { - error_msg_ = "Invalid row record data"; - return error_code_; + ServerError error_code = SERVER_SUCCESS; + std::string error_msg; + ConvertRowRecordToFloatArray(record_array_, table_info.dimension_, vec_f, error_code, error_msg); + if(error_code != SERVER_SUCCESS) { + return SetError(error_code, error_msg); } rc.Record("prepare vectors data"); @@ -420,25 +384,20 @@ ServerError AddVectorTask::OnExecute() { stat = DBWrapper::DB()->InsertVectors(table_name_, vec_count, vec_f.data(), record_ids_); rc.Record("add vectors to engine"); if(!stat.ok()) { - error_code_ = SERVER_UNEXPECTED_ERROR; - error_msg_ = "Engine failed: " + stat.ToString(); - SERVER_LOG_ERROR << error_msg_; - return error_code_; + return SetError(SERVER_CACHE_ERROR, "Cache error: " + stat.ToString()); } if(record_ids_.size() != vec_count) { - SERVER_LOG_ERROR << "Vector ID not returned"; - return SERVER_UNEXPECTED_ERROR; + std::string msg = "Add " + std::to_string(vec_count) + " vectors but only return " + + std::to_string(record_ids_.size()) + " id"; + return SetError(SERVER_ILLEGAL_VECTOR_ID, msg); } rc.Record("do insert"); rc.Elapse("totally cost"); } catch (std::exception& ex) { - error_code_ = SERVER_UNEXPECTED_ERROR; - error_msg_ = ex.what(); - SERVER_LOG_ERROR << error_msg_; - return error_code_; + return SetError(SERVER_UNEXPECTED_ERROR, ex.what()); } return SERVER_SUCCESS; @@ -477,17 +436,14 @@ ServerError SearchVectorTask::OnExecute() { //step 1: check arguments if (table_name_.empty()) { - error_code_ = SERVER_INVALID_ARGUMENT; - error_msg_ = "Table name cannot be empty"; - SERVER_LOG_ERROR << error_msg_; - return error_code_; + return SetError(SERVER_INVALID_TABLE_NAME, "Empty table name"); } - if(top_k_ <= 0 || record_array_.empty()) { - error_code_ = SERVER_INVALID_ARGUMENT; - error_msg_ = "Invalid topk value, or query record array is empty"; - SERVER_LOG_ERROR << error_msg_; - return error_code_; + if(top_k_ <= 0) { + return SetError(SERVER_INVALID_TOPK, "Invalid topk: " + std::to_string(top_k_)); + } + if(record_array_.empty()) { + return SetError(SERVER_INVALID_ROWRECORD_ARRAY, "Row record array is empty"); } //step 2: check table existence @@ -495,28 +451,29 @@ ServerError SearchVectorTask::OnExecute() { table_info.table_id_ = table_name_; engine::Status stat = DBWrapper::DB()->DescribeTable(table_info); if(!stat.ok()) { - error_code_ = SERVER_TABLE_NOT_EXIST; - error_msg_ = "Engine failed: " + stat.ToString(); - SERVER_LOG_ERROR << error_msg_; - return error_code_; + if(stat.IsNotFound()) { + return SetError(SERVER_TABLE_NOT_EXIST, "Table " + table_name_ + " not exists"); + } else { + return SetError(DB_META_TRANSACTION_FAILED, "Engine failed: " + stat.ToString()); + } } //step 3: check date range, and convert to db dates std::vector dates; - error_code_ = ConvertTimeRangeToDBDates(range_array_, dates); - if(error_code_ != SERVER_SUCCESS) { - error_msg_ = "Invalid query range"; - return error_code_; + ServerError error_code = SERVER_SUCCESS; + std::string error_msg; + ConvertTimeRangeToDBDates(range_array_, dates, error_code, error_msg); + if(error_code != SERVER_SUCCESS) { + return SetError(error_code, error_msg); } rc.Record("check validation"); //step 3: prepare float data std::vector vec_f; - error_code_ = ConvertRowRecordToFloatArray(record_array_, table_info.dimension_, vec_f); - if(error_code_ != SERVER_SUCCESS) { - error_msg_ = "Invalid row record data"; - return error_code_; + ConvertRowRecordToFloatArray(record_array_, table_info.dimension_, vec_f, error_code, error_msg); + if(error_code != SERVER_SUCCESS) { + return SetError(error_code, error_msg); } rc.Record("prepare vector data"); @@ -533,13 +490,17 @@ ServerError SearchVectorTask::OnExecute() { rc.Record("search vectors from engine"); if(!stat.ok()) { - SERVER_LOG_ERROR << "Engine failed: " << stat.ToString(); - return SERVER_UNEXPECTED_ERROR; + return SetError(DB_META_TRANSACTION_FAILED, "Engine failed: " + stat.ToString()); + } + + if(results.empty()) { + return SERVER_SUCCESS; //empty table } if(results.size() != record_count) { - SERVER_LOG_ERROR << "Search result not returned"; - return SERVER_UNEXPECTED_ERROR; + std::string msg = "Search " + std::to_string(record_count) + " vectors but only return " + + std::to_string(results.size()) + " results"; + return SetError(SERVER_ILLEGAL_SEARCH_RESULT, msg); } rc.Record("do search"); @@ -564,10 +525,7 @@ ServerError SearchVectorTask::OnExecute() { rc.Elapse("totally cost"); } catch (std::exception& ex) { - error_code_ = SERVER_UNEXPECTED_ERROR; - error_msg_ = ex.what(); - SERVER_LOG_ERROR << error_msg_; - return error_code_; + return SetError(SERVER_UNEXPECTED_ERROR, ex.what()); } return SERVER_SUCCESS; @@ -591,20 +549,14 @@ ServerError GetTableRowCountTask::OnExecute() { //step 1: check arguments if (table_name_.empty()) { - error_code_ = SERVER_INVALID_ARGUMENT; - error_msg_ = "Table name cannot be empty"; - SERVER_LOG_ERROR << error_msg_; - return error_code_; + return SetError(SERVER_INVALID_TABLE_NAME, "Empty table name"); } //step 2: get row count uint64_t row_count = 0; engine::Status stat = DBWrapper::DB()->GetTableRowCount(table_name_, row_count); if (!stat.ok()) { - error_code_ = SERVER_UNEXPECTED_ERROR; - error_msg_ = "Engine failed: " + stat.ToString(); - SERVER_LOG_ERROR << error_msg_; - return error_code_; + return SetError(DB_META_TRANSACTION_FAILED, "Engine failed: " + stat.ToString()); } row_count_ = (int64_t) row_count; @@ -612,10 +564,7 @@ ServerError GetTableRowCountTask::OnExecute() { rc.Elapse("totally cost"); } catch (std::exception& ex) { - error_code_ = SERVER_UNEXPECTED_ERROR; - error_msg_ = ex.what(); - SERVER_LOG_ERROR << error_msg_; - return error_code_; + return SetError(SERVER_UNEXPECTED_ERROR, ex.what()); } return SERVER_SUCCESS; diff --git a/cpp/src/thrift/gen-cpp/milvus_types.cpp b/cpp/src/thrift/gen-cpp/milvus_types.cpp index 203faa2e52..af77fda0ad 100644 --- a/cpp/src/thrift/gen-cpp/milvus_types.cpp +++ b/cpp/src/thrift/gen-cpp/milvus_types.cpp @@ -15,23 +15,51 @@ namespace milvus { namespace thrift { int _kErrorCodeValues[] = { ErrorCode::SUCCESS, + ErrorCode::UNEXPECTED_ERROR, ErrorCode::CONNECT_FAILED, ErrorCode::PERMISSION_DENIED, ErrorCode::TABLE_NOT_EXISTS, ErrorCode::ILLEGAL_ARGUMENT, ErrorCode::ILLEGAL_RANGE, - ErrorCode::ILLEGAL_DIMENSION + ErrorCode::ILLEGAL_DIMENSION, + ErrorCode::ILLEGAL_INDEX_TYPE, + ErrorCode::ILLEGAL_TABLE_NAME, + ErrorCode::ILLEGAL_TOPK, + ErrorCode::ILLEGAL_ROWRECORD, + ErrorCode::ILLEGAL_VECTOR_ID, + ErrorCode::ILLEGAL_SEARCH_RESULT, + ErrorCode::FILE_NOT_FOUND, + ErrorCode::META_FAILED, + ErrorCode::CACHE_FAILED, + ErrorCode::CANNOT_CREATE_FOLDER, + ErrorCode::CANNOT_CREATE_FILE, + ErrorCode::CANNOT_DELETE_FOLDER, + ErrorCode::CANNOT_DELETE_FILE }; const char* _kErrorCodeNames[] = { "SUCCESS", + "UNEXPECTED_ERROR", "CONNECT_FAILED", "PERMISSION_DENIED", "TABLE_NOT_EXISTS", "ILLEGAL_ARGUMENT", "ILLEGAL_RANGE", - "ILLEGAL_DIMENSION" + "ILLEGAL_DIMENSION", + "ILLEGAL_INDEX_TYPE", + "ILLEGAL_TABLE_NAME", + "ILLEGAL_TOPK", + "ILLEGAL_ROWRECORD", + "ILLEGAL_VECTOR_ID", + "ILLEGAL_SEARCH_RESULT", + "FILE_NOT_FOUND", + "META_FAILED", + "CACHE_FAILED", + "CANNOT_CREATE_FOLDER", + "CANNOT_CREATE_FILE", + "CANNOT_DELETE_FOLDER", + "CANNOT_DELETE_FILE" }; -const std::map _ErrorCode_VALUES_TO_NAMES(::apache::thrift::TEnumIterator(7, _kErrorCodeValues, _kErrorCodeNames), ::apache::thrift::TEnumIterator(-1, NULL, NULL)); +const std::map _ErrorCode_VALUES_TO_NAMES(::apache::thrift::TEnumIterator(21, _kErrorCodeValues, _kErrorCodeNames), ::apache::thrift::TEnumIterator(-1, NULL, NULL)); std::ostream& operator<<(std::ostream& out, const ErrorCode::type& val) { std::map::const_iterator it = _ErrorCode_VALUES_TO_NAMES.find(val); diff --git a/cpp/src/thrift/gen-cpp/milvus_types.h b/cpp/src/thrift/gen-cpp/milvus_types.h index ea4178aa99..8840e94fd9 100644 --- a/cpp/src/thrift/gen-cpp/milvus_types.h +++ b/cpp/src/thrift/gen-cpp/milvus_types.h @@ -23,12 +23,26 @@ namespace milvus { namespace thrift { struct ErrorCode { enum type { SUCCESS = 0, - CONNECT_FAILED = 1, - PERMISSION_DENIED = 2, - TABLE_NOT_EXISTS = 3, - ILLEGAL_ARGUMENT = 4, - ILLEGAL_RANGE = 5, - ILLEGAL_DIMENSION = 6 + UNEXPECTED_ERROR = 1, + CONNECT_FAILED = 2, + PERMISSION_DENIED = 3, + TABLE_NOT_EXISTS = 4, + ILLEGAL_ARGUMENT = 5, + ILLEGAL_RANGE = 6, + ILLEGAL_DIMENSION = 7, + ILLEGAL_INDEX_TYPE = 8, + ILLEGAL_TABLE_NAME = 9, + ILLEGAL_TOPK = 10, + ILLEGAL_ROWRECORD = 11, + ILLEGAL_VECTOR_ID = 12, + ILLEGAL_SEARCH_RESULT = 13, + FILE_NOT_FOUND = 14, + META_FAILED = 15, + CACHE_FAILED = 16, + CANNOT_CREATE_FOLDER = 17, + CANNOT_CREATE_FILE = 18, + CANNOT_DELETE_FOLDER = 19, + CANNOT_DELETE_FILE = 20 }; }; diff --git a/cpp/src/thrift/milvus.thrift b/cpp/src/thrift/milvus.thrift index 48116256e8..88ba223f02 100644 --- a/cpp/src/thrift/milvus.thrift +++ b/cpp/src/thrift/milvus.thrift @@ -15,12 +15,26 @@ namespace netcore milvus.thrift enum ErrorCode { SUCCESS = 0, + UNEXPECTED_ERROR, CONNECT_FAILED, PERMISSION_DENIED, TABLE_NOT_EXISTS, ILLEGAL_ARGUMENT, ILLEGAL_RANGE, ILLEGAL_DIMENSION, + ILLEGAL_INDEX_TYPE, + ILLEGAL_TABLE_NAME, + ILLEGAL_TOPK, + ILLEGAL_ROWRECORD, + ILLEGAL_VECTOR_ID, + ILLEGAL_SEARCH_RESULT, + FILE_NOT_FOUND, + META_FAILED, + CACHE_FAILED, + CANNOT_CREATE_FOLDER, + CANNOT_CREATE_FILE, + CANNOT_DELETE_FOLDER, + CANNOT_DELETE_FILE, } exception Exception { diff --git a/cpp/src/utils/Error.h b/cpp/src/utils/Error.h index bfb19f47a9..8c4da70339 100644 --- a/cpp/src/utils/Error.h +++ b/cpp/src/utils/Error.h @@ -24,18 +24,35 @@ ToGlobalServerErrorCode(const ServerError error_code) { return SERVER_ERROR_CODE_BASE + error_code; } -constexpr ServerError SERVER_UNEXPECTED_ERROR = ToGlobalServerErrorCode(0x001); -constexpr ServerError SERVER_UNSUPPORTED_ERROR = ToGlobalServerErrorCode(0x002); -constexpr ServerError SERVER_NULL_POINTER = ToGlobalServerErrorCode(0x003); -constexpr ServerError SERVER_INVALID_ARGUMENT = ToGlobalServerErrorCode(0x004); -constexpr ServerError SERVER_FILE_NOT_FOUND = ToGlobalServerErrorCode(0x005); -constexpr ServerError SERVER_NOT_IMPLEMENT = ToGlobalServerErrorCode(0x006); -constexpr ServerError SERVER_BLOCKING_QUEUE_EMPTY = ToGlobalServerErrorCode(0x007); -constexpr ServerError SERVER_TABLE_NOT_EXIST = ToGlobalServerErrorCode(0x008); -constexpr ServerError SERVER_INVALID_TIME_RANGE = ToGlobalServerErrorCode(0x009); -constexpr ServerError SERVER_INVALID_VECTOR_DIMENSION = ToGlobalServerErrorCode(0x00a); -constexpr ServerError SERVER_LICENSE_VALIDATION_FAIL = ToGlobalServerErrorCode(0x00b); -constexpr ServerError SERVER_LICENSE_FILE_NOT_EXIST = ToGlobalServerErrorCode(0x00c); +constexpr ServerError SERVER_UNEXPECTED_ERROR = ToGlobalServerErrorCode(1); +constexpr ServerError SERVER_UNSUPPORTED_ERROR = ToGlobalServerErrorCode(2); +constexpr ServerError SERVER_NULL_POINTER = ToGlobalServerErrorCode(3); +constexpr ServerError SERVER_INVALID_ARGUMENT = ToGlobalServerErrorCode(4); +constexpr ServerError SERVER_FILE_NOT_FOUND = ToGlobalServerErrorCode(5); +constexpr ServerError SERVER_NOT_IMPLEMENT = ToGlobalServerErrorCode(6); +constexpr ServerError SERVER_BLOCKING_QUEUE_EMPTY = ToGlobalServerErrorCode(7); +constexpr ServerError SERVER_CANNOT_CREATE_FOLDER = ToGlobalServerErrorCode(8); +constexpr ServerError SERVER_CANNOT_CREATE_FILE = ToGlobalServerErrorCode(9); +constexpr ServerError SERVER_CANNOT_DELETE_FOLDER = ToGlobalServerErrorCode(10); +constexpr ServerError SERVER_CANNOT_DELETE_FILE = ToGlobalServerErrorCode(11); + +constexpr ServerError SERVER_TABLE_NOT_EXIST = ToGlobalServerErrorCode(100); +constexpr ServerError SERVER_INVALID_TABLE_NAME = ToGlobalServerErrorCode(101); +constexpr ServerError SERVER_INVALID_TABLE_DIMENSION = ToGlobalServerErrorCode(102); +constexpr ServerError SERVER_INVALID_TIME_RANGE = ToGlobalServerErrorCode(103); +constexpr ServerError SERVER_INVALID_VECTOR_DIMENSION = ToGlobalServerErrorCode(104); +constexpr ServerError SERVER_INVALID_INDEX_TYPE = ToGlobalServerErrorCode(105); +constexpr ServerError SERVER_INVALID_ROWRECORD = ToGlobalServerErrorCode(106); +constexpr ServerError SERVER_INVALID_ROWRECORD_ARRAY = ToGlobalServerErrorCode(107); +constexpr ServerError SERVER_INVALID_TOPK = ToGlobalServerErrorCode(108); +constexpr ServerError SERVER_ILLEGAL_VECTOR_ID = ToGlobalServerErrorCode(109); +constexpr ServerError SERVER_ILLEGAL_SEARCH_RESULT = ToGlobalServerErrorCode(110); +constexpr ServerError SERVER_CACHE_ERROR = ToGlobalServerErrorCode(111); + +constexpr ServerError SERVER_LICENSE_FILE_NOT_EXIST = ToGlobalServerErrorCode(500); +constexpr ServerError SERVER_LICENSE_VALIDATION_FAIL = ToGlobalServerErrorCode(501); + +constexpr ServerError DB_META_TRANSACTION_FAILED = ToGlobalServerErrorCode(1000); class ServerException : public std::exception { public: diff --git a/cpp/unittest/db/meta_tests.cpp b/cpp/unittest/db/meta_tests.cpp index d71a942d3e..9baef712ab 100644 --- a/cpp/unittest/db/meta_tests.cpp +++ b/cpp/unittest/db/meta_tests.cpp @@ -17,39 +17,39 @@ using namespace zilliz::milvus::engine; -TEST_F(MetaTest, GROUP_TEST) { - auto table_id = "meta_test_group"; +TEST_F(MetaTest, TABLE_TEST) { + auto table_id = "meta_test_table"; - meta::TableSchema group; - group.table_id_ = table_id; - auto status = impl_->CreateTable(group); + meta::TableSchema table; + table.table_id_ = table_id; + auto status = impl_->CreateTable(table); ASSERT_TRUE(status.ok()); - auto gid = group.id_; - group.id_ = -1; - status = impl_->DescribeTable(group); + auto gid = table.id_; + table.id_ = -1; + status = impl_->DescribeTable(table); ASSERT_TRUE(status.ok()); - ASSERT_EQ(group.id_, gid); - ASSERT_EQ(group.table_id_, table_id); + ASSERT_EQ(table.id_, gid); + ASSERT_EQ(table.table_id_, table_id); - group.table_id_ = "not_found"; - status = impl_->DescribeTable(group); + table.table_id_ = "not_found"; + status = impl_->DescribeTable(table); ASSERT_TRUE(!status.ok()); - group.table_id_ = table_id; - status = impl_->CreateTable(group); - ASSERT_TRUE(!status.ok()); + table.table_id_ = table_id; + status = impl_->CreateTable(table); + ASSERT_TRUE(status.ok()); } -TEST_F(MetaTest, table_file_TEST) { - auto table_id = "meta_test_group"; +TEST_F(MetaTest, TABLE_FILE_TEST) { + auto table_id = "meta_test_table"; - meta::TableSchema group; - group.table_id_ = table_id; - auto status = impl_->CreateTable(group); + meta::TableSchema table; + table.table_id_ = table_id; + auto status = impl_->CreateTable(table); meta::TableFileSchema table_file; - table_file.table_id_ = group.table_id_; + table_file.table_id_ = table.table_id_; status = impl_->CreateTableFile(table_file); ASSERT_TRUE(status.ok()); ASSERT_EQ(table_file.file_type_, meta::TableFileSchema::NEW); @@ -104,15 +104,15 @@ TEST_F(MetaTest, ARCHIVE_TEST_DAYS) { options.archive_conf = ArchiveConf("delete", ss.str()); auto impl = meta::DBMetaImpl(options); - auto table_id = "meta_test_group"; + auto table_id = "meta_test_table"; - meta::TableSchema group; - group.table_id_ = table_id; - auto status = impl.CreateTable(group); + meta::TableSchema table; + table.table_id_ = table_id; + auto status = impl.CreateTable(table); meta::TableFilesSchema files; meta::TableFileSchema table_file; - table_file.table_id_ = group.table_id_; + table_file.table_id_ = table.table_id_; auto cnt = 100; long ts = utils::GetMicroSecTimeStamp(); @@ -156,13 +156,13 @@ TEST_F(MetaTest, ARCHIVE_TEST_DISK) { auto impl = meta::DBMetaImpl(options); auto table_id = "meta_test_group"; - meta::TableSchema group; - group.table_id_ = table_id; - auto status = impl.CreateTable(group); + meta::TableSchema table; + table.table_id_ = table_id; + auto status = impl.CreateTable(table); meta::TableFilesSchema files; meta::TableFileSchema table_file; - table_file.table_id_ = group.table_id_; + table_file.table_id_ = table.table_id_; auto cnt = 10; auto each_size = 2UL; @@ -198,9 +198,9 @@ TEST_F(MetaTest, ARCHIVE_TEST_DISK) { TEST_F(MetaTest, TABLE_FILES_TEST) { auto table_id = "meta_test_group"; - meta::TableSchema group; - group.table_id_ = table_id; - auto status = impl_->CreateTable(group); + meta::TableSchema table; + table.table_id_ = table_id; + auto status = impl_->CreateTable(table); int new_files_cnt = 4; int raw_files_cnt = 5; @@ -208,7 +208,7 @@ TEST_F(MetaTest, TABLE_FILES_TEST) { int index_files_cnt = 7; meta::TableFileSchema table_file; - table_file.table_id_ = group.table_id_; + table_file.table_id_ = table.table_id_; for (auto i=0; iCreateTableFile(table_file); @@ -241,7 +241,7 @@ TEST_F(MetaTest, TABLE_FILES_TEST) { ASSERT_EQ(files.size(), to_index_files_cnt); meta::DatePartionedTableFilesSchema dated_files; - status = impl_->FilesToMerge(group.table_id_, dated_files); + status = impl_->FilesToMerge(table.table_id_, dated_files); ASSERT_TRUE(status.ok()); ASSERT_EQ(dated_files[table_file.date_].size(), raw_files_cnt); From 59c457cd5b1f5e82288fa340ee3b6a5879352fd1 Mon Sep 17 00:00:00 2001 From: groot Date: Thu, 27 Jun 2019 19:57:11 +0800 Subject: [PATCH 6/6] add more error handling Former-commit-id: 0105341107a2d54e23921b71168001cb5b218447 --- cpp/src/server/DBWrapper.cpp | 10 +++++++--- cpp/src/server/MilvusServer.cpp | 4 +++- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/cpp/src/server/DBWrapper.cpp b/cpp/src/server/DBWrapper.cpp index 2a583a660d..7892a57f2b 100644 --- a/cpp/src/server/DBWrapper.cpp +++ b/cpp/src/server/DBWrapper.cpp @@ -37,12 +37,16 @@ DBWrapper::DBWrapper() { opt.meta.archive_conf.SetCriterias(criterial); //create db root folder - CommonUtil::CreateDirectory(opt.meta.path); + ServerError err = CommonUtil::CreateDirectory(opt.meta.path); + if(err != SERVER_SUCCESS) { + std::cout << "ERROR! Failed to create database root path: " << opt.meta.path << std::endl; + kill(0, SIGUSR1); + } zilliz::milvus::engine::DB::Open(opt, &db_); if(db_ == nullptr) { - SERVER_LOG_ERROR << "Failed to open db"; - throw ServerException(SERVER_NULL_POINTER, "Failed to open db"); + std::cout << "ERROR! Failed to open database" << std::endl; + kill(0, SIGUSR1); } } diff --git a/cpp/src/server/MilvusServer.cpp b/cpp/src/server/MilvusServer.cpp index 5e42eb3d7a..322460968f 100644 --- a/cpp/src/server/MilvusServer.cpp +++ b/cpp/src/server/MilvusServer.cpp @@ -23,6 +23,7 @@ #include #include +#include namespace zilliz { namespace milvus { @@ -92,7 +93,8 @@ MilvusServer::StartService() { return; } } catch (apache::thrift::TException& ex) { - //SERVER_LOG_ERROR << "Server encounter exception: " << ex.what(); + std::cout << "ERROR! " << ex.what() << std::endl; + kill(0, SIGUSR1); } }