diff --git a/CHANGELOG.md b/CHANGELOG.md index 0e3284f9f6..1e978754cd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -46,6 +46,7 @@ Please mark all change in change log and use the ticket from JIRA. - \#606 - No log generated during building index with CPU - \#631 - FAISS isn't compiled with O3 option - \#649 - Typo "partiton" should be "partition" +- \#654 - Random crash when frequently insert vector one by one ## Feature - \#12 - Pure CPU version for Milvus diff --git a/core/src/db/DBImpl.cpp b/core/src/db/DBImpl.cpp index 88689857ca..b51ea698b9 100644 --- a/core/src/db/DBImpl.cpp +++ b/core/src/db/DBImpl.cpp @@ -52,7 +52,7 @@ constexpr uint64_t METRIC_ACTION_INTERVAL = 1; constexpr uint64_t COMPACT_ACTION_INTERVAL = 1; constexpr uint64_t INDEX_ACTION_INTERVAL = 1; -static const Status SHUTDOWN_ERROR = Status(DB_ERROR, "Milsvus server is shutdown!"); +static const Status SHUTDOWN_ERROR = Status(DB_ERROR, "Milvus server is shutdown!"); void TraverseFiles(const meta::DatePartionedTableFilesSchema& date_files, meta::TableFilesSchema& files_array) { diff --git a/core/src/db/insert/MemManagerImpl.cpp b/core/src/db/insert/MemManagerImpl.cpp index 69c3397eb9..c7ca3b3c33 100644 --- a/core/src/db/insert/MemManagerImpl.cpp +++ b/core/src/db/insert/MemManagerImpl.cpp @@ -116,6 +116,7 @@ MemManagerImpl::EraseMemVector(const std::string& table_id) { size_t MemManagerImpl::GetCurrentMutableMem() { size_t total_mem = 0; + std::unique_lock lock(mutex_); for (auto& kv : mem_id_map_) { auto memTable = kv.second; total_mem += memTable->GetCurrentMem(); @@ -126,6 +127,7 @@ MemManagerImpl::GetCurrentMutableMem() { size_t MemManagerImpl::GetCurrentImmutableMem() { size_t total_mem = 0; + std::unique_lock lock(serialization_mtx_); for (auto& mem_table : immu_mem_list_) { total_mem += mem_table->GetCurrentMem(); } diff --git a/core/src/server/grpc_impl/request/CmdRequest.cpp b/core/src/server/grpc_impl/request/CmdRequest.cpp index 4af9db03ef..67e41acd02 100644 --- a/core/src/server/grpc_impl/request/CmdRequest.cpp +++ b/core/src/server/grpc_impl/request/CmdRequest.cpp @@ -38,7 +38,7 @@ CmdRequest::Create(const std::string& cmd, std::string& result) { Status CmdRequest::OnExecute() { std::string hdr = "CmdRequest(cmd=" + cmd_ + ")"; - TimeRecorder rc(hdr); + TimeRecorderAuto rc(hdr); if (cmd_ == "version") { result_ = MILVUS_VERSION; diff --git a/core/src/server/grpc_impl/request/CountTableRequest.cpp b/core/src/server/grpc_impl/request/CountTableRequest.cpp index b90a33bf61..b8bad6ca74 100644 --- a/core/src/server/grpc_impl/request/CountTableRequest.cpp +++ b/core/src/server/grpc_impl/request/CountTableRequest.cpp @@ -40,7 +40,7 @@ Status CountTableRequest::OnExecute() { try { std::string hdr = "CountTableRequest(table=" + table_name_ + ")"; - TimeRecorder rc(hdr); + TimeRecorderAuto rc(hdr); // step 1: check arguments auto status = ValidationUtil::ValidateTableName(table_name_); @@ -60,8 +60,6 @@ CountTableRequest::OnExecute() { } row_count_ = static_cast(row_count); - - rc.ElapseFromBegin("total cost"); } catch (std::exception& ex) { return Status(SERVER_UNEXPECTED_ERROR, ex.what()); } diff --git a/core/src/server/grpc_impl/request/CreateIndexRequest.cpp b/core/src/server/grpc_impl/request/CreateIndexRequest.cpp index c7628048ff..9cb7a4c9e7 100644 --- a/core/src/server/grpc_impl/request/CreateIndexRequest.cpp +++ b/core/src/server/grpc_impl/request/CreateIndexRequest.cpp @@ -45,7 +45,7 @@ Status CreateIndexRequest::OnExecute() { try { std::string hdr = "CreateIndexRequest(table=" + index_param_->table_name() + ")"; - TimeRecorder rc(hdr); + TimeRecorderAuto rc(hdr); // step 1: check arguments std::string table_name_ = index_param_->table_name(); @@ -83,8 +83,6 @@ CreateIndexRequest::OnExecute() { if (!status.ok()) { return status; } - - rc.ElapseFromBegin("totally cost"); } catch (std::exception& ex) { return Status(SERVER_UNEXPECTED_ERROR, ex.what()); } diff --git a/core/src/server/grpc_impl/request/CreatePartitionRequest.cpp b/core/src/server/grpc_impl/request/CreatePartitionRequest.cpp index 1a18cdfcad..892e4ef357 100644 --- a/core/src/server/grpc_impl/request/CreatePartitionRequest.cpp +++ b/core/src/server/grpc_impl/request/CreatePartitionRequest.cpp @@ -46,7 +46,7 @@ CreatePartitionRequest::OnExecute() { std::string hdr = "CreatePartitionRequest(table=" + partition_param_->table_name() + ", partition_name=" + partition_param_->partition_name() + ", partition_tag=" + partition_param_->tag() + ")"; - TimeRecorder rc(hdr); + TimeRecorderAuto rc(hdr); try { // step 1: check arguments @@ -79,8 +79,6 @@ CreatePartitionRequest::OnExecute() { return Status(SERVER_UNEXPECTED_ERROR, ex.what()); } - rc.ElapseFromBegin("totally cost"); - return Status::OK(); } diff --git a/core/src/server/grpc_impl/request/CreateTableRequest.cpp b/core/src/server/grpc_impl/request/CreateTableRequest.cpp index 55d953aa6f..0223890616 100644 --- a/core/src/server/grpc_impl/request/CreateTableRequest.cpp +++ b/core/src/server/grpc_impl/request/CreateTableRequest.cpp @@ -45,7 +45,7 @@ Status CreateTableRequest::OnExecute() { std::string hdr = "CreateTableRequest(table=" + schema_->table_name() + ", dimension=" + std::to_string(schema_->dimension()) + ")"; - TimeRecorder rc(hdr); + TimeRecorderAuto rc(hdr); try { // step 1: check arguments @@ -89,8 +89,6 @@ CreateTableRequest::OnExecute() { return Status(SERVER_UNEXPECTED_ERROR, ex.what()); } - rc.ElapseFromBegin("totally cost"); - return Status::OK(); } diff --git a/core/src/server/grpc_impl/request/DeleteByDateRequest.cpp b/core/src/server/grpc_impl/request/DeleteByDateRequest.cpp index 4a4d414803..d2a5f91ae3 100644 --- a/core/src/server/grpc_impl/request/DeleteByDateRequest.cpp +++ b/core/src/server/grpc_impl/request/DeleteByDateRequest.cpp @@ -46,7 +46,7 @@ DeleteByDateRequest::Create(const ::milvus::grpc::DeleteByDateParam* delete_by_r Status DeleteByDateRequest::OnExecute() { try { - TimeRecorder rc("DeleteByDateRequest"); + TimeRecorderAuto rc("DeleteByDateRequest"); // step 1: check arguments std::string table_name = delete_by_range_param_->table_name(); @@ -67,7 +67,7 @@ DeleteByDateRequest::OnExecute() { } } - rc.ElapseFromBegin("check validation"); + rc.RecordSection("check validation"); // step 3: check date range, and convert to db dates std::vector dates; diff --git a/core/src/server/grpc_impl/request/DescribeIndexRequest.cpp b/core/src/server/grpc_impl/request/DescribeIndexRequest.cpp index d57d47bf2c..a478f28e0a 100644 --- a/core/src/server/grpc_impl/request/DescribeIndexRequest.cpp +++ b/core/src/server/grpc_impl/request/DescribeIndexRequest.cpp @@ -40,7 +40,7 @@ Status DescribeIndexRequest::OnExecute() { try { std::string hdr = "DescribeIndexRequest(table=" + table_name_ + ")"; - TimeRecorder rc(hdr); + TimeRecorderAuto rc(hdr); // step 1: check arguments auto status = ValidationUtil::ValidateTableName(table_name_); @@ -58,8 +58,6 @@ DescribeIndexRequest::OnExecute() { index_param_->set_table_name(table_name_); index_param_->mutable_index()->set_index_type(index.engine_type_); index_param_->mutable_index()->set_nlist(index.nlist_); - - rc.ElapseFromBegin("totally cost"); } catch (std::exception& ex) { return Status(SERVER_UNEXPECTED_ERROR, ex.what()); } diff --git a/core/src/server/grpc_impl/request/DescribeTableRequest.cpp b/core/src/server/grpc_impl/request/DescribeTableRequest.cpp index 3bd97ef7e8..847f81dc19 100644 --- a/core/src/server/grpc_impl/request/DescribeTableRequest.cpp +++ b/core/src/server/grpc_impl/request/DescribeTableRequest.cpp @@ -39,7 +39,7 @@ DescribeTableRequest::Create(const std::string& table_name, ::milvus::grpc::Tabl Status DescribeTableRequest::OnExecute() { std::string hdr = "DescribeTableRequest(table=" + table_name_ + ")"; - TimeRecorder rc(hdr); + TimeRecorderAuto rc(hdr); try { // step 1: check arguments @@ -64,8 +64,6 @@ DescribeTableRequest::OnExecute() { return Status(SERVER_UNEXPECTED_ERROR, ex.what()); } - rc.ElapseFromBegin("totally cost"); - return Status::OK(); } diff --git a/core/src/server/grpc_impl/request/DropIndexRequest.cpp b/core/src/server/grpc_impl/request/DropIndexRequest.cpp index 619ea753ba..d573686f05 100644 --- a/core/src/server/grpc_impl/request/DropIndexRequest.cpp +++ b/core/src/server/grpc_impl/request/DropIndexRequest.cpp @@ -40,7 +40,7 @@ Status DropIndexRequest::OnExecute() { try { std::string hdr = "DropIndexRequest(table=" + table_name_ + ")"; - TimeRecorder rc(hdr); + TimeRecorderAuto rc(hdr); // step 1: check arguments auto status = ValidationUtil::ValidateTableName(table_name_); @@ -63,8 +63,6 @@ DropIndexRequest::OnExecute() { if (!status.ok()) { return status; } - - rc.ElapseFromBegin("totally cost"); } catch (std::exception& ex) { return Status(SERVER_UNEXPECTED_ERROR, ex.what()); } diff --git a/core/src/server/grpc_impl/request/DropPartitionRequest.cpp b/core/src/server/grpc_impl/request/DropPartitionRequest.cpp index 1ec189986a..96913adb9f 100644 --- a/core/src/server/grpc_impl/request/DropPartitionRequest.cpp +++ b/core/src/server/grpc_impl/request/DropPartitionRequest.cpp @@ -42,7 +42,7 @@ DropPartitionRequest::OnExecute() { std::string hdr = "DropPartitionRequest(table=" + partition_param_->table_name() + ", partition_name=" + partition_param_->partition_name() + ", partition_tag=" + partition_param_->tag() + ")"; - TimeRecorder rc(hdr); + TimeRecorderAuto rc(hdr); std::string table_name = partition_param_->table_name(); std::string partition_name = partition_param_->partition_name(); diff --git a/core/src/server/grpc_impl/request/DropTableRequest.cpp b/core/src/server/grpc_impl/request/DropTableRequest.cpp index 6f81a5b2a0..ee4989d6f8 100644 --- a/core/src/server/grpc_impl/request/DropTableRequest.cpp +++ b/core/src/server/grpc_impl/request/DropTableRequest.cpp @@ -61,7 +61,7 @@ DropTableRequest::OnExecute() { } } - rc.ElapseFromBegin("check validation"); + rc.RecordSection("check validation"); // step 3: Drop table std::vector dates; diff --git a/core/src/server/grpc_impl/request/HasTableRequest.cpp b/core/src/server/grpc_impl/request/HasTableRequest.cpp index 434580efdf..d171730073 100644 --- a/core/src/server/grpc_impl/request/HasTableRequest.cpp +++ b/core/src/server/grpc_impl/request/HasTableRequest.cpp @@ -40,7 +40,7 @@ Status HasTableRequest::OnExecute() { try { std::string hdr = "HasTableRequest(table=" + table_name_ + ")"; - TimeRecorder rc(hdr); + TimeRecorderAuto rc(hdr); // step 1: check arguments auto status = ValidationUtil::ValidateTableName(table_name_); @@ -53,8 +53,6 @@ HasTableRequest::OnExecute() { if (!status.ok()) { return status; } - - rc.ElapseFromBegin("totally cost"); } catch (std::exception& ex) { return Status(SERVER_UNEXPECTED_ERROR, ex.what()); } diff --git a/core/src/server/grpc_impl/request/InsertRequest.cpp b/core/src/server/grpc_impl/request/InsertRequest.cpp index df65c679ed..c178822468 100644 --- a/core/src/server/grpc_impl/request/InsertRequest.cpp +++ b/core/src/server/grpc_impl/request/InsertRequest.cpp @@ -122,8 +122,6 @@ InsertRequest::OnExecute() { table_info.dimension_ * sizeof(float)); } - rc.ElapseFromBegin("prepare vectors data"); - // step 5: insert vectors auto vec_count = static_cast(insert_param_->row_record_array_size()); std::vector vec_ids(insert_param_->row_id_array_size(), 0); @@ -133,9 +131,9 @@ InsertRequest::OnExecute() { memcpy(target_data, src_data, static_cast(sizeof(int64_t) * insert_param_->row_id_array_size())); } + rc.RecordSection("prepare vectors data"); status = DBWrapper::DB()->InsertVectors(insert_param_->table_name(), insert_param_->partition_tag(), vec_count, vec_f.data(), vec_ids); - rc.ElapseFromBegin("add vectors to engine"); if (!status.ok()) { return status; } diff --git a/core/src/server/grpc_impl/request/PreloadTableRequest.cpp b/core/src/server/grpc_impl/request/PreloadTableRequest.cpp index 3c46524afe..e659a9359d 100644 --- a/core/src/server/grpc_impl/request/PreloadTableRequest.cpp +++ b/core/src/server/grpc_impl/request/PreloadTableRequest.cpp @@ -40,7 +40,7 @@ Status PreloadTableRequest::OnExecute() { try { std::string hdr = "PreloadTableRequest(table=" + table_name_ + ")"; - TimeRecorder rc(hdr); + TimeRecorderAuto rc(hdr); // step 1: check arguments auto status = ValidationUtil::ValidateTableName(table_name_); @@ -53,8 +53,6 @@ PreloadTableRequest::OnExecute() { if (!status.ok()) { return status; } - - rc.ElapseFromBegin("totally cost"); } catch (std::exception& ex) { return Status(SERVER_UNEXPECTED_ERROR, ex.what()); } diff --git a/core/src/server/grpc_impl/request/ShowPartitionsRequest.cpp b/core/src/server/grpc_impl/request/ShowPartitionsRequest.cpp index 0d0809b171..9ba57785bd 100644 --- a/core/src/server/grpc_impl/request/ShowPartitionsRequest.cpp +++ b/core/src/server/grpc_impl/request/ShowPartitionsRequest.cpp @@ -41,7 +41,7 @@ ShowPartitionsRequest::Create(const std::string& table_name, ::milvus::grpc::Par Status ShowPartitionsRequest::OnExecute() { std::string hdr = "ShowPartitionsRequest(table=" + table_name_ + ")"; - TimeRecorder rc(hdr); + TimeRecorderAuto rc(hdr); auto status = ValidationUtil::ValidateTableName(table_name_); if (!status.ok()) { diff --git a/core/src/server/grpc_impl/request/ShowTablesRequest.cpp b/core/src/server/grpc_impl/request/ShowTablesRequest.cpp index 404e08197e..9ad46a4a83 100644 --- a/core/src/server/grpc_impl/request/ShowTablesRequest.cpp +++ b/core/src/server/grpc_impl/request/ShowTablesRequest.cpp @@ -38,7 +38,7 @@ ShowTablesRequest::Create(::milvus::grpc::TableNameList* table_name_list) { Status ShowTablesRequest::OnExecute() { - TimeRecorder rc("ShowTablesRequest"); + TimeRecorderAuto rc("ShowTablesRequest"); std::vector schema_array; auto statuts = DBWrapper::DB()->AllTables(schema_array);