admin/milvus
1
0
Fork 0
mirror of https://gitee.com/milvus-io/milvus.git synced 2025-12-29 06:55:27 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #655 from yhmo/0.6.0

Browse Source 
#654 Random crash when frequently insert vector one by one
This commit is contained in:
Jin Hai 2019-12-03 14:13:21 +08:00 committed by GitHub
commit be8a3e2af1
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
19 changed files with 21 additions and 38 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
CHANGELOG.md
View File

@ -46,6 +46,7 @@ Please mark all change in change log and use the ticket from JIRA.
- \#606 - No log generated during building index with CPU
- \#631 - FAISS isn't compiled with O3 option
- \#649 - Typo "partiton" should be "partition"
- \#654 - Random crash when frequently insert vector one by one
## Feature
- \#12 - Pure CPU version for Milvus

2
core/src/db/DBImpl.cpp
View File

@ -52,7 +52,7 @@ constexpr uint64_t METRIC_ACTION_INTERVAL = 1;
constexpr uint64_t COMPACT_ACTION_INTERVAL = 1;
constexpr uint64_t INDEX_ACTION_INTERVAL = 1;
static const Status SHUTDOWN_ERROR = Status(DB_ERROR, "Milsvus server is shutdown!");
static const Status SHUTDOWN_ERROR = Status(DB_ERROR, "Milvus server is shutdown!");
void
TraverseFiles(const meta::DatePartionedTableFilesSchema& date_files, meta::TableFilesSchema& files_array) {

2
core/src/db/insert/MemManagerImpl.cpp
View File

@ -116,6 +116,7 @@ MemManagerImpl::EraseMemVector(const std::string& table_id) {
size_t
MemManagerImpl::GetCurrentMutableMem() {
size_t total_mem = 0;
std::unique_lock<std::mutex> lock(mutex_);
for (auto& kv : mem_id_map_) {
auto memTable = kv.second;
total_mem += memTable->GetCurrentMem();
@ -126,6 +127,7 @@ MemManagerImpl::GetCurrentMutableMem() {
size_t
MemManagerImpl::GetCurrentImmutableMem() {
size_t total_mem = 0;
std::unique_lock<std::mutex> lock(serialization_mtx_);
for (auto& mem_table : immu_mem_list_) {
total_mem += mem_table->GetCurrentMem();
}

2
core/src/server/grpc_impl/request/CmdRequest.cpp
View File

@ -38,7 +38,7 @@ CmdRequest::Create(const std::string& cmd, std::string& result) {
Status
CmdRequest::OnExecute() {
std::string hdr = "CmdRequest(cmd=" + cmd_ + ")";
TimeRecorder rc(hdr);
TimeRecorderAuto rc(hdr);
if (cmd_ == "version") {
result_ = MILVUS_VERSION;

4
core/src/server/grpc_impl/request/CountTableRequest.cpp
View File

@ -40,7 +40,7 @@ Status
CountTableRequest::OnExecute() {
try {
std::string hdr = "CountTableRequest(table=" + table_name_ + ")";
TimeRecorder rc(hdr);
TimeRecorderAuto rc(hdr);
// step 1: check arguments
auto status = ValidationUtil::ValidateTableName(table_name_);
@ -60,8 +60,6 @@ CountTableRequest::OnExecute() {
}
row_count_ = static_cast<int64_t>(row_count);
rc.ElapseFromBegin("total cost");
} catch (std::exception& ex) {
return Status(SERVER_UNEXPECTED_ERROR, ex.what());
}

4
core/src/server/grpc_impl/request/CreateIndexRequest.cpp
View File

@ -45,7 +45,7 @@ Status
CreateIndexRequest::OnExecute() {
try {
std::string hdr = "CreateIndexRequest(table=" + index_param_->table_name() + ")";
TimeRecorder rc(hdr);
TimeRecorderAuto rc(hdr);
// step 1: check arguments
std::string table_name_ = index_param_->table_name();
@ -83,8 +83,6 @@ CreateIndexRequest::OnExecute() {
if (!status.ok()) {
return status;
}
rc.ElapseFromBegin("totally cost");
} catch (std::exception& ex) {
return Status(SERVER_UNEXPECTED_ERROR, ex.what());
}

4
core/src/server/grpc_impl/request/CreatePartitionRequest.cpp
View File

@ -46,7 +46,7 @@ CreatePartitionRequest::OnExecute() {
std::string hdr = "CreatePartitionRequest(table=" + partition_param_->table_name() +
", partition_name=" + partition_param_->partition_name() +
", partition_tag=" + partition_param_->tag() + ")";
TimeRecorder rc(hdr);
TimeRecorderAuto rc(hdr);
try {
// step 1: check arguments
@ -79,8 +79,6 @@ CreatePartitionRequest::OnExecute() {
return Status(SERVER_UNEXPECTED_ERROR, ex.what());
}
rc.ElapseFromBegin("totally cost");
return Status::OK();
}

4
core/src/server/grpc_impl/request/CreateTableRequest.cpp
View File

@ -45,7 +45,7 @@ Status
CreateTableRequest::OnExecute() {
std::string hdr = "CreateTableRequest(table=" + schema_->table_name() +
", dimension=" + std::to_string(schema_->dimension()) + ")";
TimeRecorder rc(hdr);
TimeRecorderAuto rc(hdr);
try {
// step 1: check arguments
@ -89,8 +89,6 @@ CreateTableRequest::OnExecute() {
return Status(SERVER_UNEXPECTED_ERROR, ex.what());
}
rc.ElapseFromBegin("totally cost");
return Status::OK();
}

4
core/src/server/grpc_impl/request/DeleteByDateRequest.cpp
View File

@ -46,7 +46,7 @@ DeleteByDateRequest::Create(const ::milvus::grpc::DeleteByDateParam* delete_by_r
Status
DeleteByDateRequest::OnExecute() {
try {
TimeRecorder rc("DeleteByDateRequest");
TimeRecorderAuto rc("DeleteByDateRequest");
// step 1: check arguments
std::string table_name = delete_by_range_param_->table_name();
@ -67,7 +67,7 @@ DeleteByDateRequest::OnExecute() {
}
}
rc.ElapseFromBegin("check validation");
rc.RecordSection("check validation");
// step 3: check date range, and convert to db dates
std::vector<DB_DATE> dates;

4
core/src/server/grpc_impl/request/DescribeIndexRequest.cpp
View File

@ -40,7 +40,7 @@ Status
DescribeIndexRequest::OnExecute() {
try {
std::string hdr = "DescribeIndexRequest(table=" + table_name_ + ")";
TimeRecorder rc(hdr);
TimeRecorderAuto rc(hdr);
// step 1: check arguments
auto status = ValidationUtil::ValidateTableName(table_name_);
@ -58,8 +58,6 @@ DescribeIndexRequest::OnExecute() {
index_param_->set_table_name(table_name_);
index_param_->mutable_index()->set_index_type(index.engine_type_);
index_param_->mutable_index()->set_nlist(index.nlist_);
rc.ElapseFromBegin("totally cost");
} catch (std::exception& ex) {
return Status(SERVER_UNEXPECTED_ERROR, ex.what());
}

4
core/src/server/grpc_impl/request/DescribeTableRequest.cpp
View File

@ -39,7 +39,7 @@ DescribeTableRequest::Create(const std::string& table_name, ::milvus::grpc::Tabl
Status
DescribeTableRequest::OnExecute() {
std::string hdr = "DescribeTableRequest(table=" + table_name_ + ")";
TimeRecorder rc(hdr);
TimeRecorderAuto rc(hdr);
try {
// step 1: check arguments
@ -64,8 +64,6 @@ DescribeTableRequest::OnExecute() {
return Status(SERVER_UNEXPECTED_ERROR, ex.what());
}
rc.ElapseFromBegin("totally cost");
return Status::OK();
}

4
core/src/server/grpc_impl/request/DropIndexRequest.cpp
View File

@ -40,7 +40,7 @@ Status
DropIndexRequest::OnExecute() {
try {
std::string hdr = "DropIndexRequest(table=" + table_name_ + ")";
TimeRecorder rc(hdr);
TimeRecorderAuto rc(hdr);
// step 1: check arguments
auto status = ValidationUtil::ValidateTableName(table_name_);
@ -63,8 +63,6 @@ DropIndexRequest::OnExecute() {
if (!status.ok()) {
return status;
}
rc.ElapseFromBegin("totally cost");
} catch (std::exception& ex) {
return Status(SERVER_UNEXPECTED_ERROR, ex.what());
}

2
core/src/server/grpc_impl/request/DropPartitionRequest.cpp
View File

@ -42,7 +42,7 @@ DropPartitionRequest::OnExecute() {
std::string hdr = "DropPartitionRequest(table=" + partition_param_->table_name() +
", partition_name=" + partition_param_->partition_name() +
", partition_tag=" + partition_param_->tag() + ")";
TimeRecorder rc(hdr);
TimeRecorderAuto rc(hdr);
std::string table_name = partition_param_->table_name();
std::string partition_name = partition_param_->partition_name();

2
core/src/server/grpc_impl/request/DropTableRequest.cpp
View File

@ -61,7 +61,7 @@ DropTableRequest::OnExecute() {
}
}
rc.ElapseFromBegin("check validation");
rc.RecordSection("check validation");
// step 3: Drop table
std::vector<DB_DATE> dates;

4
core/src/server/grpc_impl/request/HasTableRequest.cpp
View File

@ -40,7 +40,7 @@ Status
HasTableRequest::OnExecute() {
try {
std::string hdr = "HasTableRequest(table=" + table_name_ + ")";
TimeRecorder rc(hdr);
TimeRecorderAuto rc(hdr);
// step 1: check arguments
auto status = ValidationUtil::ValidateTableName(table_name_);
@ -53,8 +53,6 @@ HasTableRequest::OnExecute() {
if (!status.ok()) {
return status;
}
rc.ElapseFromBegin("totally cost");
} catch (std::exception& ex) {
return Status(SERVER_UNEXPECTED_ERROR, ex.what());
}

4
core/src/server/grpc_impl/request/InsertRequest.cpp
View File

@ -122,8 +122,6 @@ InsertRequest::OnExecute() {
table_info.dimension_ * sizeof(float));
}
rc.ElapseFromBegin("prepare vectors data");
// step 5: insert vectors
auto vec_count = static_cast<uint64_t>(insert_param_->row_record_array_size());
std::vector<int64_t> vec_ids(insert_param_->row_id_array_size(), 0);
@ -133,9 +131,9 @@ InsertRequest::OnExecute() {
memcpy(target_data, src_data, static_cast<size_t>(sizeof(int64_t) * insert_param_->row_id_array_size()));
}
rc.RecordSection("prepare vectors data");
status = DBWrapper::DB()->InsertVectors(insert_param_->table_name(), insert_param_->partition_tag(), vec_count,
vec_f.data(), vec_ids);
rc.ElapseFromBegin("add vectors to engine");
if (!status.ok()) {
return status;
}

4
core/src/server/grpc_impl/request/PreloadTableRequest.cpp
View File

@ -40,7 +40,7 @@ Status
PreloadTableRequest::OnExecute() {
try {
std::string hdr = "PreloadTableRequest(table=" + table_name_ + ")";
TimeRecorder rc(hdr);
TimeRecorderAuto rc(hdr);
// step 1: check arguments
auto status = ValidationUtil::ValidateTableName(table_name_);
@ -53,8 +53,6 @@ PreloadTableRequest::OnExecute() {
if (!status.ok()) {
return status;
}
rc.ElapseFromBegin("totally cost");
} catch (std::exception& ex) {
return Status(SERVER_UNEXPECTED_ERROR, ex.what());
}

2
core/src/server/grpc_impl/request/ShowPartitionsRequest.cpp
View File

@ -41,7 +41,7 @@ ShowPartitionsRequest::Create(const std::string& table_name, ::milvus::grpc::Par
Status
ShowPartitionsRequest::OnExecute() {
std::string hdr = "ShowPartitionsRequest(table=" + table_name_ + ")";
TimeRecorder rc(hdr);
TimeRecorderAuto rc(hdr);
auto status = ValidationUtil::ValidateTableName(table_name_);
if (!status.ok()) {

2
core/src/server/grpc_impl/request/ShowTablesRequest.cpp
View File

@ -38,7 +38,7 @@ ShowTablesRequest::Create(::milvus::grpc::TableNameList* table_name_list) {
Status
ShowTablesRequest::OnExecute() {
TimeRecorder rc("ShowTablesRequest");
TimeRecorderAuto rc("ShowTablesRequest");
std::vector<engine::meta::TableSchema> schema_array;
auto statuts = DBWrapper::DB()->AllTables(schema_array);