admin/milvus
1
0
Fork 0
mirror of https://gitee.com/milvus-io/milvus.git synced 2025-12-29 06:55:27 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'upstream/0.6.0' into 0.6.0

Browse Source
This commit is contained in:
Yukikaze-CZR 2019-12-03 14:41:44 +08:00
commit c018bf0a08
24 changed files with 30 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
CHANGELOG.md
View File

@ -48,6 +48,7 @@ Please mark all change in change log and use the ticket from JIRA.
- \#631 - FAISS isn't compiled with O3 option
- \#636 - [CPU] Create index PQ should be failed if table metric type set Inner Product
- \#649 - Typo "partiton" should be "partition"
- \#654 - Random crash when frequently insert vector one by one
## Feature
- \#12 - Pure CPU version for Milvus

4
core/src/db/DBImpl.cpp
View File

@ -52,7 +52,7 @@ constexpr uint64_t METRIC_ACTION_INTERVAL = 1;
constexpr uint64_t COMPACT_ACTION_INTERVAL = 1;
constexpr uint64_t INDEX_ACTION_INTERVAL = 1;
static const Status SHUTDOWN_ERROR = Status(DB_ERROR, "Milsvus server is shutdown!");
static const Status SHUTDOWN_ERROR = Status(DB_ERROR, "Milvus server is shutdown!");
void
TraverseFiles(const meta::DatePartionedTableFilesSchema& date_files, meta::TableFilesSchema& files_array) {
@ -800,7 +800,7 @@ DBImpl::BackgroundCompaction(std::set<std::string> table_ids) {
{
uint64_t ttl = 10 * meta::SECOND; // default: file will be hard-deleted few seconds after soft-deleted
if (options_.mode_ == DBOptions::MODE::CLUSTER_WRITABLE) {
ttl = meta::H_SEC;
ttl = meta::HOUR;
}
meta_ptr_->CleanUpFilesWithTTL(ttl, &ongoing_files_checker_);

2
core/src/db/insert/MemManagerImpl.cpp
View File

@ -116,6 +116,7 @@ MemManagerImpl::EraseMemVector(const std::string& table_id) {
size_t
MemManagerImpl::GetCurrentMutableMem() {
size_t total_mem = 0;
std::unique_lock<std::mutex> lock(mutex_);
for (auto& kv : mem_id_map_) {
auto memTable = kv.second;
total_mem += memTable->GetCurrentMem();
@ -126,6 +127,7 @@ MemManagerImpl::GetCurrentMutableMem() {
size_t
MemManagerImpl::GetCurrentImmutableMem() {
size_t total_mem = 0;
std::unique_lock<std::mutex> lock(serialization_mtx_);
for (auto& mem_table : immu_mem_list_) {
total_mem += mem_table->GetCurrentMem();
}

8
core/src/db/meta/MetaConsts.h
View File

@ -27,10 +27,10 @@ const size_t US_PS = 1000 * MS_PS;
const size_t NS_PS = 1000 * US_PS;
const size_t SECOND = 1UL;
const size_t M_SEC = 60 * SECOND;
const size_t H_SEC = 60 * M_SEC;
const size_t D_SEC = 24 * H_SEC;
const size_t W_SEC = 7 * D_SEC;
const size_t MINUTE = 60 * SECOND;
const size_t HOUR = 60 * MINUTE;
const size_t DAY = 24 * HOUR;
const size_t WEEK = 7 * DAY;
// This value is to ignore small raw files when building index.
// The reason is:

2
core/src/db/meta/MySQLMetaImpl.cpp
View File

@ -1664,7 +1664,7 @@ MySQLMetaImpl::Archive() {
auto& criteria = kv.first;
auto& limit = kv.second;
if (criteria == engine::ARCHIVE_CONF_DAYS) {
size_t usecs = limit * D_SEC * US_PS;
size_t usecs = limit * DAY * US_PS;
int64_t now = utils::GetMicroSecTimeStamp();
try {

2
core/src/db/meta/SqliteMetaImpl.cpp
View File

@ -1204,7 +1204,7 @@ SqliteMetaImpl::Archive() {
auto& criteria = kv.first;
auto& limit = kv.second;
if (criteria == engine::ARCHIVE_CONF_DAYS) {
int64_t usecs = limit * D_SEC * US_PS;
int64_t usecs = limit * DAY * US_PS;
int64_t now = utils::GetMicroSecTimeStamp();
try {
// multi-threads call sqlite update may get exception('bad logic', etc), so we add a lock here

2
core/src/server/grpc_impl/request/CmdRequest.cpp
View File

@ -38,7 +38,7 @@ CmdRequest::Create(const std::string& cmd, std::string& result) {
Status
CmdRequest::OnExecute() {
std::string hdr = "CmdRequest(cmd=" + cmd_ + ")";
TimeRecorder rc(hdr);
TimeRecorderAuto rc(hdr);
if (cmd_ == "version") {
result_ = MILVUS_VERSION;

4
core/src/server/grpc_impl/request/CountTableRequest.cpp
View File

@ -40,7 +40,7 @@ Status
CountTableRequest::OnExecute() {
try {
std::string hdr = "CountTableRequest(table=" + table_name_ + ")";
TimeRecorder rc(hdr);
TimeRecorderAuto rc(hdr);
// step 1: check arguments
auto status = ValidationUtil::ValidateTableName(table_name_);
@ -60,8 +60,6 @@ CountTableRequest::OnExecute() {
}
row_count_ = static_cast<int64_t>(row_count);
rc.ElapseFromBegin("total cost");
} catch (std::exception& ex) {
return Status(SERVER_UNEXPECTED_ERROR, ex.what());
}

4
core/src/server/grpc_impl/request/CreateIndexRequest.cpp
View File

@ -46,7 +46,7 @@ Status
CreateIndexRequest::OnExecute() {
try {
std::string hdr = "CreateIndexRequest(table=" + index_param_->table_name() + ")";
TimeRecorder rc(hdr);
TimeRecorderAuto rc(hdr);
// step 1: check arguments
std::string table_name_ = index_param_->table_name();
@ -98,8 +98,6 @@ CreateIndexRequest::OnExecute() {
if (!status.ok()) {
return status;
}
rc.ElapseFromBegin("totally cost");
} catch (std::exception& ex) {
return Status(SERVER_UNEXPECTED_ERROR, ex.what());
}

4
core/src/server/grpc_impl/request/CreatePartitionRequest.cpp
View File

@ -46,7 +46,7 @@ CreatePartitionRequest::OnExecute() {
std::string hdr = "CreatePartitionRequest(table=" + partition_param_->table_name() +
", partition_name=" + partition_param_->partition_name() +
", partition_tag=" + partition_param_->tag() + ")";
TimeRecorder rc(hdr);
TimeRecorderAuto rc(hdr);
try {
// step 1: check arguments
@ -79,8 +79,6 @@ CreatePartitionRequest::OnExecute() {
return Status(SERVER_UNEXPECTED_ERROR, ex.what());
}
rc.ElapseFromBegin("totally cost");
return Status::OK();
}

4
core/src/server/grpc_impl/request/CreateTableRequest.cpp
View File

@ -45,7 +45,7 @@ Status
CreateTableRequest::OnExecute() {
std::string hdr = "CreateTableRequest(table=" + schema_->table_name() +
", dimension=" + std::to_string(schema_->dimension()) + ")";
TimeRecorder rc(hdr);
TimeRecorderAuto rc(hdr);
try {
// step 1: check arguments
@ -89,8 +89,6 @@ CreateTableRequest::OnExecute() {
return Status(SERVER_UNEXPECTED_ERROR, ex.what());
}
rc.ElapseFromBegin("totally cost");
return Status::OK();
}

4
core/src/server/grpc_impl/request/DeleteByDateRequest.cpp
View File

@ -46,7 +46,7 @@ DeleteByDateRequest::Create(const ::milvus::grpc::DeleteByDateParam* delete_by_r
Status
DeleteByDateRequest::OnExecute() {
try {
TimeRecorder rc("DeleteByDateRequest");
TimeRecorderAuto rc("DeleteByDateRequest");
// step 1: check arguments
std::string table_name = delete_by_range_param_->table_name();
@ -67,7 +67,7 @@ DeleteByDateRequest::OnExecute() {
}
}
rc.ElapseFromBegin("check validation");
rc.RecordSection("check validation");
// step 3: check date range, and convert to db dates
std::vector<DB_DATE> dates;

4
core/src/server/grpc_impl/request/DescribeIndexRequest.cpp
View File

@ -40,7 +40,7 @@ Status
DescribeIndexRequest::OnExecute() {
try {
std::string hdr = "DescribeIndexRequest(table=" + table_name_ + ")";
TimeRecorder rc(hdr);
TimeRecorderAuto rc(hdr);
// step 1: check arguments
auto status = ValidationUtil::ValidateTableName(table_name_);
@ -58,8 +58,6 @@ DescribeIndexRequest::OnExecute() {
index_param_->set_table_name(table_name_);
index_param_->mutable_index()->set_index_type(index.engine_type_);
index_param_->mutable_index()->set_nlist(index.nlist_);
rc.ElapseFromBegin("totally cost");
} catch (std::exception& ex) {
return Status(SERVER_UNEXPECTED_ERROR, ex.what());
}

4
core/src/server/grpc_impl/request/DescribeTableRequest.cpp
View File

@ -39,7 +39,7 @@ DescribeTableRequest::Create(const std::string& table_name, ::milvus::grpc::Tabl
Status
DescribeTableRequest::OnExecute() {
std::string hdr = "DescribeTableRequest(table=" + table_name_ + ")";
TimeRecorder rc(hdr);
TimeRecorderAuto rc(hdr);
try {
// step 1: check arguments
@ -64,8 +64,6 @@ DescribeTableRequest::OnExecute() {
return Status(SERVER_UNEXPECTED_ERROR, ex.what());
}
rc.ElapseFromBegin("totally cost");
return Status::OK();
}

4
core/src/server/grpc_impl/request/DropIndexRequest.cpp
View File

@ -40,7 +40,7 @@ Status
DropIndexRequest::OnExecute() {
try {
std::string hdr = "DropIndexRequest(table=" + table_name_ + ")";
TimeRecorder rc(hdr);
TimeRecorderAuto rc(hdr);
// step 1: check arguments
auto status = ValidationUtil::ValidateTableName(table_name_);
@ -63,8 +63,6 @@ DropIndexRequest::OnExecute() {
if (!status.ok()) {
return status;
}
rc.ElapseFromBegin("totally cost");
} catch (std::exception& ex) {
return Status(SERVER_UNEXPECTED_ERROR, ex.what());
}

2
core/src/server/grpc_impl/request/DropPartitionRequest.cpp
View File

@ -42,7 +42,7 @@ DropPartitionRequest::OnExecute() {
std::string hdr = "DropPartitionRequest(table=" + partition_param_->table_name() +
", partition_name=" + partition_param_->partition_name() +
", partition_tag=" + partition_param_->tag() + ")";
TimeRecorder rc(hdr);
TimeRecorderAuto rc(hdr);
std::string table_name = partition_param_->table_name();
std::string partition_name = partition_param_->partition_name();

2
core/src/server/grpc_impl/request/DropTableRequest.cpp
View File

@ -61,7 +61,7 @@ DropTableRequest::OnExecute() {
}
}
rc.ElapseFromBegin("check validation");
rc.RecordSection("check validation");
// step 3: Drop table
std::vector<DB_DATE> dates;

4
core/src/server/grpc_impl/request/HasTableRequest.cpp
View File

@ -40,7 +40,7 @@ Status
HasTableRequest::OnExecute() {
try {
std::string hdr = "HasTableRequest(table=" + table_name_ + ")";
TimeRecorder rc(hdr);
TimeRecorderAuto rc(hdr);
// step 1: check arguments
auto status = ValidationUtil::ValidateTableName(table_name_);
@ -53,8 +53,6 @@ HasTableRequest::OnExecute() {
if (!status.ok()) {
return status;
}
rc.ElapseFromBegin("totally cost");
} catch (std::exception& ex) {
return Status(SERVER_UNEXPECTED_ERROR, ex.what());
}

4
core/src/server/grpc_impl/request/InsertRequest.cpp
View File

@ -122,8 +122,6 @@ InsertRequest::OnExecute() {
table_info.dimension_ * sizeof(float));
}
rc.ElapseFromBegin("prepare vectors data");
// step 5: insert vectors
auto vec_count = static_cast<uint64_t>(insert_param_->row_record_array_size());
std::vector<int64_t> vec_ids(insert_param_->row_id_array_size(), 0);
@ -133,9 +131,9 @@ InsertRequest::OnExecute() {
memcpy(target_data, src_data, static_cast<size_t>(sizeof(int64_t) * insert_param_->row_id_array_size()));
}
rc.RecordSection("prepare vectors data");
status = DBWrapper::DB()->InsertVectors(insert_param_->table_name(), insert_param_->partition_tag(), vec_count,
vec_f.data(), vec_ids);
rc.ElapseFromBegin("add vectors to engine");
if (!status.ok()) {
return status;
}

4
core/src/server/grpc_impl/request/PreloadTableRequest.cpp
View File

@ -40,7 +40,7 @@ Status
PreloadTableRequest::OnExecute() {
try {
std::string hdr = "PreloadTableRequest(table=" + table_name_ + ")";
TimeRecorder rc(hdr);
TimeRecorderAuto rc(hdr);
// step 1: check arguments
auto status = ValidationUtil::ValidateTableName(table_name_);
@ -53,8 +53,6 @@ PreloadTableRequest::OnExecute() {
if (!status.ok()) {
return status;
}
rc.ElapseFromBegin("totally cost");
} catch (std::exception& ex) {
return Status(SERVER_UNEXPECTED_ERROR, ex.what());
}

2
core/src/server/grpc_impl/request/ShowPartitionsRequest.cpp
View File

@ -41,7 +41,7 @@ ShowPartitionsRequest::Create(const std::string& table_name, ::milvus::grpc::Par
Status
ShowPartitionsRequest::OnExecute() {
std::string hdr = "ShowPartitionsRequest(table=" + table_name_ + ")";
TimeRecorder rc(hdr);
TimeRecorderAuto rc(hdr);
auto status = ValidationUtil::ValidateTableName(table_name_);
if (!status.ok()) {

2
core/src/server/grpc_impl/request/ShowTablesRequest.cpp
View File

@ -38,7 +38,7 @@ ShowTablesRequest::Create(::milvus::grpc::TableNameList* table_name_list) {
Status
ShowTablesRequest::OnExecute() {
TimeRecorder rc("ShowTablesRequest");
TimeRecorderAuto rc("ShowTablesRequest");
std::vector<engine::meta::TableSchema> schema_array;
auto statuts = DBWrapper::DB()->AllTables(schema_array);

2
core/unittest/db/test_meta.cpp
View File

@ -141,7 +141,7 @@ TEST_F(MetaTest, ARCHIVE_TEST_DAYS) {
status = impl.CreateTableFile(table_file);
table_file.file_type_ = milvus::engine::meta::TableFileSchema::NEW;
int day = rand_r(&seed) % (days_num * 2);
table_file.created_on_ = ts - day * milvus::engine::meta::D_SEC * milvus::engine::meta::US_PS - 10000;
table_file.created_on_ = ts - day * milvus::engine::meta::DAY * milvus::engine::meta::US_PS - 10000;
status = impl.UpdateTableFile(table_file);
files.push_back(table_file);
days.push_back(day);

2
core/unittest/db/test_meta_mysql.cpp
View File

@ -145,7 +145,7 @@ TEST_F(MySqlMetaTest, ARCHIVE_TEST_DAYS) {
status = impl.CreateTableFile(table_file);
table_file.file_type_ = milvus::engine::meta::TableFileSchema::NEW;
int day = rand_r(&seed) % (days_num * 2);
table_file.created_on_ = ts - day * milvus::engine::meta::D_SEC * milvus::engine::meta::US_PS - 10000;
table_file.created_on_ = ts - day * milvus::engine::meta::DAY * milvus::engine::meta::US_PS - 10000;
status = impl.UpdateTableFile(table_file);
files.push_back(table_file);
days.push_back(day);