From e8f759c22b64d3ec5e9d55964dfe2934e6e7f8e5 Mon Sep 17 00:00:00 2001 From: bigbraver Date: Sun, 19 Apr 2020 11:54:23 +0800 Subject: [PATCH] code optimize (#1986) Signed-off-by: bigbraver --- core/src/db/Constants.h | 14 +++++--------- core/src/db/DBImpl.cpp | 4 ++-- core/src/db/Options.h | 2 +- core/src/db/insert/MemManagerImpl.cpp | 2 +- core/src/db/meta/MetaTypes.h | 2 +- core/src/db/meta/MySQLMetaImpl.cpp | 2 +- core/src/db/meta/SqliteMetaImpl.cpp | 2 +- core/unittest/db/test_db.cpp | 4 ++-- core/unittest/db/test_db_mysql.cpp | 4 ++-- core/unittest/db/test_delete.cpp | 2 +- core/unittest/db/test_meta.cpp | 2 +- core/unittest/db/test_meta_mysql.cpp | 4 ++-- 12 files changed, 20 insertions(+), 24 deletions(-) diff --git a/core/src/db/Constants.h b/core/src/db/Constants.h index bbfc67d6bc..c8afeed1cd 100644 --- a/core/src/db/Constants.h +++ b/core/src/db/Constants.h @@ -16,18 +16,14 @@ namespace milvus { namespace engine { -constexpr uint64_t K = 1024UL; -constexpr uint64_t M = K * K; -constexpr uint64_t G = K * M; -constexpr uint64_t T = K * G; +constexpr int64_t KB = 1LL << 10; +constexpr int64_t MB = 1LL << 20; +constexpr int64_t GB = 1LL << 30; +constexpr int64_t TB = 1LL << 40; -constexpr uint64_t MAX_TABLE_FILE_MEM = 128 * M; +constexpr int64_t MAX_TABLE_FILE_MEM = 128 * MB; constexpr int FLOAT_TYPE_SIZE = sizeof(float); -static constexpr uint64_t ONE_KB = K; -static constexpr uint64_t ONE_MB = ONE_KB * ONE_KB; -static constexpr uint64_t ONE_GB = ONE_KB * ONE_MB; - } // namespace engine } // namespace milvus diff --git a/core/src/db/DBImpl.cpp b/core/src/db/DBImpl.cpp index 310d70955c..9529b99ad3 100644 --- a/core/src/db/DBImpl.cpp +++ b/core/src/db/DBImpl.cpp @@ -204,7 +204,7 @@ DBImpl::CreateCollection(meta::CollectionSchema& collection_schema) { } meta::CollectionSchema temp_schema = collection_schema; - temp_schema.index_file_size_ *= ONE_MB; // store as MB + temp_schema.index_file_size_ *= MB; // store as MB if (options_.wal_enable_) { temp_schema.flush_lsn_ = wal_mgr_->CreateCollection(collection_schema.collection_id_); } @@ -257,7 +257,7 @@ DBImpl::DescribeCollection(meta::CollectionSchema& collection_schema) { } auto stat = meta_ptr_->DescribeCollection(collection_schema); - collection_schema.index_file_size_ /= ONE_MB; // return as MB + collection_schema.index_file_size_ /= MB; // return as MB return stat; } diff --git a/core/src/db/Options.h b/core/src/db/Options.h index 5d87b3de96..473b18614d 100644 --- a/core/src/db/Options.h +++ b/core/src/db/Options.h @@ -68,7 +68,7 @@ struct DBOptions { DBMetaOptions meta_; int mode_ = MODE::SINGLE; - size_t insert_buffer_size_ = 4 * ONE_GB; + size_t insert_buffer_size_ = 4 * GB; bool insert_cache_immediately_ = false; int64_t auto_flush_interval_ = 1; diff --git a/core/src/db/insert/MemManagerImpl.cpp b/core/src/db/insert/MemManagerImpl.cpp index 50750fce2a..5de84bf7bd 100644 --- a/core/src/db/insert/MemManagerImpl.cpp +++ b/core/src/db/insert/MemManagerImpl.cpp @@ -319,7 +319,7 @@ MemManagerImpl::GetMaxLSN(const MemList& tables) { void MemManagerImpl::OnInsertBufferSizeChanged(int64_t value) { - options_.insert_buffer_size_ = value * ONE_GB; + options_.insert_buffer_size_ = value * GB; } } // namespace engine diff --git a/core/src/db/meta/MetaTypes.h b/core/src/db/meta/MetaTypes.h index 63b37c8bed..d4a6a801e4 100644 --- a/core/src/db/meta/MetaTypes.h +++ b/core/src/db/meta/MetaTypes.h @@ -27,7 +27,7 @@ namespace meta { constexpr int32_t DEFAULT_ENGINE_TYPE = (int)EngineType::FAISS_IDMAP; constexpr int32_t DEFAULT_NLIST = 16384; constexpr int32_t DEFAULT_METRIC_TYPE = (int)MetricType::L2; -constexpr int32_t DEFAULT_INDEX_FILE_SIZE = ONE_GB; +constexpr int32_t DEFAULT_INDEX_FILE_SIZE = GB; constexpr char CURRENT_VERSION[] = MILVUS_VERSION; constexpr int64_t FLAG_MASK_NO_USERID = 0x1; diff --git a/core/src/db/meta/MySQLMetaImpl.cpp b/core/src/db/meta/MySQLMetaImpl.cpp index 9ce2867042..8d00eb6e2f 100644 --- a/core/src/db/meta/MySQLMetaImpl.cpp +++ b/core/src/db/meta/MySQLMetaImpl.cpp @@ -2073,7 +2073,7 @@ MySQLMetaImpl::Archive() { uint64_t sum = 0; Size(sum); - auto to_delete = (sum - limit * G); + auto to_delete = (sum - limit * GB); DiscardFiles(to_delete); LOG_ENGINE_DEBUG_ << "Archive files to free disk"; diff --git a/core/src/db/meta/SqliteMetaImpl.cpp b/core/src/db/meta/SqliteMetaImpl.cpp index 3921a988a2..7c6d3e1335 100644 --- a/core/src/db/meta/SqliteMetaImpl.cpp +++ b/core/src/db/meta/SqliteMetaImpl.cpp @@ -1461,7 +1461,7 @@ SqliteMetaImpl::Archive() { uint64_t sum = 0; Size(sum); - int64_t to_delete = (int64_t)sum - limit * G; + int64_t to_delete = (int64_t)sum - limit * GB; DiscardFiles(to_delete); LOG_ENGINE_DEBUG_ << "Archive files to free disk"; diff --git a/core/unittest/db/test_db.cpp b/core/unittest/db/test_db.cpp index 4251930456..48ae801daf 100644 --- a/core/unittest/db/test_db.cpp +++ b/core/unittest/db/test_db.cpp @@ -200,7 +200,7 @@ TEST_F(DBTest, DB_TEST) { std::vector tags; stat = db_->Query(dummy_context_, COLLECTION_NAME, tags, k, json_params, qxb, result_ids, result_distances); - ss << "Search " << j << " With Size " << count / milvus::engine::M << " M"; + ss << "Search " << j << " With Size " << count / milvus::engine::MB << " MB"; STOP_TIMER(ss.str()); ASSERT_TRUE(stat.ok()); @@ -946,7 +946,7 @@ TEST_F(DBTest2, ARHIVE_DISK_CHECK) { db_->Size(size); LOG(DEBUG) << "size=" << size; - ASSERT_LE(size, 1 * milvus::engine::G); + ASSERT_LE(size, 1 * milvus::engine::GB); } TEST_F(DBTest2, DELETE_TEST) { diff --git a/core/unittest/db/test_db_mysql.cpp b/core/unittest/db/test_db_mysql.cpp index 32671170d3..88faf12fd7 100644 --- a/core/unittest/db/test_db_mysql.cpp +++ b/core/unittest/db/test_db_mysql.cpp @@ -92,7 +92,7 @@ TEST_F(MySqlDBTest, DB_TEST) { std::vector tags; stat = db_->Query(dummy_context_, COLLECTION_NAME, tags, k, json_params, qxb, result_ids, result_distances); - ss << "Search " << j << " With Size " << count / milvus::engine::M << " M"; + ss << "Search " << j << " With Size " << count / milvus::engine::MB << " MB"; STOP_TIMER(ss.str()); ASSERT_TRUE(stat.ok()); @@ -250,7 +250,7 @@ TEST_F(MySqlDBTest, ARHIVE_DISK_CHECK) { db_->Size(size); LOG(DEBUG) << "size=" << size; - ASSERT_LE(size, 1 * milvus::engine::G); + ASSERT_LE(size, 1 * milvus::engine::GB); FIU_ENABLE_FIU("MySQLMetaImpl.Size.null_connection"); stat = db_->Size(size); diff --git a/core/unittest/db/test_delete.cpp b/core/unittest/db/test_delete.cpp index dd4368ac2e..2db0bd8728 100644 --- a/core/unittest/db/test_delete.cpp +++ b/core/unittest/db/test_delete.cpp @@ -740,7 +740,7 @@ TEST_F(CompactTest, compact_basic) { TEST_F(CompactTest, compact_with_index) { milvus::engine::meta::CollectionSchema collection_info = BuildCollectionSchema(); - collection_info.index_file_size_ = milvus::engine::ONE_KB; + collection_info.index_file_size_ = milvus::engine::KB; collection_info.engine_type_ = (int32_t)milvus::engine::EngineType::FAISS_IVFSQ8; auto stat = db_->CreateCollection(collection_info); diff --git a/core/unittest/db/test_meta.cpp b/core/unittest/db/test_meta.cpp index ca13d62904..ca787a30a5 100644 --- a/core/unittest/db/test_meta.cpp +++ b/core/unittest/db/test_meta.cpp @@ -506,7 +506,7 @@ TEST_F(MetaTest, ARCHIVE_TEST_DISK) { for (auto i = 0; i < cnt; ++i) { status = impl.CreateCollectionFile(table_file); table_file.file_type_ = milvus::engine::meta::SegmentSchema::NEW; - table_file.file_size_ = each_size * milvus::engine::G; + table_file.file_size_ = each_size * milvus::engine::GB; status = impl.UpdateCollectionFile(table_file); files.push_back(table_file); ids.push_back(table_file.id_); diff --git a/core/unittest/db/test_meta_mysql.cpp b/core/unittest/db/test_meta_mysql.cpp index a3fd02adc6..b2fd07ddfc 100644 --- a/core/unittest/db/test_meta_mysql.cpp +++ b/core/unittest/db/test_meta_mysql.cpp @@ -446,7 +446,7 @@ TEST_F(MySqlMetaTest, ARCHIVE_TEST_DISK) { for (auto i = 0; i < cnt; ++i) { status = impl.CreateCollectionFile(table_file); table_file.file_type_ = milvus::engine::meta::SegmentSchema::NEW; - table_file.file_size_ = each_size * milvus::engine::G; + table_file.file_size_ = each_size * milvus::engine::GB; status = impl.UpdateCollectionFile(table_file); files.push_back(table_file); ids.push_back(table_file.id_); @@ -618,7 +618,7 @@ TEST_F(MySqlMetaTest, COLLECTION_FILES_TEST) { ASSERT_EQ(status.code(), milvus::DB_NOT_FOUND); table_file.file_type_ = milvus::engine::meta::SegmentSchema::RAW; - table_file.file_size_ = milvus::engine::ONE_GB + 1; + table_file.file_size_ = milvus::engine::GB + 1; status = impl_->UpdateCollectionFile(table_file); ASSERT_TRUE(status.ok()); #if 0