diff --git a/cpp/coverage.sh b/cpp/coverage.sh index cc509b611b..a0a4cfd03f 100755 --- a/cpp/coverage.sh +++ b/cpp/coverage.sh @@ -14,6 +14,26 @@ DIR_LCOV_OUTPUT="lcov_out" DIR_GCNO="cmake_build" DIR_UNITTEST="milvus/bin" +MYSQL_USER_NAME=root +MYSQL_PASSWORD=Fantast1c +MYSQL_HOST='192.168.1.194' +MYSQL_PORT='3306' + +MYSQL_DB_NAME=milvus_`date +%s%N` + +function mysql_exc() +{ + cmd=$1 + mysql -h${MYSQL_HOST} -u${MYSQL_USER_NAME} -p${MYSQL_PASSWORD} -e "${cmd}" + if [ $? -ne 0 ]; then + echo "mysql $cmd run failed" + fi +} + +mysql_exc "CREATE DATABASE IF NOT EXISTS ${MYSQL_DB_NAME};" +mysql_exc "GRANT ALL PRIVILEGES ON ${MYSQL_DB_NAME}.* TO '${MYSQL_USER_NAME}'@'%';" +mysql_exc "FLUSH PRIVILEGES;" + # get baseline ${LCOV_CMD} -c -i -d ${DIR_GCNO} -o "${FILE_INFO_BASE}" if [ $? -ne 0 ]; then @@ -22,10 +42,11 @@ if [ $? -ne 0 ]; then fi for test in `ls ${DIR_UNITTEST}`; do + echo $test case ${test} in db_test) # set run args for db_test - args="mysql://root:Fantast1c@192.168.1.194:3306/test" + args="mysql://${MYSQL_USER_NAME}:${MYSQL_PASSWORD}@${MYSQL_HOST}:${MYSQL_PORT}/${MYSQL_DB_NAME}" ;; *_test) args="" @@ -38,6 +59,8 @@ for test in `ls ${DIR_UNITTEST}`; do fi done +mysql_exc "DROP DATABASE IF EXISTS ${MYSQL_DB_NAME};" + # gen test converage ${LCOV_CMD} -d ${DIR_GCNO} -o "${FILE_INFO_MILVUS}" -c # merge coverage @@ -50,4 +73,4 @@ ${LCOV_CMD} -r "${FILE_INFO_OUTPUT}" -o "${FILE_INFO_OUTPUT_NEW}" \ "*/cmake_build/*_ep-prefix/*" \ # gen html report -${LCOV_GEN_CMD} "${FILE_INFO_OUTPUT_NEW}" --output-directory ${DIR_LCOV_OUTPUT}/ +${LCOV_GEN_CMD} "${FILE_INFO_OUTPUT_NEW}" --output-directory ${DIR_LCOV_OUTPUT}/ \ No newline at end of file diff --git a/cpp/src/db/MySQLConnectionPool.h b/cpp/src/db/MySQLConnectionPool.h index 8992ba274c..6a763a9729 100644 --- a/cpp/src/db/MySQLConnectionPool.h +++ b/cpp/src/db/MySQLConnectionPool.h @@ -69,6 +69,10 @@ public: max_idle_time_ = max_idle; } + std::string getDB() { + return db_; + } + protected: // Superclass overrides diff --git a/cpp/src/db/MySQLMetaImpl.cpp b/cpp/src/db/MySQLMetaImpl.cpp index 0b0cc01e5d..92bb17168a 100644 --- a/cpp/src/db/MySQLMetaImpl.cpp +++ b/cpp/src/db/MySQLMetaImpl.cpp @@ -193,9 +193,7 @@ namespace meta { "engine_type INT DEFAULT 1 NOT NULL, " << "store_raw_data BOOL DEFAULT false NOT NULL);"; - if (options_.sql_echo) { - ENGINE_LOG_DEBUG << "MySQLMetaImpl::Initialize: " << InitializeQuery.str(); - } + ENGINE_LOG_DEBUG << "MySQLMetaImpl::Initialize: " << InitializeQuery.str(); if (!InitializeQuery.exec()) { return Status::DBTransactionError("Initialization Error", InitializeQuery.error()); @@ -212,9 +210,7 @@ namespace meta { "created_on BIGINT NOT NULL, " << "date INT DEFAULT -1 NOT NULL);"; - if (options_.sql_echo) { - ENGINE_LOG_DEBUG << "MySQLMetaImpl::Initialize: " << InitializeQuery.str(); - } + ENGINE_LOG_DEBUG << "MySQLMetaImpl::Initialize: " << InitializeQuery.str(); if (!InitializeQuery.exec()) { return Status::DBTransactionError("Initialization Error", InitializeQuery.error()); @@ -305,9 +301,7 @@ namespace meta { "WHERE table_id = " << quote << table_id << " AND " << "date in (" << dateListStr << ");"; - if (options_.sql_echo) { - ENGINE_LOG_DEBUG << "MySQLMetaImpl::DropPartitionsByDates: " << dropPartitionsByDatesQuery.str(); - } + ENGINE_LOG_DEBUG << "MySQLMetaImpl::DropPartitionsByDates: " << dropPartitionsByDatesQuery.str(); if (!dropPartitionsByDatesQuery.exec()) { ENGINE_LOG_ERROR << "QUERY ERROR WHEN DROPPING PARTITIONS BY DATES"; @@ -352,9 +346,7 @@ namespace meta { "WHERE table_id = " << quote << table_schema.table_id_ << ";"; // ENGINE_LOG_DEBUG << "Create Table : " << createTableQuery.str(); - if (options_.sql_echo) { - ENGINE_LOG_DEBUG << "MySQLMetaImpl::CreateTable: " << createTableQuery.str(); - } + ENGINE_LOG_DEBUG << "MySQLMetaImpl::CreateTable: " << createTableQuery.str(); StoreQueryResult res = createTableQuery.store(); assert(res && res.num_rows() <= 1); @@ -390,9 +382,7 @@ namespace meta { created_on << ", " << files_cnt << ", " << engine_type << ", " << store_raw_data << ");"; // ENGINE_LOG_DEBUG << "Create Table : " << createTableQuery.str(); - if (options_.sql_echo) { - ENGINE_LOG_DEBUG << "MySQLMetaImpl::CreateTable: " << createTableQuery.str(); - } + ENGINE_LOG_DEBUG << "MySQLMetaImpl::CreateTable: " << createTableQuery.str(); if (SimpleResult res = createTableQuery.execute()) { table_schema.id_ = res.insert_id(); //Might need to use SELECT LAST_INSERT_ID()? @@ -457,9 +447,7 @@ namespace meta { "SET state = " << std::to_string(TableSchema::TO_DELETE) << " " << "WHERE table_id = " << quote << table_id << ";"; - if (options_.sql_echo) { - ENGINE_LOG_DEBUG << "MySQLMetaImpl::DeleteTable: " << deleteTableQuery.str(); - } + ENGINE_LOG_DEBUG << "MySQLMetaImpl::DeleteTable: " << deleteTableQuery.str(); if (!deleteTableQuery.exec()) { ENGINE_LOG_ERROR << "QUERY ERROR WHEN DELETING TABLE"; @@ -506,9 +494,7 @@ namespace meta { "WHERE table_id = " << quote << table_id << " AND " << "file_type <> " << std::to_string(TableFileSchema::TO_DELETE) << ";"; - if (options_.sql_echo) { - ENGINE_LOG_DEBUG << "MySQLMetaImpl::DeleteTableFiles: " << deleteTableFilesQuery.str(); - } + ENGINE_LOG_DEBUG << "MySQLMetaImpl::DeleteTableFiles: " << deleteTableFilesQuery.str(); if (!deleteTableFilesQuery.exec()) { ENGINE_LOG_ERROR << "QUERY ERROR WHEN DELETING TABLE FILES"; @@ -551,9 +537,7 @@ namespace meta { "WHERE table_id = " << quote << table_schema.table_id_ << " " << "AND state <> " << std::to_string(TableSchema::TO_DELETE) << ";"; - if (options_.sql_echo) { - ENGINE_LOG_DEBUG << "MySQLMetaImpl::DescribeTable: " << describeTableQuery.str(); - } + ENGINE_LOG_DEBUG << "MySQLMetaImpl::DescribeTable: " << describeTableQuery.str(); res = describeTableQuery.store(); } //Scoped Connection @@ -618,9 +602,7 @@ namespace meta { "AND state <> " << std::to_string(TableSchema::TO_DELETE) << ") " << "AS " << quote << "check" << ";"; - if (options_.sql_echo) { - ENGINE_LOG_DEBUG << "MySQLMetaImpl::HasTable: " << hasTableQuery.str(); - } + ENGINE_LOG_DEBUG << "MySQLMetaImpl::HasTable: " << hasTableQuery.str(); res = hasTableQuery.store(); } //Scoped Connection @@ -664,9 +646,7 @@ namespace meta { "FROM Tables " << "WHERE state <> " << std::to_string(TableSchema::TO_DELETE) << ";"; - if (options_.sql_echo) { - ENGINE_LOG_DEBUG << "MySQLMetaImpl::AllTables: " << allTablesQuery.str(); - } + ENGINE_LOG_DEBUG << "MySQLMetaImpl::AllTables: " << allTablesQuery.str(); res = allTablesQuery.store(); } //Scoped Connection @@ -755,9 +735,7 @@ namespace meta { quote << file_id << ", " << file_type << ", " << size << ", " << updated_time << ", " << created_on << ", " << date << ");"; - if (options_.sql_echo) { - ENGINE_LOG_DEBUG << "MySQLMetaImpl::CreateTableFile: " << createTableFileQuery.str(); - } + ENGINE_LOG_DEBUG << "MySQLMetaImpl::CreateTableFile: " << createTableFileQuery.str(); if (SimpleResult res = createTableFileQuery.execute()) { file_schema.id_ = res.insert_id(); //Might need to use SELECT LAST_INSERT_ID()? @@ -821,9 +799,7 @@ namespace meta { "FROM TableFiles " << "WHERE file_type = " << std::to_string(TableFileSchema::TO_INDEX) << ";"; - if (options_.sql_echo) { - ENGINE_LOG_DEBUG << "MySQLMetaImpl::FilesToIndex: " << filesToIndexQuery.str(); - } + ENGINE_LOG_DEBUG << "MySQLMetaImpl::FilesToIndex: " << filesToIndexQuery.str(); res = filesToIndexQuery.store(); } //Scoped Connection @@ -911,9 +887,7 @@ namespace meta { "file_type = " << std::to_string(TableFileSchema::TO_INDEX) << " OR " << "file_type = " << std::to_string(TableFileSchema::INDEX) << ");"; - if (options_.sql_echo) { - ENGINE_LOG_DEBUG << "MySQLMetaImpl::FilesToSearch: " << filesToSearchQuery.str(); - } + ENGINE_LOG_DEBUG << "MySQLMetaImpl::FilesToSearch: " << filesToSearchQuery.str(); res = filesToSearchQuery.store(); @@ -936,9 +910,7 @@ namespace meta { "file_type = " << std::to_string(TableFileSchema::TO_INDEX) << " OR " << "file_type = " << std::to_string(TableFileSchema::INDEX) << ");"; - if (options_.sql_echo) { - ENGINE_LOG_DEBUG << "MySQLMetaImpl::FilesToSearch: " << filesToSearchQuery.str(); - } + ENGINE_LOG_DEBUG << "MySQLMetaImpl::FilesToSearch: " << filesToSearchQuery.str(); res = filesToSearchQuery.store(); @@ -1023,9 +995,7 @@ namespace meta { "file_type = " << std::to_string(TableFileSchema::RAW) << " " << "ORDER BY size DESC" << ";"; - if (options_.sql_echo) { - ENGINE_LOG_DEBUG << "MySQLMetaImpl::FilesToMerge: " << filesToMergeQuery.str(); - } + ENGINE_LOG_DEBUG << "MySQLMetaImpl::FilesToMerge: " << filesToMergeQuery.str(); res = filesToMergeQuery.store(); } //Scoped Connection @@ -1116,9 +1086,7 @@ namespace meta { "WHERE table_id = " << quote << table_id << " AND " << "(" << idStr << ");"; - if (options_.sql_echo) { - ENGINE_LOG_DEBUG << "MySQLMetaImpl::GetTableFiles: " << getTableFileQuery.str(); - } + ENGINE_LOG_DEBUG << "MySQLMetaImpl::GetTableFiles: " << getTableFileQuery.str(); res = getTableFileQuery.store(); } //Scoped Connection @@ -1200,9 +1168,7 @@ namespace meta { "WHERE created_on < " << std::to_string(now - usecs) << " AND " << "file_type <> " << std::to_string(TableFileSchema::TO_DELETE) << ";"; - if (options_.sql_echo) { - ENGINE_LOG_DEBUG << "MySQLMetaImpl::Archive: " << archiveQuery.str(); - } + ENGINE_LOG_DEBUG << "MySQLMetaImpl::Archive: " << archiveQuery.str(); if (!archiveQuery.exec()) { return Status::DBTransactionError("QUERY ERROR DURING ARCHIVE", archiveQuery.error()); @@ -1251,9 +1217,7 @@ namespace meta { "FROM TableFiles " << "WHERE file_type <> " << std::to_string(TableFileSchema::TO_DELETE) << ";"; - if (options_.sql_echo) { - ENGINE_LOG_DEBUG << "MySQLMetaImpl::Size: " << getSizeQuery.str(); - } + ENGINE_LOG_DEBUG << "MySQLMetaImpl::Size: " << getSizeQuery.str(); res = getSizeQuery.store(); } //Scoped Connection @@ -1315,9 +1279,7 @@ namespace meta { "ORDER BY id ASC " << "LIMIT 10;"; - if (options_.sql_echo) { - ENGINE_LOG_DEBUG << "MySQLMetaImpl::DiscardFiles: " << discardFilesQuery.str(); - } + ENGINE_LOG_DEBUG << "MySQLMetaImpl::DiscardFiles: " << discardFilesQuery.str(); // std::cout << discardFilesQuery.str() << std::endl; StoreQueryResult res = discardFilesQuery.store(); @@ -1349,9 +1311,7 @@ namespace meta { "updated_time = " << std::to_string(utils::GetMicroSecTimeStamp()) << " " << "WHERE " << idsToDiscardStr << ";"; - if (options_.sql_echo) { - ENGINE_LOG_DEBUG << "MySQLMetaImpl::DiscardFiles: " << discardFilesQuery.str(); - } + ENGINE_LOG_DEBUG << "MySQLMetaImpl::DiscardFiles: " << discardFilesQuery.str(); status = discardFilesQuery.exec(); if (!status) { @@ -1397,9 +1357,7 @@ namespace meta { updateTableFileQuery << "SELECT state FROM Tables " << "WHERE table_id = " << quote << file_schema.table_id_ << ";"; - if (options_.sql_echo) { - ENGINE_LOG_DEBUG << "MySQLMetaImpl::UpdateTableFile: " << updateTableFileQuery.str(); - } + ENGINE_LOG_DEBUG << "MySQLMetaImpl::UpdateTableFile: " << updateTableFileQuery.str(); StoreQueryResult res = updateTableFileQuery.store(); @@ -1434,9 +1392,7 @@ namespace meta { "date = " << date << " " << "WHERE id = " << id << ";"; - if (options_.sql_echo) { - ENGINE_LOG_DEBUG << "MySQLMetaImpl::UpdateTableFile: " << updateTableFileQuery.str(); - } + ENGINE_LOG_DEBUG << "MySQLMetaImpl::UpdateTableFile: " << updateTableFileQuery.str(); // std::cout << updateTableFileQuery.str() << std::endl; @@ -1491,9 +1447,7 @@ namespace meta { "AND state <> " << std::to_string(TableSchema::TO_DELETE) << ") " << "AS " << quote << "check" << ";"; - if (options_.sql_echo) { - ENGINE_LOG_DEBUG << "MySQLMetaImpl::UpdateTableFiles: " << updateTableFilesQuery.str(); - } + ENGINE_LOG_DEBUG << "MySQLMetaImpl::UpdateTableFiles: " << updateTableFilesQuery.str(); StoreQueryResult res = updateTableFilesQuery.store(); @@ -1530,9 +1484,7 @@ namespace meta { "date = " << date << " " << "WHERE id = " << id << ";"; - if (options_.sql_echo) { - ENGINE_LOG_DEBUG << "MySQLMetaImpl::UpdateTableFiles: " << updateTableFilesQuery.str(); - } + ENGINE_LOG_DEBUG << "MySQLMetaImpl::UpdateTableFiles: " << updateTableFilesQuery.str(); if (!updateTableFilesQuery.exec()) { ENGINE_LOG_ERROR << "QUERY ERROR WHEN UPDATING TABLE FILES"; @@ -1582,9 +1534,7 @@ namespace meta { "WHERE file_type = " << std::to_string(TableFileSchema::TO_DELETE) << " AND " << "updated_time < " << std::to_string(now - seconds * US_PS) << ";"; - if (options_.sql_echo) { - ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUpFilesWithTTL: " << cleanUpFilesWithTTLQuery.str(); - } + ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUpFilesWithTTL: " << cleanUpFilesWithTTLQuery.str(); StoreQueryResult res = cleanUpFilesWithTTLQuery.store(); @@ -1628,9 +1578,7 @@ namespace meta { cleanUpFilesWithTTLQuery << "DELETE FROM TableFiles WHERE " << idsToDeleteStr << ";"; - if (options_.sql_echo) { - ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUpFilesWithTTL: " << cleanUpFilesWithTTLQuery.str(); - } + ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUpFilesWithTTL: " << cleanUpFilesWithTTLQuery.str(); if (!cleanUpFilesWithTTLQuery.exec()) { ENGINE_LOG_ERROR << "QUERY ERROR WHEN CLEANING UP FILES WITH TTL"; @@ -1669,9 +1617,7 @@ namespace meta { "FROM Tables " << "WHERE state = " << std::to_string(TableSchema::TO_DELETE) << ";"; - if (options_.sql_echo) { - ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUpFilesWithTTL: " << cleanUpFilesWithTTLQuery.str(); - } + ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUpFilesWithTTL: " << cleanUpFilesWithTTLQuery.str(); StoreQueryResult res = cleanUpFilesWithTTLQuery.store(); assert(res); @@ -1697,9 +1643,7 @@ namespace meta { cleanUpFilesWithTTLQuery << "DELETE FROM Tables WHERE " << idsToDeleteStr << ";"; - if (options_.sql_echo) { - ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUpFilesWithTTL: " << cleanUpFilesWithTTLQuery.str(); - } + ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUpFilesWithTTL: " << cleanUpFilesWithTTLQuery.str(); if (!cleanUpFilesWithTTLQuery.exec()) { ENGINE_LOG_ERROR << "QUERY ERROR WHEN CLEANING UP FILES WITH TTL"; @@ -1733,17 +1677,26 @@ namespace meta { // ENGINE_LOG_WARNING << "MySQLMetaImpl::CleanUp: connection in use = " << mysql_connection_pool_->getConnectionsInUse(); // } - ENGINE_LOG_DEBUG << "Remove table file type as NEW"; Query cleanUpQuery = connectionPtr->query(); - cleanUpQuery << "DELETE FROM TableFiles WHERE file_type = " << std::to_string(TableFileSchema::NEW) << ";"; + cleanUpQuery << "SELECT table_name " << + "FROM information_schema.tables " << + "WHERE table_schema = " << quote << mysql_connection_pool_->getDB() << quote << " " << + "AND table_name = " << quote << "TableFiles" << quote << ";"; + + ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUp: " << cleanUpQuery.str(); + + StoreQueryResult res = cleanUpQuery.store(); + assert(res); + if (!res.empty()) { + ENGINE_LOG_DEBUG << "Remove table file type as NEW"; + cleanUpQuery << "DELETE FROM TableFiles WHERE file_type = " << std::to_string(TableFileSchema::NEW) << ";"; - if (options_.sql_echo) { ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUp: " << cleanUpQuery.str(); - } - if (!cleanUpQuery.exec()) { - ENGINE_LOG_ERROR << "QUERY ERROR WHEN CLEANING UP FILES"; - return Status::DBTransactionError("Clean up Error", cleanUpQuery.error()); + if (!cleanUpQuery.exec()) { + ENGINE_LOG_ERROR << "QUERY ERROR WHEN CLEANING UP FILES"; + return Status::DBTransactionError("Clean up Error", cleanUpQuery.error()); + } } } catch (const BadQuery& er) { @@ -1791,9 +1744,7 @@ namespace meta { "file_type = " << std::to_string(TableFileSchema::TO_INDEX) << " OR " << "file_type = " << std::to_string(TableFileSchema::INDEX) << ");"; - if (options_.sql_echo) { - ENGINE_LOG_DEBUG << "MySQLMetaImpl::Count: " << countQuery.str(); - } + ENGINE_LOG_DEBUG << "MySQLMetaImpl::Count: " << countQuery.str(); res = countQuery.store(); } //Scoped Connection @@ -1838,9 +1789,7 @@ namespace meta { Query dropTableQuery = connectionPtr->query(); dropTableQuery << "DROP TABLE IF EXISTS Tables, TableFiles;"; - if (options_.sql_echo) { - ENGINE_LOG_DEBUG << "MySQLMetaImpl::DropAll: " << dropTableQuery.str(); - } + ENGINE_LOG_DEBUG << "MySQLMetaImpl::DropAll: " << dropTableQuery.str(); if (dropTableQuery.exec()) { return Status::OK(); diff --git a/cpp/src/db/Options.h b/cpp/src/db/Options.h index 609e3ca245..39d0a15019 100644 --- a/cpp/src/db/Options.h +++ b/cpp/src/db/Options.h @@ -45,7 +45,6 @@ struct DBMetaOptions { std::string path; std::string backend_uri; ArchiveConf archive_conf = ArchiveConf("delete"); - bool sql_echo = false; }; // DBMetaOptions struct Options { diff --git a/cpp/src/server/DBWrapper.cpp b/cpp/src/server/DBWrapper.cpp index bf859b3b4f..fca15cb65a 100644 --- a/cpp/src/server/DBWrapper.cpp +++ b/cpp/src/server/DBWrapper.cpp @@ -23,17 +23,6 @@ DBWrapper::DBWrapper() { if(index_size > 0) {//ensure larger than zero, unit is MB opt.index_trigger_size = (size_t)index_size * engine::ONE_MB; } - std::string sql_echo = config.GetValue(CONFIG_DB_SQL_ECHO, "off"); - if (sql_echo == "on") { - opt.meta.sql_echo = true; - } - else if (sql_echo == "off") { - opt.meta.sql_echo = false; - } - else { - std::cout << "ERROR: sql_echo specified in db_config is not one of ['on', 'off']" << std::endl; - kill(0, SIGUSR1); - } ConfigNode& serverConfig = ServerConfig::GetInstance().GetConfig(CONFIG_SERVER); std::string mode = serverConfig.GetValue(CONFIG_CLUSTER_MODE, "single"); diff --git a/cpp/src/server/ServerConfig.h b/cpp/src/server/ServerConfig.h index 768430f023..f337275a46 100644 --- a/cpp/src/server/ServerConfig.h +++ b/cpp/src/server/ServerConfig.h @@ -27,7 +27,6 @@ static const std::string CONFIG_DB_PATH = "db_path"; static const std::string CONFIG_DB_INDEX_TRIGGER_SIZE = "index_building_threshold"; static const std::string CONFIG_DB_ARCHIVE_DISK = "archive_disk_threshold"; static const std::string CONFIG_DB_ARCHIVE_DAYS = "archive_days_threshold"; -static const std::string CONFIG_DB_SQL_ECHO = "sql_echo"; static const std::string CONFIG_LOG = "log_config"; diff --git a/cpp/unittest/db/mysql_db_test.cpp b/cpp/unittest/db/mysql_db_test.cpp index db3c84751e..907aa8a0c4 100644 --- a/cpp/unittest/db/mysql_db_test.cpp +++ b/cpp/unittest/db/mysql_db_test.cpp @@ -18,28 +18,28 @@ using namespace zilliz::milvus; namespace { -static const std::string TABLE_NAME = "test_group"; -static constexpr int64_t TABLE_DIM = 256; -static constexpr int64_t VECTOR_COUNT = 250000; -static constexpr int64_t INSERT_LOOP = 100000; + static const std::string TABLE_NAME = "test_group"; + static constexpr int64_t TABLE_DIM = 256; + static constexpr int64_t VECTOR_COUNT = 250000; + static constexpr int64_t INSERT_LOOP = 100000; -engine::meta::TableSchema BuildTableSchema() { - engine::meta::TableSchema table_info; - table_info.dimension_ = TABLE_DIM; - table_info.table_id_ = TABLE_NAME; - table_info.engine_type_ = (int)engine::EngineType::FAISS_IDMAP; - return table_info; -} - -void BuildVectors(int64_t n, std::vector& vectors) { - vectors.clear(); - vectors.resize(n*TABLE_DIM); - float* data = vectors.data(); - for(int i = 0; i < n; i++) { - for(int j = 0; j < TABLE_DIM; j++) data[TABLE_DIM * i + j] = drand48(); - data[TABLE_DIM * i] += i / 2000.; + engine::meta::TableSchema BuildTableSchema() { + engine::meta::TableSchema table_info; + table_info.dimension_ = TABLE_DIM; + table_info.table_id_ = TABLE_NAME; + table_info.engine_type_ = (int)engine::EngineType::FAISS_IDMAP; + return table_info; + } + + void BuildVectors(int64_t n, std::vector& vectors) { + vectors.clear(); + vectors.resize(n*TABLE_DIM); + float* data = vectors.data(); + for(int i = 0; i < n; i++) { + for(int j = 0; j < TABLE_DIM; j++) data[TABLE_DIM * i + j] = drand48(); + data[TABLE_DIM * i] += i / 2000.; + } } -} } @@ -69,10 +69,13 @@ TEST_F(MySQLDBTest, DB_TEST) { std::vector qxb; BuildVectors(qb, qxb); + db_->InsertVectors(TABLE_NAME, qb, qxb.data(), target_ids); + ASSERT_EQ(target_ids.size(), qb); + std::thread search([&]() { engine::QueryResults results; int k = 10; - std::this_thread::sleep_for(std::chrono::seconds(2)); + std::this_thread::sleep_for(std::chrono::seconds(5)); INIT_TIMER; std::stringstream ss; @@ -91,7 +94,15 @@ TEST_F(MySQLDBTest, DB_TEST) { ASSERT_STATS(stat); for (auto k=0; k= prev_count); - std::this_thread::sleep_for(std::chrono::seconds(1)); + std::this_thread::sleep_for(std::chrono::seconds(3)); } }); int loop = INSERT_LOOP; for (auto i=0; iInsertVectors(TABLE_NAME, qb, qxb.data(), target_ids); - ASSERT_EQ(target_ids.size(), qb); - } else { - db_->InsertVectors(TABLE_NAME, nb, xb.data(), vector_ids); - } +// if (i==10) { +// db_->InsertVectors(TABLE_NAME, qb, qxb.data(), target_ids); +// ASSERT_EQ(target_ids.size(), qb); +// } else { +// db_->InsertVectors(TABLE_NAME, nb, xb.data(), vector_ids); +// } + db_->InsertVectors(TABLE_NAME, nb, xb.data(), vector_ids); std::this_thread::sleep_for(std::chrono::microseconds(1)); } diff --git a/cpp/unittest/db/mysql_meta_test.cpp b/cpp/unittest/db/mysql_meta_test.cpp index 436086acb3..76d7846362 100644 --- a/cpp/unittest/db/mysql_meta_test.cpp +++ b/cpp/unittest/db/mysql_meta_test.cpp @@ -21,184 +21,7 @@ using namespace zilliz::milvus::engine; -//TEST_F(MySQLTest, InitializeTest) { -// DBMetaOptions options; -// //dialect+driver://username:password@host:port/database -// options.backend_uri = "mysql://root:1234@:/test"; -// meta::MySQLMetaImpl impl(options); -// auto status = impl.Initialize(); -// std::cout << status.ToString() << std::endl; -// ASSERT_TRUE(status.ok()); -//} - -TEST_F(MySQLTest, core) { - DBMetaOptions options; -// //dialect+driver://username:password@host:port/database -// options.backend_uri = "mysql://root:1234@:/test"; -// options.path = "/tmp/vecwise_test"; - try { - options = getDBMetaOptions(); - } catch(std::exception& ex) { - ASSERT_TRUE(false); - return; - } - - int mode = Options::MODE::SINGLE; - meta::MySQLMetaImpl impl(options, mode); -// auto status = impl.Initialize(); -// ASSERT_TRUE(status.ok()); - - meta::TableSchema schema1; - schema1.table_id_ = "test1"; - schema1.dimension_ = 123; - - auto status = impl.CreateTable(schema1); -// std::cout << status.ToString() << std::endl; - ASSERT_TRUE(status.ok()); - - meta::TableSchema schema2; - schema2.table_id_ = "test2"; - schema2.dimension_ = 321; - status = impl.CreateTable(schema2); -// std::cout << status.ToString() << std::endl; - ASSERT_TRUE(status.ok()); - - status = impl.CreateTable(schema2); -// std::cout << status.ToString() << std::endl; -// ASSERT_THROW(impl.CreateTable(schema), mysqlpp::BadQuery); - ASSERT_TRUE(status.ok()); - - status = impl.DeleteTable(schema2.table_id_); -// std::cout << status.ToString() << std::endl; - ASSERT_TRUE(status.ok()); - - size_t id1 = schema1.id_; - long created_on1 = schema1.created_on_; - status = impl.DescribeTable(schema1); - ASSERT_TRUE(status.ok()); - ASSERT_EQ(schema1.id_, id1); - ASSERT_EQ(schema1.table_id_, "test1"); - ASSERT_EQ(schema1.created_on_, created_on1); - ASSERT_EQ(schema1.files_cnt_, 0); - ASSERT_EQ(schema1.engine_type_, 1); - ASSERT_EQ(schema1.store_raw_data_, false); - - bool check; - status = impl.HasTable("test1", check); - ASSERT_TRUE(status.ok()); - ASSERT_EQ(check, true); - - std::vector table_schema_array; - status = impl.AllTables(table_schema_array); - ASSERT_TRUE(status.ok()); - ASSERT_EQ(table_schema_array.size(), 1); - meta::TableSchema resultSchema = table_schema_array[0]; - ASSERT_EQ(resultSchema.id_, id1); - ASSERT_EQ(resultSchema.table_id_, "test1"); - ASSERT_EQ(resultSchema.dimension_, 123); - ASSERT_EQ(resultSchema.files_cnt_, 0); - ASSERT_EQ(resultSchema.engine_type_, 1); - ASSERT_EQ(resultSchema.store_raw_data_, false); - - meta::TableFileSchema tableFileSchema; - tableFileSchema.table_id_ = "test1"; - - status = impl.CreateTableFile(tableFileSchema); -// std::cout << status.ToString() << std::endl; - ASSERT_TRUE(status.ok()); - - tableFileSchema.file_type_ = meta::TableFileSchema::TO_INDEX; - status = impl.UpdateTableFile(tableFileSchema); -// std::cout << status.ToString() << std::endl; - ASSERT_TRUE(status.ok()); - - meta::TableFilesSchema filesToIndex; - status = impl.FilesToIndex(filesToIndex); - ASSERT_TRUE(status.ok()); - ASSERT_EQ(filesToIndex.size(), 1); - meta::TableFileSchema fileToIndex = filesToIndex[0]; - ASSERT_EQ(fileToIndex.table_id_, "test1"); - ASSERT_EQ(fileToIndex.dimension_, 123); - -// meta::TableFilesSchema filesToIndex; -// status = impl.FilesToIndex(filesToIndex); -// ASSERT_TRUE(status.ok()); -// ASSERT_EQ(filesToIndex.size(), 0); - - meta::DatesT partition; - partition.push_back(tableFileSchema.date_); - meta::DatePartionedTableFilesSchema filesToSearch; - status = impl.FilesToSearch(tableFileSchema.table_id_, partition, filesToSearch); - ASSERT_TRUE(status.ok()); - ASSERT_EQ(filesToSearch.size(), 1); - ASSERT_EQ(filesToSearch[tableFileSchema.date_].size(), 1); - meta::TableFileSchema fileToSearch = filesToSearch[tableFileSchema.date_][0]; - ASSERT_EQ(fileToSearch.table_id_, "test1"); - ASSERT_EQ(fileToSearch.dimension_, 123); - - tableFileSchema.file_type_ = meta::TableFileSchema::RAW; - status = impl.UpdateTableFile(tableFileSchema); - ASSERT_TRUE(status.ok()); - - meta::DatePartionedTableFilesSchema filesToMerge; - status = impl.FilesToMerge(tableFileSchema.table_id_, filesToMerge); -// std::cout << status.ToString() << std::endl; - ASSERT_TRUE(status.ok()); - ASSERT_EQ(filesToMerge.size(), 1); - ASSERT_EQ(filesToMerge[tableFileSchema.date_].size(), 1); - meta::TableFileSchema fileToMerge = filesToMerge[tableFileSchema.date_][0]; - ASSERT_EQ(fileToMerge.table_id_, "test1"); - ASSERT_EQ(fileToMerge.dimension_, 123); - - meta::TableFilesSchema resultTableFilesSchema; - std::vector ids; - ids.push_back(tableFileSchema.id_); - status = impl.GetTableFiles(tableFileSchema.table_id_, ids, resultTableFilesSchema); - ASSERT_TRUE(status.ok()); - ASSERT_EQ(resultTableFilesSchema.size(), 1); - meta::TableFileSchema resultTableFileSchema = resultTableFilesSchema[0]; -// ASSERT_EQ(resultTableFileSchema.id_, tableFileSchema.id_); - ASSERT_EQ(resultTableFileSchema.table_id_, tableFileSchema.table_id_); - ASSERT_EQ(resultTableFileSchema.file_id_, tableFileSchema.file_id_); - ASSERT_EQ(resultTableFileSchema.file_type_, tableFileSchema.file_type_); - ASSERT_EQ(resultTableFileSchema.size_, tableFileSchema.size_); - ASSERT_EQ(resultTableFileSchema.date_, tableFileSchema.date_); - ASSERT_EQ(resultTableFileSchema.engine_type_, tableFileSchema.engine_type_); - ASSERT_EQ(resultTableFileSchema.dimension_, tableFileSchema.dimension_); - - tableFileSchema.size_ = 234; - meta::TableSchema schema3; - schema3.table_id_ = "test3"; - schema3.dimension_ = 321; - status = impl.CreateTable(schema3); - ASSERT_TRUE(status.ok()); - meta::TableFileSchema tableFileSchema2; - tableFileSchema2.table_id_ = "test3"; - tableFileSchema2.size_ = 345; - status = impl.CreateTableFile(tableFileSchema2); - ASSERT_TRUE(status.ok()); - meta::TableFilesSchema filesToUpdate; - filesToUpdate.emplace_back(tableFileSchema); - filesToUpdate.emplace_back(tableFileSchema2); - status = impl.UpdateTableFile(tableFileSchema); - ASSERT_TRUE(status.ok()); - - uint64_t resultSize; - status = impl.Size(resultSize); -// std::cout << status.ToString() << std::endl; - ASSERT_TRUE(status.ok()); - ASSERT_EQ(resultSize, tableFileSchema.size_ + tableFileSchema2.size_); - - uint64_t countResult; - status = impl.Count(tableFileSchema.table_id_, countResult); - ASSERT_TRUE(status.ok()); - - status = impl.DropAll(); - ASSERT_TRUE(status.ok()); - -} - -TEST_F(MySQLTest, GROUP_TEST) { +TEST_F(MySQLTest, TABLE_TEST) { DBMetaOptions options; try { options = getDBMetaOptions(); @@ -210,38 +33,37 @@ TEST_F(MySQLTest, GROUP_TEST) { int mode = Options::MODE::SINGLE; meta::MySQLMetaImpl impl(options, mode); - auto table_id = "meta_test_group"; + auto table_id = "meta_test_table"; - meta::TableSchema group; - group.table_id_ = table_id; - auto status = impl.CreateTable(group); + meta::TableSchema table; + table.table_id_ = table_id; + auto status = impl.CreateTable(table); ASSERT_TRUE(status.ok()); - auto gid = group.id_; - group.id_ = -1; - status = impl.DescribeTable(group); + auto gid = table.id_; + table.id_ = -1; + status = impl.DescribeTable(table); ASSERT_TRUE(status.ok()); - ASSERT_EQ(group.id_, gid); - ASSERT_EQ(group.table_id_, table_id); + ASSERT_EQ(table.id_, gid); + ASSERT_EQ(table.table_id_, table_id); - group.table_id_ = "not_found"; - status = impl.DescribeTable(group); + table.table_id_ = "not_found"; + status = impl.DescribeTable(table); ASSERT_TRUE(!status.ok()); - group.table_id_ = table_id; - status = impl.CreateTable(group); + table.table_id_ = table_id; + status = impl.CreateTable(table); ASSERT_TRUE(status.ok()); - group.table_id_ = ""; - status = impl.CreateTable(group); + table.table_id_ = ""; + status = impl.CreateTable(table); ASSERT_TRUE(status.ok()); - status = impl.DropAll(); ASSERT_TRUE(status.ok()); } -TEST_F(MySQLTest, table_file_TEST) { +TEST_F(MySQLTest, TABLE_FILE_TEST) { DBMetaOptions options; try { options = getDBMetaOptions(); @@ -253,17 +75,16 @@ TEST_F(MySQLTest, table_file_TEST) { int mode = Options::MODE::SINGLE; meta::MySQLMetaImpl impl(options, mode); - auto table_id = "meta_test_group"; + auto table_id = "meta_test_table"; - meta::TableSchema group; - group.table_id_ = table_id; - group.dimension_ = 256; - auto status = impl.CreateTable(group); + meta::TableSchema table; + table.table_id_ = table_id; + table.dimension_ = 256; + auto status = impl.CreateTable(table); meta::TableFileSchema table_file; - table_file.table_id_ = group.table_id_; + table_file.table_id_ = table.table_id_; status = impl.CreateTableFile(table_file); -// std::cout << status.ToString() << std::endl; ASSERT_TRUE(status.ok()); ASSERT_EQ(table_file.file_type_, meta::TableFileSchema::NEW); @@ -332,15 +153,15 @@ TEST_F(MySQLTest, ARCHIVE_TEST_DAYS) { int mode = Options::MODE::SINGLE; meta::MySQLMetaImpl impl(options, mode); - auto table_id = "meta_test_group"; + auto table_id = "meta_test_table"; - meta::TableSchema group; - group.table_id_ = table_id; - auto status = impl.CreateTable(group); + meta::TableSchema table; + table.table_id_ = table_id; + auto status = impl.CreateTable(table); meta::TableFilesSchema files; meta::TableFileSchema table_file; - table_file.table_id_ = group.table_id_; + table_file.table_id_ = table.table_id_; auto cnt = 100; long ts = utils::GetMicroSecTimeStamp(); @@ -391,13 +212,13 @@ TEST_F(MySQLTest, ARCHIVE_TEST_DISK) { auto impl = meta::MySQLMetaImpl(options, mode); auto table_id = "meta_test_group"; - meta::TableSchema group; - group.table_id_ = table_id; - auto status = impl.CreateTable(group); + meta::TableSchema table; + table.table_id_ = table_id; + auto status = impl.CreateTable(table); meta::TableFilesSchema files; meta::TableFileSchema table_file; - table_file.table_id_ = group.table_id_; + table_file.table_id_ = table.table_id_; auto cnt = 10; auto each_size = 2UL; @@ -445,9 +266,9 @@ TEST_F(MySQLTest, TABLE_FILES_TEST) { auto table_id = "meta_test_group"; - meta::TableSchema group; - group.table_id_ = table_id; - auto status = impl.CreateTable(group); + meta::TableSchema table; + table.table_id_ = table_id; + auto status = impl.CreateTable(table); int new_files_cnt = 4; int raw_files_cnt = 5; @@ -455,7 +276,7 @@ TEST_F(MySQLTest, TABLE_FILES_TEST) { int index_files_cnt = 7; meta::TableFileSchema table_file; - table_file.table_id_ = group.table_id_; + table_file.table_id_ = table.table_id_; for (auto i=0; i