From 48f0a43f22b743902c48209686407c5175fbf86a Mon Sep 17 00:00:00 2001 From: zhiru Date: Thu, 13 Jun 2019 11:47:06 +0800 Subject: [PATCH 01/43] add mysqlNotes Former-commit-id: 8b4e613a1aff02ec2c0b99f7f80abc40fe0fbd6c --- cpp/mysqlNotes | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 cpp/mysqlNotes diff --git a/cpp/mysqlNotes b/cpp/mysqlNotes new file mode 100644 index 0000000000..f40db40215 --- /dev/null +++ b/cpp/mysqlNotes @@ -0,0 +1,7 @@ +sudo apt-get install mysql-server +sudo apt-get install libmysqlclient-dev + +Install MySQL++ +./configure +make +sudo make install From 1c31909a6866ef83122c5ebe0be13512cf24ee7e Mon Sep 17 00:00:00 2001 From: zhiru Date: Thu, 13 Jun 2019 14:27:13 +0800 Subject: [PATCH 02/43] temp update Former-commit-id: 7512647247fffebde1b599b4b90caff8a7787bb7 --- cpp/src/db/Options.h | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/cpp/src/db/Options.h b/cpp/src/db/Options.h index cd6f1f98b8..fd7e5045be 100644 --- a/cpp/src/db/Options.h +++ b/cpp/src/db/Options.h @@ -37,6 +37,11 @@ struct DBMetaOptions { ArchiveConf archive_conf = ArchiveConf("delete"); }; // DBMetaOptions +struct MySQLMetaOptions { + std::string path; + +}; + struct Options { Options(); From 0f8ac6dc65ebaa5696a6f5ffae9731cf0c4baef6 Mon Sep 17 00:00:00 2001 From: zhiru Date: Fri, 14 Jun 2019 10:15:44 +0800 Subject: [PATCH 03/43] temp update Former-commit-id: 6dc00e1940e209fb5e65fb6cf31d8f6f0e0a4c1e --- cpp/src/db/MySQLMetaImpl.h | 80 ++++++ cpp/src/db/Options.h | 6 - cpp/unittest/faiss_wrapper/wrapper_test.cpp | 254 +++++++++++--------- 3 files changed, 214 insertions(+), 126 deletions(-) create mode 100644 cpp/src/db/MySQLMetaImpl.h diff --git a/cpp/src/db/MySQLMetaImpl.h b/cpp/src/db/MySQLMetaImpl.h new file mode 100644 index 0000000000..e099fbb262 --- /dev/null +++ b/cpp/src/db/MySQLMetaImpl.h @@ -0,0 +1,80 @@ +/******************************************************************************* + * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved + * Unauthorized copying of this file, via any medium is strictly prohibited. + * Proprietary and confidential. + ******************************************************************************/ +#pragma once + +#include "Meta.h" +#include "Options.h" + +namespace zilliz { +namespace milvus { +namespace engine { +namespace meta { + +// auto StoragePrototype(const std::string& path); + + class MySQLMetaImpl : public Meta { + public: + MySQLMetaImpl(const DBMetaOptions& options_); + + virtual Status CreateTable(TableSchema& table_schema) override; + virtual Status DeleteTable(const std::string& table_id) override; + virtual Status DescribeTable(TableSchema& group_info_) override; + virtual Status HasTable(const std::string& table_id, bool& has_or_not) override; + virtual Status AllTables(std::vector& table_schema_array) override; + + virtual Status CreateTableFile(TableFileSchema& file_schema) override; + virtual Status DropPartitionsByDates(const std::string& table_id, + const DatesT& dates) override; + + virtual Status GetTableFile(TableFileSchema& file_schema) override; + + virtual Status UpdateTableFile(TableFileSchema& file_schema) override; + + virtual Status UpdateTableFiles(TableFilesSchema& files) override; + + virtual Status FilesToSearch(const std::string& table_id, + const DatesT& partition, + DatePartionedTableFilesSchema& files) override; + + virtual Status FilesToMerge(const std::string& table_id, + DatePartionedTableFilesSchema& files) override; + + virtual Status FilesToDelete(const std::string& table_id, + const DatesT& partition, + DatePartionedTableFilesSchema& files) override; + + virtual Status FilesToIndex(TableFilesSchema&) override; + + virtual Status Archive() override; + + virtual Status Size(uint64_t& result) override; + + virtual Status CleanUp() override; + + virtual Status CleanUpFilesWithTTL(uint16_t seconds) override; + + virtual Status DropAll() override; + + virtual Status Count(const std::string& table_id, uint64_t& result) override; + + virtual ~MySQLMetaImpl(); + + private: + Status NextFileId(std::string& file_id); + Status NextTableId(std::string& table_id); + Status DiscardFiles(long to_discard_size); + std::string GetTablePath(const std::string& table_id); + std::string GetTableDatePartitionPath(const std::string& table_id, DateT& date); + void GetTableFilePath(TableFileSchema& group_file); + Status Initialize(); + + const DBMetaOptions options_; + }; // DBMetaImpl + +} // namespace meta +} // namespace engine +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/db/Options.h b/cpp/src/db/Options.h index ec757de003..6eddd26269 100644 --- a/cpp/src/db/Options.h +++ b/cpp/src/db/Options.h @@ -37,12 +37,6 @@ struct DBMetaOptions { ArchiveConf archive_conf = ArchiveConf("delete"); }; // DBMetaOptions -struct MySQLMetaOptions { - std::string path; - -}; - - struct Options { Options(); uint16_t memory_sync_interval = 1; diff --git a/cpp/unittest/faiss_wrapper/wrapper_test.cpp b/cpp/unittest/faiss_wrapper/wrapper_test.cpp index 67a6c3cde8..c1b4ff2cb0 100644 --- a/cpp/unittest/faiss_wrapper/wrapper_test.cpp +++ b/cpp/unittest/faiss_wrapper/wrapper_test.cpp @@ -9,130 +9,144 @@ #include "wrapper/Operand.h" #include "wrapper/Index.h" #include "wrapper/IndexBuilder.h" - +#include using namespace zilliz::milvus::engine; +TEST(xxx, Wrapper_Test){ + // std::string xxx = "dialect+driver://username:password@host:port/database"; -TEST(operand_test, Wrapper_Test) { - using std::cout; - using std::endl; + //mysql://scott:tiger@localhost/mydatabase - auto opd = std::make_shared(); - opd->index_type = "IVF"; - opd->preproc = "OPQ"; - opd->postproc = "PQ"; - opd->metric_type = "L2"; - opd->d = 64; + std::string littel_xx = "dixx://"; + std::regex xx_regex("([a-zA-Z0-9_-\\.]*):\\/\\/([a-zA-Z0-9_-\\.]*)\\:([a-zA-Z0-9_-\\.]*)\\@([a-zA-Z0-9_-\\.]*)\\:([0-9]*)\\/([a-zA-Z0-9_-\\.]*)"); + std::smatch pieces_match; + std::regex_match(littel_xx, pieces_match, xx_regex); - auto opd_str = operand_to_str(opd); - auto new_opd = str_to_operand(opd_str); - - // TODO: fix all place where using opd to build index. - assert(new_opd->get_index_type(10000) == opd->get_index_type(10000)); + for(auto &x : pieces_match) { + std::cout << "hehhe: " << x.str() << std::endl; + } } -TEST(build_test, Wrapper_Test) { - // dimension of the vectors to index - int d = 3; - - // make a set of nt training vectors in the unit cube - size_t nt = 10000; - - // a reasonable number of cetroids to index nb vectors - int ncentroids = 16; - - std::random_device rd; - std::mt19937 gen(rd()); - - std::vector xb; - std::vector ids; - - //prepare train data - std::uniform_real_distribution<> dis_xt(-1.0, 1.0); - std::vector xt(nt * d); - for (size_t i = 0; i < nt * d; i++) { - xt[i] = dis_xt(gen); - } - - //train the index - auto opd = std::make_shared(); - opd->index_type = "IVF"; - opd->d = d; - opd->ncent = ncentroids; - IndexBuilderPtr index_builder_1 = GetIndexBuilder(opd); - auto index_1 = index_builder_1->build_all(0, xb, ids, nt, xt); - ASSERT_TRUE(index_1 != nullptr); - - // size of the database we plan to index - size_t nb = 100000; - - //prepare raw data - xb.resize(nb); - ids.resize(nb); - for (size_t i = 0; i < nb; i++) { - xb[i] = dis_xt(gen); - ids[i] = i; - } - index_1->add_with_ids(nb, xb.data(), ids.data()); - - //search in first quadrant - int nq = 1, k = 10; - std::vector xq = {0.5, 0.5, 0.5}; - float *result_dists = new float[k]; - long *result_ids = new long[k]; - index_1->search(nq, xq.data(), k, result_dists, result_ids); - - for (int i = 0; i < k; i++) { - if (result_ids[i] < 0) { - ASSERT_TRUE(false); - break; - } - - long id = result_ids[i]; - std::cout << "No." << id << " [" << xb[id * 3] << ", " << xb[id * 3 + 1] << ", " - << xb[id * 3 + 2] << "] distance = " << result_dists[i] << std::endl; - - //makesure result vector is in first quadrant - ASSERT_TRUE(xb[id * 3] > 0.0); - ASSERT_TRUE(xb[id * 3 + 1] > 0.0); - ASSERT_TRUE(xb[id * 3 + 2] > 0.0); - } - - delete[] result_dists; - delete[] result_ids; -} - -TEST(gpu_build_test, Wrapper_Test) { - using std::vector; - - int d = 256; - int nb = 3 * 1000 * 100; - int nq = 100; - vector xb(d * nb); - vector xq(d * nq); - vector ids(nb); - - std::random_device rd; - std::mt19937 gen(rd()); - std::uniform_real_distribution<> dis_xt(-1.0, 1.0); - for (auto &e : xb) { e = float(dis_xt(gen)); } - for (auto &e : xq) { e = float(dis_xt(gen)); } - for (int i = 0; i < nb; ++i) { ids[i] = i; } - - auto opd = std::make_shared(); - opd->index_type = "IVF"; - opd->d = d; - opd->ncent = 256; - - IndexBuilderPtr index_builder_1 = GetIndexBuilder(opd); - auto index_1 = index_builder_1->build_all(nb, xb.data(), ids.data()); - assert(index_1->ntotal == nb); - assert(index_1->dim == d); - - // sanity check: search 5 first vectors of xb - int k = 1; - vector I(5 * k); - vector D(5 * k); - index_1->search(5, xb.data(), k, D.data(), I.data()); - for (int i = 0; i < 5; ++i) { assert(i == I[i]); } -} +//TEST(operand_test, Wrapper_Test) { +// using std::cout; +// using std::endl; +// +// auto opd = std::make_shared(); +// opd->index_type = "IVF"; +// opd->preproc = "OPQ"; +// opd->postproc = "PQ"; +// opd->metric_type = "L2"; +// opd->d = 64; +// +// auto opd_str = operand_to_str(opd); +// auto new_opd = str_to_operand(opd_str); +// +// // TODO: fix all place where using opd to build index. +// assert(new_opd->get_index_type(10000) == opd->get_index_type(10000)); +//} +// +//TEST(build_test, Wrapper_Test) { +// // dimension of the vectors to index +// int d = 3; +// +// // make a set of nt training vectors in the unit cube +// size_t nt = 10000; +// +// // a reasonable number of cetroids to index nb vectors +// int ncentroids = 16; +// +// std::random_device rd; +// std::mt19937 gen(rd()); +// +// std::vector xb; +// std::vector ids; +// +// //prepare train data +// std::uniform_real_distribution<> dis_xt(-1.0, 1.0); +// std::vector xt(nt * d); +// for (size_t i = 0; i < nt * d; i++) { +// xt[i] = dis_xt(gen); +// } +// +// //train the index +// auto opd = std::make_shared(); +// opd->index_type = "IVF"; +// opd->d = d; +// opd->ncent = ncentroids; +// IndexBuilderPtr index_builder_1 = GetIndexBuilder(opd); +// auto index_1 = index_builder_1->build_all(0, xb, ids, nt, xt); +// ASSERT_TRUE(index_1 != nullptr); +// +// // size of the database we plan to index +// size_t nb = 100000; +// +// //prepare raw data +// xb.resize(nb); +// ids.resize(nb); +// for (size_t i = 0; i < nb; i++) { +// xb[i] = dis_xt(gen); +// ids[i] = i; +// } +// index_1->add_with_ids(nb, xb.data(), ids.data()); +// +// //search in first quadrant +// int nq = 1, k = 10; +// std::vector xq = {0.5, 0.5, 0.5}; +// float *result_dists = new float[k]; +// long *result_ids = new long[k]; +// index_1->search(nq, xq.data(), k, result_dists, result_ids); +// +// for (int i = 0; i < k; i++) { +// if (result_ids[i] < 0) { +// ASSERT_TRUE(false); +// break; +// } +// +// long id = result_ids[i]; +// std::cout << "No." << id << " [" << xb[id * 3] << ", " << xb[id * 3 + 1] << ", " +// << xb[id * 3 + 2] << "] distance = " << result_dists[i] << std::endl; +// +// //makesure result vector is in first quadrant +// ASSERT_TRUE(xb[id * 3] > 0.0); +// ASSERT_TRUE(xb[id * 3 + 1] > 0.0); +// ASSERT_TRUE(xb[id * 3 + 2] > 0.0); +// } +// +// delete[] result_dists; +// delete[] result_ids; +//} +// +//TEST(gpu_build_test, Wrapper_Test) { +// using std::vector; +// +// int d = 256; +// int nb = 3 * 1000 * 100; +// int nq = 100; +// vector xb(d * nb); +// vector xq(d * nq); +// vector ids(nb); +// +// std::random_device rd; +// std::mt19937 gen(rd()); +// std::uniform_real_distribution<> dis_xt(-1.0, 1.0); +// for (auto &e : xb) { e = float(dis_xt(gen)); } +// for (auto &e : xq) { e = float(dis_xt(gen)); } +// for (int i = 0; i < nb; ++i) { ids[i] = i; } +// +// auto opd = std::make_shared(); +// opd->index_type = "IVF"; +// opd->d = d; +// opd->ncent = 256; +// +// IndexBuilderPtr index_builder_1 = GetIndexBuilder(opd); +// auto index_1 = index_builder_1->build_all(nb, xb.data(), ids.data()); +// assert(index_1->ntotal == nb); +// assert(index_1->dim == d); +// +// // sanity check: search 5 first vectors of xb +// int k = 1; +// vector I(5 * k); +// vector D(5 * k); +// index_1->search(5, xb.data(), k, D.data(), I.data()); +// for (int i = 0; i < 5; ++i) { assert(i == I[i]); } +//} From 48fa9efd4435de8dd674479568bf90cba6db6068 Mon Sep 17 00:00:00 2001 From: zhiru Date: Fri, 14 Jun 2019 15:32:28 +0800 Subject: [PATCH 04/43] temp update Former-commit-id: 42adb2a912930d35c53d7fefca1f35d556b45c05 --- cpp/unittest/faiss_wrapper/wrapper_test.cpp | 254 +++++++++----------- 1 file changed, 120 insertions(+), 134 deletions(-) diff --git a/cpp/unittest/faiss_wrapper/wrapper_test.cpp b/cpp/unittest/faiss_wrapper/wrapper_test.cpp index c1b4ff2cb0..67a6c3cde8 100644 --- a/cpp/unittest/faiss_wrapper/wrapper_test.cpp +++ b/cpp/unittest/faiss_wrapper/wrapper_test.cpp @@ -9,144 +9,130 @@ #include "wrapper/Operand.h" #include "wrapper/Index.h" #include "wrapper/IndexBuilder.h" -#include + using namespace zilliz::milvus::engine; -TEST(xxx, Wrapper_Test){ - // std::string xxx = "dialect+driver://username:password@host:port/database"; - //mysql://scott:tiger@localhost/mydatabase +TEST(operand_test, Wrapper_Test) { + using std::cout; + using std::endl; - std::string littel_xx = "dixx://"; - std::regex xx_regex("([a-zA-Z0-9_-\\.]*):\\/\\/([a-zA-Z0-9_-\\.]*)\\:([a-zA-Z0-9_-\\.]*)\\@([a-zA-Z0-9_-\\.]*)\\:([0-9]*)\\/([a-zA-Z0-9_-\\.]*)"); - std::smatch pieces_match; - std::regex_match(littel_xx, pieces_match, xx_regex); + auto opd = std::make_shared(); + opd->index_type = "IVF"; + opd->preproc = "OPQ"; + opd->postproc = "PQ"; + opd->metric_type = "L2"; + opd->d = 64; - for(auto &x : pieces_match) { - std::cout << "hehhe: " << x.str() << std::endl; - } + auto opd_str = operand_to_str(opd); + auto new_opd = str_to_operand(opd_str); + + // TODO: fix all place where using opd to build index. + assert(new_opd->get_index_type(10000) == opd->get_index_type(10000)); } -//TEST(operand_test, Wrapper_Test) { -// using std::cout; -// using std::endl; -// -// auto opd = std::make_shared(); -// opd->index_type = "IVF"; -// opd->preproc = "OPQ"; -// opd->postproc = "PQ"; -// opd->metric_type = "L2"; -// opd->d = 64; -// -// auto opd_str = operand_to_str(opd); -// auto new_opd = str_to_operand(opd_str); -// -// // TODO: fix all place where using opd to build index. -// assert(new_opd->get_index_type(10000) == opd->get_index_type(10000)); -//} -// -//TEST(build_test, Wrapper_Test) { -// // dimension of the vectors to index -// int d = 3; -// -// // make a set of nt training vectors in the unit cube -// size_t nt = 10000; -// -// // a reasonable number of cetroids to index nb vectors -// int ncentroids = 16; -// -// std::random_device rd; -// std::mt19937 gen(rd()); -// -// std::vector xb; -// std::vector ids; -// -// //prepare train data -// std::uniform_real_distribution<> dis_xt(-1.0, 1.0); -// std::vector xt(nt * d); -// for (size_t i = 0; i < nt * d; i++) { -// xt[i] = dis_xt(gen); -// } -// -// //train the index -// auto opd = std::make_shared(); -// opd->index_type = "IVF"; -// opd->d = d; -// opd->ncent = ncentroids; -// IndexBuilderPtr index_builder_1 = GetIndexBuilder(opd); -// auto index_1 = index_builder_1->build_all(0, xb, ids, nt, xt); -// ASSERT_TRUE(index_1 != nullptr); -// -// // size of the database we plan to index -// size_t nb = 100000; -// -// //prepare raw data -// xb.resize(nb); -// ids.resize(nb); -// for (size_t i = 0; i < nb; i++) { -// xb[i] = dis_xt(gen); -// ids[i] = i; -// } -// index_1->add_with_ids(nb, xb.data(), ids.data()); -// -// //search in first quadrant -// int nq = 1, k = 10; -// std::vector xq = {0.5, 0.5, 0.5}; -// float *result_dists = new float[k]; -// long *result_ids = new long[k]; -// index_1->search(nq, xq.data(), k, result_dists, result_ids); -// -// for (int i = 0; i < k; i++) { -// if (result_ids[i] < 0) { -// ASSERT_TRUE(false); -// break; -// } -// -// long id = result_ids[i]; -// std::cout << "No." << id << " [" << xb[id * 3] << ", " << xb[id * 3 + 1] << ", " -// << xb[id * 3 + 2] << "] distance = " << result_dists[i] << std::endl; -// -// //makesure result vector is in first quadrant -// ASSERT_TRUE(xb[id * 3] > 0.0); -// ASSERT_TRUE(xb[id * 3 + 1] > 0.0); -// ASSERT_TRUE(xb[id * 3 + 2] > 0.0); -// } -// -// delete[] result_dists; -// delete[] result_ids; -//} -// -//TEST(gpu_build_test, Wrapper_Test) { -// using std::vector; -// -// int d = 256; -// int nb = 3 * 1000 * 100; -// int nq = 100; -// vector xb(d * nb); -// vector xq(d * nq); -// vector ids(nb); -// -// std::random_device rd; -// std::mt19937 gen(rd()); -// std::uniform_real_distribution<> dis_xt(-1.0, 1.0); -// for (auto &e : xb) { e = float(dis_xt(gen)); } -// for (auto &e : xq) { e = float(dis_xt(gen)); } -// for (int i = 0; i < nb; ++i) { ids[i] = i; } -// -// auto opd = std::make_shared(); -// opd->index_type = "IVF"; -// opd->d = d; -// opd->ncent = 256; -// -// IndexBuilderPtr index_builder_1 = GetIndexBuilder(opd); -// auto index_1 = index_builder_1->build_all(nb, xb.data(), ids.data()); -// assert(index_1->ntotal == nb); -// assert(index_1->dim == d); -// -// // sanity check: search 5 first vectors of xb -// int k = 1; -// vector I(5 * k); -// vector D(5 * k); -// index_1->search(5, xb.data(), k, D.data(), I.data()); -// for (int i = 0; i < 5; ++i) { assert(i == I[i]); } -//} +TEST(build_test, Wrapper_Test) { + // dimension of the vectors to index + int d = 3; + + // make a set of nt training vectors in the unit cube + size_t nt = 10000; + + // a reasonable number of cetroids to index nb vectors + int ncentroids = 16; + + std::random_device rd; + std::mt19937 gen(rd()); + + std::vector xb; + std::vector ids; + + //prepare train data + std::uniform_real_distribution<> dis_xt(-1.0, 1.0); + std::vector xt(nt * d); + for (size_t i = 0; i < nt * d; i++) { + xt[i] = dis_xt(gen); + } + + //train the index + auto opd = std::make_shared(); + opd->index_type = "IVF"; + opd->d = d; + opd->ncent = ncentroids; + IndexBuilderPtr index_builder_1 = GetIndexBuilder(opd); + auto index_1 = index_builder_1->build_all(0, xb, ids, nt, xt); + ASSERT_TRUE(index_1 != nullptr); + + // size of the database we plan to index + size_t nb = 100000; + + //prepare raw data + xb.resize(nb); + ids.resize(nb); + for (size_t i = 0; i < nb; i++) { + xb[i] = dis_xt(gen); + ids[i] = i; + } + index_1->add_with_ids(nb, xb.data(), ids.data()); + + //search in first quadrant + int nq = 1, k = 10; + std::vector xq = {0.5, 0.5, 0.5}; + float *result_dists = new float[k]; + long *result_ids = new long[k]; + index_1->search(nq, xq.data(), k, result_dists, result_ids); + + for (int i = 0; i < k; i++) { + if (result_ids[i] < 0) { + ASSERT_TRUE(false); + break; + } + + long id = result_ids[i]; + std::cout << "No." << id << " [" << xb[id * 3] << ", " << xb[id * 3 + 1] << ", " + << xb[id * 3 + 2] << "] distance = " << result_dists[i] << std::endl; + + //makesure result vector is in first quadrant + ASSERT_TRUE(xb[id * 3] > 0.0); + ASSERT_TRUE(xb[id * 3 + 1] > 0.0); + ASSERT_TRUE(xb[id * 3 + 2] > 0.0); + } + + delete[] result_dists; + delete[] result_ids; +} + +TEST(gpu_build_test, Wrapper_Test) { + using std::vector; + + int d = 256; + int nb = 3 * 1000 * 100; + int nq = 100; + vector xb(d * nb); + vector xq(d * nq); + vector ids(nb); + + std::random_device rd; + std::mt19937 gen(rd()); + std::uniform_real_distribution<> dis_xt(-1.0, 1.0); + for (auto &e : xb) { e = float(dis_xt(gen)); } + for (auto &e : xq) { e = float(dis_xt(gen)); } + for (int i = 0; i < nb; ++i) { ids[i] = i; } + + auto opd = std::make_shared(); + opd->index_type = "IVF"; + opd->d = d; + opd->ncent = 256; + + IndexBuilderPtr index_builder_1 = GetIndexBuilder(opd); + auto index_1 = index_builder_1->build_all(nb, xb.data(), ids.data()); + assert(index_1->ntotal == nb); + assert(index_1->dim == d); + + // sanity check: search 5 first vectors of xb + int k = 1; + vector I(5 * k); + vector D(5 * k); + index_1->search(5, xb.data(), k, D.data(), I.data()); + for (int i = 0; i < 5; ++i) { assert(i == I[i]); } +} From d9076789d0ce194d86ff8e69a21cef7bb9ab9b18 Mon Sep 17 00:00:00 2001 From: zhiru Date: Fri, 14 Jun 2019 19:02:06 +0800 Subject: [PATCH 05/43] update Former-commit-id: c72896aebcabce77cb12c4912b75aa109468a225 --- cpp/src/db/MySQLMetaImpl.h | 3 ++- cpp/unittest/db/CMakeLists.txt | 5 ++++- cpp/unittest/db/utils.h | 6 ++++++ 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/cpp/src/db/MySQLMetaImpl.h b/cpp/src/db/MySQLMetaImpl.h index e099fbb262..3e8bcdeeec 100644 --- a/cpp/src/db/MySQLMetaImpl.h +++ b/cpp/src/db/MySQLMetaImpl.h @@ -19,6 +19,8 @@ namespace meta { public: MySQLMetaImpl(const DBMetaOptions& options_); + Status Initialize(); + virtual Status CreateTable(TableSchema& table_schema) override; virtual Status DeleteTable(const std::string& table_id) override; virtual Status DescribeTable(TableSchema& group_info_) override; @@ -69,7 +71,6 @@ namespace meta { std::string GetTablePath(const std::string& table_id); std::string GetTableDatePartitionPath(const std::string& table_id, DateT& date); void GetTableFilePath(TableFileSchema& group_file); - Status Initialize(); const DBMetaOptions options_; }; // DBMetaImpl diff --git a/cpp/unittest/db/CMakeLists.txt b/cpp/unittest/db/CMakeLists.txt index ecfeb1e92c..9471f5d69d 100644 --- a/cpp/unittest/db/CMakeLists.txt +++ b/cpp/unittest/db/CMakeLists.txt @@ -28,10 +28,13 @@ set(db_test_src ${db_scheduler_srcs} ${wrapper_src} ${require_files} + MySQLMetaImpl_test.cpp utils.cpp db_tests.cpp meta_tests.cpp) +include_directories(/usr/include/mysql) + cuda_add_executable(db_test ${db_test_src}) set(db_libs @@ -44,4 +47,4 @@ set(db_libs lz4 ) -target_link_libraries(db_test ${db_libs} ${unittest_libs}) +target_link_libraries(db_test ${db_libs} ${unittest_libs} /usr/local/lib/libmysqlpp.so) diff --git a/cpp/unittest/db/utils.h b/cpp/unittest/db/utils.h index cf0ff360f1..2116072f20 100644 --- a/cpp/unittest/db/utils.h +++ b/cpp/unittest/db/utils.h @@ -8,6 +8,7 @@ #include #include +#include #include "db/DB.h" #include "db/DBMetaImpl.h" @@ -55,3 +56,8 @@ protected: virtual void SetUp() override; virtual void TearDown() override; }; + +class MySQLTest : public DBTest { +protected: + //std::shared_ptr impl_; +}; From 47537d078366aca0f71aceb403c52186cd8c8d89 Mon Sep 17 00:00:00 2001 From: zhiru Date: Sun, 16 Jun 2019 10:24:23 +0800 Subject: [PATCH 06/43] update Former-commit-id: d6d4456d9b9638a7ac149155e1cd851b3105dc0e --- cpp/src/db/MySQLMetaImpl.cpp | 982 +++++++++++++++++++++++++ cpp/unittest/db/MySQLMetaImpl_test.cpp | 28 + 2 files changed, 1010 insertions(+) create mode 100644 cpp/src/db/MySQLMetaImpl.cpp create mode 100644 cpp/unittest/db/MySQLMetaImpl_test.cpp diff --git a/cpp/src/db/MySQLMetaImpl.cpp b/cpp/src/db/MySQLMetaImpl.cpp new file mode 100644 index 0000000000..5dacb3f3b3 --- /dev/null +++ b/cpp/src/db/MySQLMetaImpl.cpp @@ -0,0 +1,982 @@ +/******************************************************************************* + * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved + * Unauthorized copying of this file, via any medium is strictly prohibited. + * Proprietary and confidential. + ******************************************************************************/ +#include "MySQLMetaImpl.h" +#include "IDGenerator.h" +#include "Utils.h" +#include "Log.h" +#include "MetaConsts.h" +#include "Factories.h" +#include "metrics/Metrics.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "mysql++/mysql++.h" + +namespace zilliz { +namespace milvus { +namespace engine { +namespace meta { + + using namespace mysqlpp; + +// static std::unique_ptr connectionPtr(new Connection()); + static Connection* connectionPtr = new Connection(); + + namespace { + + void HandleException(std::exception &e) { + ENGINE_LOG_DEBUG << "Engine meta exception: " << e.what(); + throw e; + } + + } + + std::string MySQLMetaImpl::GetTablePath(const std::string &table_id) { + return options_.path + "/tables/" + table_id; + } + + std::string MySQLMetaImpl::GetTableDatePartitionPath(const std::string &table_id, DateT &date) { + std::stringstream ss; + ss << GetTablePath(table_id) << "/" << date; + return ss.str(); + } + + void MySQLMetaImpl::GetTableFilePath(TableFileSchema &group_file) { + if (group_file.date_ == EmptyDate) { + group_file.date_ = Meta::GetDate(); + } + std::stringstream ss; + ss << GetTableDatePartitionPath(group_file.table_id_, group_file.date_) + << "/" << group_file.file_id_; + group_file.location_ = ss.str(); + } + + Status MySQLMetaImpl::NextTableId(std::string &table_id) { + std::stringstream ss; + SimpleIDGenerator g; + ss << g.GetNextIDNumber(); + table_id = ss.str(); + return Status::OK(); + } + + Status MySQLMetaImpl::NextFileId(std::string &file_id) { + std::stringstream ss; + SimpleIDGenerator g; + ss << g.GetNextIDNumber(); + file_id = ss.str(); + return Status::OK(); + } + + MySQLMetaImpl::MySQLMetaImpl(const DBMetaOptions &options_) + : options_(options_) { + //Initialize(); + } + + Status MySQLMetaImpl::Initialize() { +// if (!boost::filesystem::is_directory(options_.path)) { +// auto ret = boost::filesystem::create_directory(options_.path); +// if (!ret) { +// ENGINE_LOG_ERROR << "Create directory " << options_.path << " Error"; +// } +// assert(ret); +// } + +// ConnectorPtr = std::make_unique(StoragePrototype(options_.path + "/meta.sqlite")); +// +// ConnectorPtr->sync_schema(); +// ConnectorPtr->open_forever(); // thread safe option +// ConnectorPtr->pragma.journal_mode(journal_mode::WAL); // WAL => write ahead log + + std::string uri = options_.backend_uri; + + std::string dialectRegex = "(.*)"; + std::string usernameRegex = "(.*)"; + std::string passwordRegex = "(.*)"; + std::string hostRegex = "(.*)"; + std::string portRegex = "(.*)"; + std::string dbNameRegex = "(.*)"; + std::string uriRegexStr = dialectRegex + "\\:\\/\\/" + + usernameRegex + "\\:" + + passwordRegex + "\\@" + + hostRegex + "\\:" + + portRegex + "\\/" + + dbNameRegex; + std::regex uriRegex(uriRegexStr); + std::smatch pieces_match; + + if (std::regex_match(uri, pieces_match, uriRegex)) { + std::string dialect = pieces_match[1].str(); + std::transform(dialect.begin(), dialect.end(), dialect.begin(), ::tolower); + if (dialect.find("mysql") == std::string::npos) { + return Status::Error("URI's dialect is not MySQL"); + } + const char* username = pieces_match[2].str().c_str(); + const char* password = pieces_match[3].str().c_str(); + const char* serverAddress = pieces_match[4].str().c_str(); + unsigned int port = 0; + if (!pieces_match[5].str().empty()) { + port = std::stoi(pieces_match[5].str()); + } + const char* dbName = pieces_match[6].str().c_str(); + //std::cout << dbName << " " << serverAddress << " " << username << " " << password << " " << port << std::endl; + connectionPtr->set_option(new MultiStatementsOption(true)); + + try { + if (!connectionPtr->connect(dbName, serverAddress, username, password, port)) { + return Status::Error("DB connection failed: ", connectionPtr->error()); + } + + CleanUp(); + + Query InitializeQuery = connectionPtr->query(); + + InitializeQuery << "DROP TABLE IF EXISTS meta, metaFile;"; + InitializeQuery << "CREATE TABLE meta (" << + "id BIGINT AUTO INCREMENT PRIMARY KEY, " << + "table_id VARCHAR(255) UNIQUE, " << + "dimension SMALLINT, " << + "created_on BIGINT, " << + "files_cnt BIGINT DEFAULT 0, " << + "engine_type INT DEFAULT 1, " << + "store_raw_data BOOL DEFAULT false);"; + InitializeQuery << "CREATE TABLE metaFile (" << + "id BIGINT AUTO INCREMENT PRIMARY KEY, " << + "table_id VARCHAR(255), " << + "engine_type INT DEFAULT 1, " << + "file_id VARCHAR(255), " << + "file_type INT DEFAULT 0, " << + "size BIGINT DEFAULT 0, " << + "updated_time BIGINT, " << + "created_on BIGINT, " << + "date INT DEFAULT -1);"; + + if (InitializeQuery.exec()) { + return Status::OK(); + } else { + return Status::DBTransactionError("Initialization Error: ", InitializeQuery.error()); + } + } catch (const ConnectionFailed& er) { + return Status::DBTransactionError("Failed to connect to MySQL server: ", er.what()); + } catch (const BadQuery& er) { + // Handle any query errors + return Status::DBTransactionError("QUERY ERROR DURING INITIALIZATION: ", er.what()); + } catch (const Exception& er) { + // Catch-all for any other MySQL++ exceptions + return Status::DBTransactionError("GENERAL ERROR DURING INITIALIZATION: ", er.what()); + } + } + else { + return Status::Error("Wrong URI format"); + } + } + +// PXU TODO: Temp solution. Will fix later + Status MySQLMetaImpl::DropPartitionsByDates(const std::string &table_id, + const DatesT &dates) { +// if (dates.size() == 0) { +// return Status::OK(); +// } +// +// TableSchema table_schema; +// table_schema.table_id_ = table_id; +// auto status = DescribeTable(table_schema); +// if (!status.ok()) { +// return status; +// } +// +// auto yesterday = GetDateWithDelta(-1); +// +// for (auto &date : dates) { +// if (date >= yesterday) { +// return Status::Error("Could not delete partitions with 2 days"); +// } +// } +// +// try { +// ConnectorPtr->update_all( +// set( +// c(&TableFileSchema::file_type_) = (int) TableFileSchema::TO_DELETE +// ), +// where( +// c(&TableFileSchema::table_id_) == table_id and +// in(&TableFileSchema::date_, dates) +// )); +// } catch (std::exception &e) { +// HandleException(e); +// } + return Status::OK(); + } + + Status MySQLMetaImpl::CreateTable(TableSchema &table_schema) { +// server::Metrics::GetInstance().MetaAccessTotalIncrement(); +// if (table_schema.table_id_.empty()) { +// NextTableId(table_schema.table_id_); +// } +// table_schema.files_cnt_ = 0; +// table_schema.id_ = -1; +// table_schema.created_on_ = utils::GetMicroSecTimeStamp(); +// auto start_time = METRICS_NOW_TIME; +// { +// try { +// Query addTableQuery = connectionPtr->query(); +// +// } catch (...) { +// return Status::DBTransactionError("Add Table Error"); +// } +// } +// auto end_time = METRICS_NOW_TIME; +// auto total_time = METRICS_MICROSECONDS(start_time, end_time); +// server::Metrics::GetInstance().MetaAccessDurationSecondsHistogramObserve(total_time); +// +// auto table_path = GetTablePath(table_schema.table_id_); +// table_schema.location_ = table_path; +// if (!boost::filesystem::is_directory(table_path)) { +// auto ret = boost::filesystem::create_directories(table_path); +// if (!ret) { +// ENGINE_LOG_ERROR << "Create directory " << table_path << " Error"; +// } +// assert(ret); +// } + + return Status::OK(); + } + + Status MySQLMetaImpl::DeleteTable(const std::string& table_id) { +// try { +// //drop the table from meta +// auto tables = ConnectorPtr->select(columns(&TableSchema::id_), +// where(c(&TableSchema::table_id_) == table_id)); +// for (auto &table : tables) { +// ConnectorPtr->remove(std::get<0>(table)); +// } +// } catch (std::exception &e) { +// HandleException(e); +// } + + return Status::OK(); + } + + Status MySQLMetaImpl::DescribeTable(TableSchema &table_schema) { +// try { +// server::Metrics::GetInstance().MetaAccessTotalIncrement(); +// auto start_time = METRICS_NOW_TIME; +// auto groups = ConnectorPtr->select(columns(&TableSchema::id_, +// &TableSchema::table_id_, +// &TableSchema::files_cnt_, +// &TableSchema::dimension_, +// &TableSchema::engine_type_, +// &TableSchema::store_raw_data_), +// where(c(&TableSchema::table_id_) == table_schema.table_id_)); +// auto end_time = METRICS_NOW_TIME; +// auto total_time = METRICS_MICROSECONDS(start_time, end_time); +// server::Metrics::GetInstance().MetaAccessDurationSecondsHistogramObserve(total_time); +// assert(groups.size() <= 1); +// if (groups.size() == 1) { +// table_schema.id_ = std::get<0>(groups[0]); +// table_schema.files_cnt_ = std::get<2>(groups[0]); +// table_schema.dimension_ = std::get<3>(groups[0]); +// table_schema.engine_type_ = std::get<4>(groups[0]); +// table_schema.store_raw_data_ = std::get<5>(groups[0]); +// } else { +// return Status::NotFound("Table " + table_schema.table_id_ + " not found"); +// } +// +// auto table_path = GetTablePath(table_schema.table_id_); +// table_schema.location_ = table_path; +// +// } catch (std::exception &e) { +// HandleException(e); +// } + + return Status::OK(); + } + + Status MySQLMetaImpl::HasTable(const std::string &table_id, bool &has_or_not) { +// try { +// server::Metrics::GetInstance().MetaAccessTotalIncrement(); +// auto start_time = METRICS_NOW_TIME; +// +// auto tables = ConnectorPtr->select(columns(&TableSchema::id_), +// where(c(&TableSchema::table_id_) == table_id)); +// auto end_time = METRICS_NOW_TIME; +// auto total_time = METRICS_MICROSECONDS(start_time, end_time); +// server::Metrics::GetInstance().MetaAccessDurationSecondsHistogramObserve(total_time); +// assert(tables.size() <= 1); +// if (tables.size() == 1) { +// has_or_not = true; +// } else { +// has_or_not = false; +// } +// } catch (std::exception &e) { +// HandleException(e); +// } + return Status::OK(); + } + + Status MySQLMetaImpl::AllTables(std::vector& table_schema_array) { +// try { +// server::Metrics::GetInstance().MetaAccessTotalIncrement(); +// auto start_time = METRICS_NOW_TIME; +// auto selected = ConnectorPtr->select(columns(&TableSchema::id_, +// &TableSchema::table_id_, +// &TableSchema::files_cnt_, +// &TableSchema::dimension_, +// &TableSchema::engine_type_, +// &TableSchema::store_raw_data_)); +// auto end_time = METRICS_NOW_TIME; +// auto total_time = METRICS_MICROSECONDS(start_time, end_time); +// server::Metrics::GetInstance().MetaAccessDurationSecondsHistogramObserve(total_time); +// for (auto &table : selected) { +// TableSchema schema; +// schema.id_ = std::get<0>(table); +// schema.table_id_ = std::get<1>(table); +// schema.files_cnt_ = std::get<2>(table); +// schema.dimension_ = std::get<3>(table); +// schema.engine_type_ = std::get<4>(table); +// schema.store_raw_data_ = std::get<5>(table); +// +// table_schema_array.emplace_back(schema); +// } +// } catch (std::exception &e) { +// HandleException(e); +// } + + return Status::OK(); + } + + Status MySQLMetaImpl::CreateTableFile(TableFileSchema &file_schema) { +// if (file_schema.date_ == EmptyDate) { +// file_schema.date_ = Meta::GetDate(); +// } +// TableSchema table_schema; +// table_schema.table_id_ = file_schema.table_id_; +// auto status = DescribeTable(table_schema); +// if (!status.ok()) { +// return status; +// } +// +// NextFileId(file_schema.file_id_); +// file_schema.file_type_ = TableFileSchema::NEW; +// file_schema.dimension_ = table_schema.dimension_; +// file_schema.size_ = 0; +// file_schema.created_on_ = utils::GetMicroSecTimeStamp(); +// file_schema.updated_time_ = file_schema.created_on_; +// file_schema.engine_type_ = table_schema.engine_type_; +// GetTableFilePath(file_schema); +// +// { +// try { +// server::Metrics::GetInstance().MetaAccessTotalIncrement(); +// auto start_time = METRICS_NOW_TIME; +// auto id = ConnectorPtr->insert(file_schema); +// file_schema.id_ = id; +// auto end_time = METRICS_NOW_TIME; +// auto total_time = METRICS_MICROSECONDS(start_time, end_time); +// server::Metrics::GetInstance().MetaAccessDurationSecondsHistogramObserve(total_time); +// } catch (...) { +// return Status::DBTransactionError("Add file Error"); +// } +// } +// +// auto partition_path = GetTableDatePartitionPath(file_schema.table_id_, file_schema.date_); +// +// if (!boost::filesystem::is_directory(partition_path)) { +// auto ret = boost::filesystem::create_directory(partition_path); +// if (!ret) { +// ENGINE_LOG_ERROR << "Create directory " << partition_path << " Error"; +// } +// assert(ret); +// } + + return Status::OK(); + } + + Status MySQLMetaImpl::FilesToIndex(TableFilesSchema &files) { +// files.clear(); +// +// try { +// server::Metrics::GetInstance().MetaAccessTotalIncrement(); +// auto start_time = METRICS_NOW_TIME; +// auto selected = ConnectorPtr->select(columns(&TableFileSchema::id_, +// &TableFileSchema::table_id_, +// &TableFileSchema::file_id_, +// &TableFileSchema::file_type_, +// &TableFileSchema::size_, +// &TableFileSchema::date_, +// &TableFileSchema::engine_type_), +// where(c(&TableFileSchema::file_type_) +// == (int) TableFileSchema::TO_INDEX)); +// auto end_time = METRICS_NOW_TIME; +// auto total_time = METRICS_MICROSECONDS(start_time, end_time); +// server::Metrics::GetInstance().MetaAccessDurationSecondsHistogramObserve(total_time); +// +// std::map groups; +// TableFileSchema table_file; +// +// for (auto &file : selected) { +// table_file.id_ = std::get<0>(file); +// table_file.table_id_ = std::get<1>(file); +// table_file.file_id_ = std::get<2>(file); +// table_file.file_type_ = std::get<3>(file); +// table_file.size_ = std::get<4>(file); +// table_file.date_ = std::get<5>(file); +// table_file.engine_type_ = std::get<6>(file); +// +// GetTableFilePath(table_file); +// auto groupItr = groups.find(table_file.table_id_); +// if (groupItr == groups.end()) { +// TableSchema table_schema; +// table_schema.table_id_ = table_file.table_id_; +// auto status = DescribeTable(table_schema); +// if (!status.ok()) { +// return status; +// } +// groups[table_file.table_id_] = table_schema; +// } +// table_file.dimension_ = groups[table_file.table_id_].dimension_; +// files.push_back(table_file); +// } +// } catch (std::exception &e) { +// HandleException(e); +// } + + return Status::OK(); + } + + Status MySQLMetaImpl::FilesToSearch(const std::string &table_id, + const DatesT &partition, + DatePartionedTableFilesSchema &files) { +// files.clear(); +// +// try { +// server::Metrics::GetInstance().MetaAccessTotalIncrement(); +// auto start_time = METRICS_NOW_TIME; +// if (partition.empty()) { +// auto selected = ConnectorPtr->select(columns(&TableFileSchema::id_, +// &TableFileSchema::table_id_, +// &TableFileSchema::file_id_, +// &TableFileSchema::file_type_, +// &TableFileSchema::size_, +// &TableFileSchema::date_, +// &TableFileSchema::engine_type_), +// where(c(&TableFileSchema::table_id_) == table_id and +// (c(&TableFileSchema::file_type_) == (int) TableFileSchema::RAW or +// c(&TableFileSchema::file_type_) +// == (int) TableFileSchema::TO_INDEX or +// c(&TableFileSchema::file_type_) +// == (int) TableFileSchema::INDEX))); +// auto end_time = METRICS_NOW_TIME; +// auto total_time = METRICS_MICROSECONDS(start_time, end_time); +// server::Metrics::GetInstance().MetaAccessDurationSecondsHistogramObserve(total_time); +// TableSchema table_schema; +// table_schema.table_id_ = table_id; +// auto status = DescribeTable(table_schema); +// if (!status.ok()) { +// return status; +// } +// +// TableFileSchema table_file; +// +// for (auto &file : selected) { +// table_file.id_ = std::get<0>(file); +// table_file.table_id_ = std::get<1>(file); +// table_file.file_id_ = std::get<2>(file); +// table_file.file_type_ = std::get<3>(file); +// table_file.size_ = std::get<4>(file); +// table_file.date_ = std::get<5>(file); +// table_file.engine_type_ = std::get<6>(file); +// table_file.dimension_ = table_schema.dimension_; +// GetTableFilePath(table_file); +// auto dateItr = files.find(table_file.date_); +// if (dateItr == files.end()) { +// files[table_file.date_] = TableFilesSchema(); +// } +// files[table_file.date_].push_back(table_file); +// } +// } +// else { +// auto selected = ConnectorPtr->select(columns(&TableFileSchema::id_, +// &TableFileSchema::table_id_, +// &TableFileSchema::file_id_, +// &TableFileSchema::file_type_, +// &TableFileSchema::size_, +// &TableFileSchema::date_), +// where(c(&TableFileSchema::table_id_) == table_id and +// in(&TableFileSchema::date_, partition) and +// (c(&TableFileSchema::file_type_) == (int) TableFileSchema::RAW or +// c(&TableFileSchema::file_type_) +// == (int) TableFileSchema::TO_INDEX or +// c(&TableFileSchema::file_type_) +// == (int) TableFileSchema::INDEX))); +// auto end_time = METRICS_NOW_TIME; +// auto total_time = METRICS_MICROSECONDS(start_time, end_time); +// server::Metrics::GetInstance().MetaAccessDurationSecondsHistogramObserve(total_time); +// TableSchema table_schema; +// table_schema.table_id_ = table_id; +// auto status = DescribeTable(table_schema); +// if (!status.ok()) { +// return status; +// } +// +// TableFileSchema table_file; +// +// for (auto &file : selected) { +// table_file.id_ = std::get<0>(file); +// table_file.table_id_ = std::get<1>(file); +// table_file.file_id_ = std::get<2>(file); +// table_file.file_type_ = std::get<3>(file); +// table_file.size_ = std::get<4>(file); +// table_file.date_ = std::get<5>(file); +// table_file.dimension_ = table_schema.dimension_; +// GetTableFilePath(table_file); +// auto dateItr = files.find(table_file.date_); +// if (dateItr == files.end()) { +// files[table_file.date_] = TableFilesSchema(); +// } +// files[table_file.date_].push_back(table_file); +// } +// +// } +// } catch (std::exception &e) { +// HandleException(e); +// } + + return Status::OK(); + } + + Status MySQLMetaImpl::FilesToMerge(const std::string &table_id, + DatePartionedTableFilesSchema &files) { +// files.clear(); +// +// try { +// server::Metrics::GetInstance().MetaAccessTotalIncrement(); +// auto start_time = METRICS_NOW_TIME; +// auto selected = ConnectorPtr->select(columns(&TableFileSchema::id_, +// &TableFileSchema::table_id_, +// &TableFileSchema::file_id_, +// &TableFileSchema::file_type_, +// &TableFileSchema::size_, +// &TableFileSchema::date_), +// where(c(&TableFileSchema::file_type_) == (int) TableFileSchema::RAW and +// c(&TableFileSchema::table_id_) == table_id)); +// auto end_time = METRICS_NOW_TIME; +// auto total_time = METRICS_MICROSECONDS(start_time, end_time); +// server::Metrics::GetInstance().MetaAccessDurationSecondsHistogramObserve(total_time); +// TableSchema table_schema; +// table_schema.table_id_ = table_id; +// auto status = DescribeTable(table_schema); +// +// if (!status.ok()) { +// return status; +// } +// +// TableFileSchema table_file; +// for (auto &file : selected) { +// table_file.id_ = std::get<0>(file); +// table_file.table_id_ = std::get<1>(file); +// table_file.file_id_ = std::get<2>(file); +// table_file.file_type_ = std::get<3>(file); +// table_file.size_ = std::get<4>(file); +// table_file.date_ = std::get<5>(file); +// table_file.dimension_ = table_schema.dimension_; +// GetTableFilePath(table_file); +// auto dateItr = files.find(table_file.date_); +// if (dateItr == files.end()) { +// files[table_file.date_] = TableFilesSchema(); +// } +// files[table_file.date_].push_back(table_file); +// } +// } catch (std::exception &e) { +// HandleException(e); +// } + + return Status::OK(); + } + + Status MySQLMetaImpl::FilesToDelete(const std::string& table_id, + const DatesT& partition, + DatePartionedTableFilesSchema& files) { +// auto now = utils::GetMicroSecTimeStamp(); +// try { +// if(partition.empty()) { +// //step 1: get table files by dates +// auto selected = ConnectorPtr->select(columns(&TableFileSchema::id_, +// &TableFileSchema::table_id_, +// &TableFileSchema::file_id_, +// &TableFileSchema::size_, +// &TableFileSchema::date_), +// where(c(&TableFileSchema::file_type_) != +// (int) TableFileSchema::TO_DELETE +// and c(&TableFileSchema::table_id_) == table_id)); +// +// //step 2: erase table files from meta +// for (auto &file : selected) { +// TableFileSchema table_file; +// table_file.id_ = std::get<0>(file); +// table_file.table_id_ = std::get<1>(file); +// table_file.file_id_ = std::get<2>(file); +// table_file.size_ = std::get<3>(file); +// table_file.date_ = std::get<4>(file); +// GetTableFilePath(table_file); +// auto dateItr = files.find(table_file.date_); +// if (dateItr == files.end()) { +// files[table_file.date_] = TableFilesSchema(); +// } +// files[table_file.date_].push_back(table_file); +// +// ConnectorPtr->remove(std::get<0>(file)); +// } +// +// } else { +// //step 1: get all table files +// auto selected = ConnectorPtr->select(columns(&TableFileSchema::id_, +// &TableFileSchema::table_id_, +// &TableFileSchema::file_id_, +// &TableFileSchema::size_, +// &TableFileSchema::date_), +// where(c(&TableFileSchema::file_type_) != +// (int) TableFileSchema::TO_DELETE +// and in(&TableFileSchema::date_, partition) +// and c(&TableFileSchema::table_id_) == table_id)); +// +// //step 2: erase table files from meta +// for (auto &file : selected) { +// TableFileSchema table_file; +// table_file.id_ = std::get<0>(file); +// table_file.table_id_ = std::get<1>(file); +// table_file.file_id_ = std::get<2>(file); +// table_file.size_ = std::get<3>(file); +// table_file.date_ = std::get<4>(file); +// GetTableFilePath(table_file); +// auto dateItr = files.find(table_file.date_); +// if (dateItr == files.end()) { +// files[table_file.date_] = TableFilesSchema(); +// } +// files[table_file.date_].push_back(table_file); +// +// ConnectorPtr->remove(std::get<0>(file)); +// } +// } +// +// } catch (std::exception &e) { +// HandleException(e); +// } + + return Status::OK(); + } + + Status MySQLMetaImpl::GetTableFile(TableFileSchema &file_schema) { + +// try { +// auto files = ConnectorPtr->select(columns(&TableFileSchema::id_, +// &TableFileSchema::table_id_, +// &TableFileSchema::file_id_, +// &TableFileSchema::file_type_, +// &TableFileSchema::size_, +// &TableFileSchema::date_), +// where(c(&TableFileSchema::file_id_) == file_schema.file_id_ and +// c(&TableFileSchema::table_id_) == file_schema.table_id_ +// )); +// assert(files.size() <= 1); +// if (files.size() == 1) { +// file_schema.id_ = std::get<0>(files[0]); +// file_schema.table_id_ = std::get<1>(files[0]); +// file_schema.file_id_ = std::get<2>(files[0]); +// file_schema.file_type_ = std::get<3>(files[0]); +// file_schema.size_ = std::get<4>(files[0]); +// file_schema.date_ = std::get<5>(files[0]); +// } else { +// return Status::NotFound("Table:" + file_schema.table_id_ + +// " File:" + file_schema.file_id_ + " not found"); +// } +// } catch (std::exception &e) { +// HandleException(e); +// } + + return Status::OK(); + } + +// PXU TODO: Support Swap + Status MySQLMetaImpl::Archive() { +// auto &criterias = options_.archive_conf.GetCriterias(); +// if (criterias.size() == 0) { +// return Status::OK(); +// } +// +// for (auto kv : criterias) { +// auto &criteria = kv.first; +// auto &limit = kv.second; +// if (criteria == "days") { +// long usecs = limit * D_SEC * US_PS; +// long now = utils::GetMicroSecTimeStamp(); +// try { +// ConnectorPtr->update_all( +// set( +// c(&TableFileSchema::file_type_) = (int) TableFileSchema::TO_DELETE +// ), +// where( +// c(&TableFileSchema::created_on_) < (long) (now - usecs) and +// c(&TableFileSchema::file_type_) != (int) TableFileSchema::TO_DELETE +// )); +// } catch (std::exception &e) { +// HandleException(e); +// } +// } +// if (criteria == "disk") { +// uint64_t sum = 0; +// Size(sum); +// +// auto to_delete = (sum - limit * G); +// DiscardFiles(to_delete); +// } +// } + + return Status::OK(); + } + + Status MySQLMetaImpl::Size(uint64_t &result) { +// result = 0; +// try { +// auto selected = ConnectorPtr->select(columns(sum(&TableFileSchema::size_)), +// where( +// c(&TableFileSchema::file_type_) != (int) TableFileSchema::TO_DELETE +// )); +// +// for (auto &sub_query : selected) { +// if (!std::get<0>(sub_query)) { +// continue; +// } +// result += (uint64_t) (*std::get<0>(sub_query)); +// } +// } catch (std::exception &e) { +// HandleException(e); +// } + + return Status::OK(); + } + + Status MySQLMetaImpl::DiscardFiles(long to_discard_size) { +// LOG(DEBUG) << "About to discard size=" << to_discard_size; +// if (to_discard_size <= 0) { +// return Status::OK(); +// } +// try { +// auto selected = ConnectorPtr->select(columns(&TableFileSchema::id_, +// &TableFileSchema::size_), +// where(c(&TableFileSchema::file_type_) +// != (int) TableFileSchema::TO_DELETE), +// order_by(&TableFileSchema::id_), +// limit(10)); +// +// std::vector ids; +// TableFileSchema table_file; +// +// for (auto &file : selected) { +// if (to_discard_size <= 0) break; +// table_file.id_ = std::get<0>(file); +// table_file.size_ = std::get<1>(file); +// ids.push_back(table_file.id_); +// ENGINE_LOG_DEBUG << "Discard table_file.id=" << table_file.file_id_ +// << " table_file.size=" << table_file.size_; +// to_discard_size -= table_file.size_; +// } +// +// if (ids.size() == 0) { +// return Status::OK(); +// } +// +// ConnectorPtr->update_all( +// set( +// c(&TableFileSchema::file_type_) = (int) TableFileSchema::TO_DELETE +// ), +// where( +// in(&TableFileSchema::id_, ids) +// )); +// +// } catch (std::exception &e) { +// HandleException(e); +// } + + return DiscardFiles(to_discard_size); + } + + Status MySQLMetaImpl::UpdateTableFile(TableFileSchema &file_schema) { +// file_schema.updated_time_ = utils::GetMicroSecTimeStamp(); +// try { +// server::Metrics::GetInstance().MetaAccessTotalIncrement(); +// auto start_time = METRICS_NOW_TIME; +// ConnectorPtr->update(file_schema); +// auto end_time = METRICS_NOW_TIME; +// auto total_time = METRICS_MICROSECONDS(start_time, end_time); +// server::Metrics::GetInstance().MetaAccessDurationSecondsHistogramObserve(total_time); +// } catch (std::exception &e) { +// ENGINE_LOG_DEBUG << "table_id= " << file_schema.table_id_ << " file_id=" << file_schema.file_id_; +// HandleException(e); +// } + return Status::OK(); + } + + Status MySQLMetaImpl::UpdateTableFiles(TableFilesSchema &files) { +// try { +// server::Metrics::GetInstance().MetaAccessTotalIncrement(); +// auto start_time = METRICS_NOW_TIME; +// auto commited = ConnectorPtr->transaction([&]() mutable { +// for (auto &file : files) { +// file.updated_time_ = utils::GetMicroSecTimeStamp(); +// ConnectorPtr->update(file); +// } +// auto end_time = METRICS_NOW_TIME; +// auto total_time = METRICS_MICROSECONDS(start_time, end_time); +// server::Metrics::GetInstance().MetaAccessDurationSecondsHistogramObserve(total_time); +// return true; +// }); +// if (!commited) { +// return Status::DBTransactionError("Update files Error"); +// } +// } catch (std::exception &e) { +// HandleException(e); +// } + return Status::OK(); + } + + Status MySQLMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) { +// auto now = utils::GetMicroSecTimeStamp(); +// try { +// auto selected = ConnectorPtr->select(columns(&TableFileSchema::id_, +// &TableFileSchema::table_id_, +// &TableFileSchema::file_id_, +// &TableFileSchema::file_type_, +// &TableFileSchema::size_, +// &TableFileSchema::date_), +// where( +// c(&TableFileSchema::file_type_) == (int) TableFileSchema::TO_DELETE +// and +// c(&TableFileSchema::updated_time_) +// > now - seconds * US_PS)); +// +// TableFilesSchema updated; +// TableFileSchema table_file; +// +// for (auto &file : selected) { +// table_file.id_ = std::get<0>(file); +// table_file.table_id_ = std::get<1>(file); +// table_file.file_id_ = std::get<2>(file); +// table_file.file_type_ = std::get<3>(file); +// table_file.size_ = std::get<4>(file); +// table_file.date_ = std::get<5>(file); +// GetTableFilePath(table_file); +// if (table_file.file_type_ == TableFileSchema::TO_DELETE) { +// boost::filesystem::remove(table_file.location_); +// } +// ConnectorPtr->remove(table_file.id_); +// /* LOG(DEBUG) << "Removing deleted id=" << table_file.id << " location=" << table_file.location << std::endl; */ +// } +// } catch (std::exception &e) { +// HandleException(e); +// } + + return Status::OK(); + } + + Status MySQLMetaImpl::CleanUp() { +// try { +// auto selected = ConnectorPtr->select(columns(&TableFileSchema::id_, +// &TableFileSchema::table_id_, +// &TableFileSchema::file_id_, +// &TableFileSchema::file_type_, +// &TableFileSchema::size_, +// &TableFileSchema::date_), +// where( +// c(&TableFileSchema::file_type_) == (int) TableFileSchema::TO_DELETE +// or +// c(&TableFileSchema::file_type_) +// == (int) TableFileSchema::NEW)); +// +// TableFilesSchema updated; +// TableFileSchema table_file; +// +// for (auto &file : selected) { +// table_file.id_ = std::get<0>(file); +// table_file.table_id_ = std::get<1>(file); +// table_file.file_id_ = std::get<2>(file); +// table_file.file_type_ = std::get<3>(file); +// table_file.size_ = std::get<4>(file); +// table_file.date_ = std::get<5>(file); +// GetTableFilePath(table_file); +// if (table_file.file_type_ == TableFileSchema::TO_DELETE) { +// boost::filesystem::remove(table_file.location_); +// } +// ConnectorPtr->remove(table_file.id_); +// /* LOG(DEBUG) << "Removing id=" << table_file.id << " location=" << table_file.location << std::endl; */ +// } +// } catch (std::exception &e) { +// HandleException(e); +// } + + return Status::OK(); + } + + Status MySQLMetaImpl::Count(const std::string &table_id, uint64_t &result) { + +// try { +// +// server::Metrics::GetInstance().MetaAccessTotalIncrement(); +// auto start_time = METRICS_NOW_TIME; +// auto selected = ConnectorPtr->select(columns(&TableFileSchema::size_, +// &TableFileSchema::date_), +// where((c(&TableFileSchema::file_type_) == (int) TableFileSchema::RAW or +// c(&TableFileSchema::file_type_) == (int) TableFileSchema::TO_INDEX +// or +// c(&TableFileSchema::file_type_) == (int) TableFileSchema::INDEX) +// and +// c(&TableFileSchema::table_id_) == table_id)); +// auto end_time = METRICS_NOW_TIME; +// auto total_time = METRICS_MICROSECONDS(start_time, end_time); +// server::Metrics::GetInstance().MetaAccessDurationSecondsHistogramObserve(total_time); +// TableSchema table_schema; +// table_schema.table_id_ = table_id; +// auto status = DescribeTable(table_schema); +// +// if (!status.ok()) { +// return status; +// } +// +// result = 0; +// for (auto &file : selected) { +// result += std::get<0>(file); +// } +// +// result /= table_schema.dimension_; +// result /= sizeof(float); +// +// } catch (std::exception &e) { +// HandleException(e); +// } + return Status::OK(); + } + + Status MySQLMetaImpl::DropAll() { +// if (boost::filesystem::is_directory(options_.path)) { +// boost::filesystem::remove_all(options_.path); +// } + return Status::OK(); + } + + MySQLMetaImpl::~MySQLMetaImpl() { + CleanUp(); + } + +} // namespace meta +} // namespace engine +} // namespace milvus +} // namespace zilliz diff --git a/cpp/unittest/db/MySQLMetaImpl_test.cpp b/cpp/unittest/db/MySQLMetaImpl_test.cpp new file mode 100644 index 0000000000..23243a2a83 --- /dev/null +++ b/cpp/unittest/db/MySQLMetaImpl_test.cpp @@ -0,0 +1,28 @@ +//////////////////////////////////////////////////////////////////////////////// +// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved +// Unauthorized copying of this file, via any medium is strictly prohibited. +// Proprietary and confidential. +//////////////////////////////////////////////////////////////////////////////// +#include +#include +#include +#include +#include + +#include "utils.h" +#include "db/MySQLMetaImpl.h" +#include "db/Factories.h" +#include "db/Utils.h" +#include "db/MetaConsts.h" + +using namespace zilliz::milvus::engine; + +TEST_F(MySQLTest, InitializeTest) { + DBMetaOptions options; + //dialect+driver://username:password@host:port/database + options.backend_uri = "mysql://root:1234@:/test"; + meta::MySQLMetaImpl impl(options); + auto status = impl.Initialize(); + std::cout << status.ToString() << std::endl; + //ASSERT_TRUE(status.ok()); +} \ No newline at end of file From 75a9b4fbd9f069319cdd99104b028b09a18d0778 Mon Sep 17 00:00:00 2001 From: zhiru Date: Thu, 20 Jun 2019 10:02:02 +0800 Subject: [PATCH 07/43] finished MySQLMetaImpl; need to test with DBImpl Former-commit-id: f3a9848f5f2d7a8b15945ae824827371316d0b79 --- cpp/src/db/DBMetaImpl.cpp | 2 +- cpp/src/db/MetaTypes.h | 2 +- cpp/src/db/MySQLMetaImpl.cpp | 1722 ++++++++++++++---------- cpp/src/db/MySQLMetaImpl.h | 3 +- cpp/unittest/CMakeLists.txt | 4 +- cpp/unittest/db/MySQLMetaImpl_test.cpp | 427 +++++- cpp/unittest/db/utils.h | 4 +- 7 files changed, 1431 insertions(+), 733 deletions(-) diff --git a/cpp/src/db/DBMetaImpl.cpp b/cpp/src/db/DBMetaImpl.cpp index 26185b84b3..ba9fe8d815 100644 --- a/cpp/src/db/DBMetaImpl.cpp +++ b/cpp/src/db/DBMetaImpl.cpp @@ -164,7 +164,7 @@ Status DBMetaImpl::DropPartitionsByDates(const std::string &table_id, Status DBMetaImpl::CreateTable(TableSchema &table_schema) { server::Metrics::GetInstance().MetaAccessTotalIncrement(); - if (table_schema.table_id_ == "") { + if (table_schema.table_id_.empty()) { NextTableId(table_schema.table_id_); } table_schema.files_cnt_ = 0; diff --git a/cpp/src/db/MetaTypes.h b/cpp/src/db/MetaTypes.h index a284d14a19..c54a974b91 100644 --- a/cpp/src/db/MetaTypes.h +++ b/cpp/src/db/MetaTypes.h @@ -24,7 +24,7 @@ struct TableSchema { size_t id_; std::string table_id_; size_t files_cnt_ = 0; - uint16_t dimension_; + uint16_t dimension_ = 0; std::string location_; long created_on_; int engine_type_ = (int)EngineType::FAISS_IDMAP; diff --git a/cpp/src/db/MySQLMetaImpl.cpp b/cpp/src/db/MySQLMetaImpl.cpp index 5dacb3f3b3..210930049d 100644 --- a/cpp/src/db/MySQLMetaImpl.cpp +++ b/cpp/src/db/MySQLMetaImpl.cpp @@ -29,8 +29,7 @@ namespace meta { using namespace mysqlpp; -// static std::unique_ptr connectionPtr(new Connection()); - static Connection* connectionPtr = new Connection(); + static std::unique_ptr connectionPtr(new Connection()); namespace { @@ -79,23 +78,19 @@ namespace meta { MySQLMetaImpl::MySQLMetaImpl(const DBMetaOptions &options_) : options_(options_) { - //Initialize(); + Initialize(); } Status MySQLMetaImpl::Initialize() { -// if (!boost::filesystem::is_directory(options_.path)) { -// auto ret = boost::filesystem::create_directory(options_.path); -// if (!ret) { -// ENGINE_LOG_ERROR << "Create directory " << options_.path << " Error"; -// } -// assert(ret); -// } -// ConnectorPtr = std::make_unique(StoragePrototype(options_.path + "/meta.sqlite")); -// -// ConnectorPtr->sync_schema(); -// ConnectorPtr->open_forever(); // thread safe option -// ConnectorPtr->pragma.journal_mode(journal_mode::WAL); // WAL => write ahead log + std::string path = options_.path; + if (!boost::filesystem::is_directory(path)) { + auto ret = boost::filesystem::create_directory(path); + if (!ret) { + ENGINE_LOG_ERROR << "Create directory " << path << " Error"; + } + assert(ret); + } std::string uri = options_.backend_uri; @@ -129,7 +124,7 @@ namespace meta { } const char* dbName = pieces_match[6].str().c_str(); //std::cout << dbName << " " << serverAddress << " " << username << " " << password << " " << port << std::endl; - connectionPtr->set_option(new MultiStatementsOption(true)); +// connectionPtr->set_option(new MultiStatementsOption(true)); try { if (!connectionPtr->connect(dbName, serverAddress, username, password, port)) { @@ -137,42 +132,54 @@ namespace meta { } CleanUp(); - Query InitializeQuery = connectionPtr->query(); - InitializeQuery << "DROP TABLE IF EXISTS meta, metaFile;"; - InitializeQuery << "CREATE TABLE meta (" << - "id BIGINT AUTO INCREMENT PRIMARY KEY, " << - "table_id VARCHAR(255) UNIQUE, " << - "dimension SMALLINT, " << - "created_on BIGINT, " << - "files_cnt BIGINT DEFAULT 0, " << - "engine_type INT DEFAULT 1, " << - "store_raw_data BOOL DEFAULT false);"; - InitializeQuery << "CREATE TABLE metaFile (" << - "id BIGINT AUTO INCREMENT PRIMARY KEY, " << - "table_id VARCHAR(255), " << - "engine_type INT DEFAULT 1, " << - "file_id VARCHAR(255), " << - "file_type INT DEFAULT 0, " << - "size BIGINT DEFAULT 0, " << - "updated_time BIGINT, " << - "created_on BIGINT, " << - "date INT DEFAULT -1);"; - - if (InitializeQuery.exec()) { - return Status::OK(); - } else { - return Status::DBTransactionError("Initialization Error: ", InitializeQuery.error()); +// InitializeQuery << "DROP TABLE IF EXISTS meta, metaFile;"; + InitializeQuery << "CREATE TABLE IF NOT EXISTS meta (" << + "id BIGINT PRIMARY KEY AUTO_INCREMENT, " << + "table_id VARCHAR(255) UNIQUE NOT NULL, " << + "dimension SMALLINT NOT NULL, " << + "created_on BIGINT NOT NULL, " << + "files_cnt BIGINT DEFAULT 0 NOT NULL, " << + "engine_type INT DEFAULT 1 NOT NULL, " << + "store_raw_data BOOL DEFAULT false NOT NULL);"; + if (!InitializeQuery.exec()) { + return Status::DBTransactionError("Initialization Error", InitializeQuery.error()); } + + InitializeQuery << "CREATE TABLE IF NOT EXISTS metaFile (" << + "id BIGINT PRIMARY KEY AUTO_INCREMENT, " << + "table_id VARCHAR(255) NOT NULL, " << + "engine_type INT DEFAULT 1 NOT NULL, " << + "file_id VARCHAR(255) NOT NULL, " << + "file_type INT DEFAULT 0 NOT NULL, " << + "size BIGINT DEFAULT 0 NOT NULL, " << + "updated_time BIGINT NOT NULL, " << + "created_on BIGINT NOT NULL, " << + "date INT DEFAULT -1 NOT NULL);"; + if (!InitializeQuery.exec()) { + return Status::DBTransactionError("Initialization Error", InitializeQuery.error()); + } + + return Status::OK(); + +// if (InitializeQuery.exec()) { +// std::cout << "XXXXXXXXXXXXXXXXXXXXXXXXX" << std::endl; +// while (InitializeQuery.more_results()) { +// InitializeQuery.store_next(); +// } +// return Status::OK(); +// } else { +// return Status::DBTransactionError("Initialization Error", InitializeQuery.error()); +// } } catch (const ConnectionFailed& er) { - return Status::DBTransactionError("Failed to connect to MySQL server: ", er.what()); + return Status::DBTransactionError("Failed to connect to database server", er.what()); } catch (const BadQuery& er) { // Handle any query errors - return Status::DBTransactionError("QUERY ERROR DURING INITIALIZATION: ", er.what()); + return Status::DBTransactionError("QUERY ERROR DURING INITIALIZATION", er.what()); } catch (const Exception& er) { // Catch-all for any other MySQL++ exceptions - return Status::DBTransactionError("GENERAL ERROR DURING INITIALIZATION: ", er.what()); + return Status::DBTransactionError("GENERAL ERROR DURING INITIALIZATION", er.what()); } } else { @@ -183,272 +190,404 @@ namespace meta { // PXU TODO: Temp solution. Will fix later Status MySQLMetaImpl::DropPartitionsByDates(const std::string &table_id, const DatesT &dates) { -// if (dates.size() == 0) { -// return Status::OK(); -// } -// -// TableSchema table_schema; -// table_schema.table_id_ = table_id; -// auto status = DescribeTable(table_schema); -// if (!status.ok()) { -// return status; -// } -// -// auto yesterday = GetDateWithDelta(-1); -// -// for (auto &date : dates) { -// if (date >= yesterday) { -// return Status::Error("Could not delete partitions with 2 days"); -// } -// } -// -// try { -// ConnectorPtr->update_all( -// set( -// c(&TableFileSchema::file_type_) = (int) TableFileSchema::TO_DELETE -// ), -// where( -// c(&TableFileSchema::table_id_) == table_id and -// in(&TableFileSchema::date_, dates) -// )); -// } catch (std::exception &e) { -// HandleException(e); -// } + if (dates.size() == 0) { + return Status::OK(); + } + + TableSchema table_schema; + table_schema.table_id_ = table_id; + auto status = DescribeTable(table_schema); + if (!status.ok()) { + return status; + } + + auto yesterday = GetDateWithDelta(-1); + + for (auto &date : dates) { + if (date >= yesterday) { + return Status::Error("Could not delete partitions within 2 days"); + } + } + + try { + + Query dropPartitionsByDatesQuery = connectionPtr->query(); + + std::stringstream dateListSS; + for (auto &date : dates) { + dateListSS << std::to_string(date) << ", "; + } + std::string dateListStr = dateListSS.str(); + dateListStr = dateListStr.substr(0, dateListStr.size() - 2); //remove the last ", " + + dropPartitionsByDatesQuery << "UPDATE metaFile " << + "SET file_type = " << std::to_string(TableFileSchema::TO_DELETE) << " " << + "WHERE table_id = " << quote << table_id << " AND " << + "date in (" << dateListStr << ");"; + + if (!dropPartitionsByDatesQuery.exec()) { + return Status::DBTransactionError("QUERY ERROR WHEN DROPPING PARTITIONS BY DATES", dropPartitionsByDatesQuery.error()); + } + + } catch (const BadQuery& er) { + // Handle any query errors + return Status::DBTransactionError("QUERY ERROR WHEN DROPPING PARTITIONS BY DATES", er.what()); + } catch (const Exception& er) { + // Catch-all for any other MySQL++ exceptions + return Status::DBTransactionError("GENERAL ERROR WHEN DROPPING PARTITIONS BY DATES", er.what()); + } return Status::OK(); } Status MySQLMetaImpl::CreateTable(TableSchema &table_schema) { -// server::Metrics::GetInstance().MetaAccessTotalIncrement(); -// if (table_schema.table_id_.empty()) { -// NextTableId(table_schema.table_id_); -// } -// table_schema.files_cnt_ = 0; -// table_schema.id_ = -1; -// table_schema.created_on_ = utils::GetMicroSecTimeStamp(); -// auto start_time = METRICS_NOW_TIME; -// { -// try { -// Query addTableQuery = connectionPtr->query(); -// -// } catch (...) { -// return Status::DBTransactionError("Add Table Error"); -// } -// } -// auto end_time = METRICS_NOW_TIME; -// auto total_time = METRICS_MICROSECONDS(start_time, end_time); -// server::Metrics::GetInstance().MetaAccessDurationSecondsHistogramObserve(total_time); -// -// auto table_path = GetTablePath(table_schema.table_id_); -// table_schema.location_ = table_path; -// if (!boost::filesystem::is_directory(table_path)) { -// auto ret = boost::filesystem::create_directories(table_path); -// if (!ret) { -// ENGINE_LOG_ERROR << "Create directory " << table_path << " Error"; -// } -// assert(ret); -// } + server::Metrics::GetInstance().MetaAccessTotalIncrement(); + if (table_schema.table_id_.empty()) { + NextTableId(table_schema.table_id_); + } + table_schema.files_cnt_ = 0; + table_schema.id_ = -1; + table_schema.created_on_ = utils::GetMicroSecTimeStamp(); + auto start_time = METRICS_NOW_TIME; + { + try { + Query createTableQuery = connectionPtr->query(); + std::string id = "NULL"; //auto-increment + std::string table_id = table_schema.table_id_; + std::string dimension = std::to_string(table_schema.dimension_); + std::string created_on = std::to_string(table_schema.created_on_); + std::string files_cnt = "0"; + std::string engine_type = std::to_string(table_schema.engine_type_); + std::string store_raw_data = table_schema.store_raw_data_ ? "true" : "false"; + createTableQuery << "INSERT INTO meta VALUES" << + "(" << id << ", " << quote << table_id << ", " << dimension << ", " << + created_on << ", " << files_cnt << ", " << engine_type << ", " << store_raw_data + << ");"; + if (SimpleResult res = createTableQuery.execute()) { + table_schema.id_ = res.insert_id(); //Might need to use SELECT LAST_INSERT_ID()? +// std::cout << table_schema.id_ << std::endl; + } + else { + return Status::DBTransactionError("Add Table Error", createTableQuery.error()); + } + + } catch (const BadQuery& er) { + // Handle any query errors + return Status::DBTransactionError("QUERY ERROR WHEN ADDING TABLE", er.what()); + } catch (const Exception& er) { + // Catch-all for any other MySQL++ exceptions + return Status::DBTransactionError("GENERAL ERROR WHEN ADDING TABLE", er.what()); + } + } + auto end_time = METRICS_NOW_TIME; + auto total_time = METRICS_MICROSECONDS(start_time, end_time); + server::Metrics::GetInstance().MetaAccessDurationSecondsHistogramObserve(total_time); + + auto table_path = GetTablePath(table_schema.table_id_); + table_schema.location_ = table_path; + if (!boost::filesystem::is_directory(table_path)) { + auto ret = boost::filesystem::create_directories(table_path); + if (!ret) { + ENGINE_LOG_ERROR << "Create directory " << table_path << " Error"; + } + assert(ret); + } return Status::OK(); } Status MySQLMetaImpl::DeleteTable(const std::string& table_id) { -// try { -// //drop the table from meta -// auto tables = ConnectorPtr->select(columns(&TableSchema::id_), -// where(c(&TableSchema::table_id_) == table_id)); -// for (auto &table : tables) { -// ConnectorPtr->remove(std::get<0>(table)); -// } -// } catch (std::exception &e) { -// HandleException(e); -// } - - return Status::OK(); + try { + //drop the table from meta + Query deleteTableQuery = connectionPtr->query(); + deleteTableQuery << "DELETE FROM meta WHERE table_id = " << quote << table_id << ";"; + if (deleteTableQuery.exec()) { + return Status::OK(); + } + else { + return Status::DBTransactionError("Delete Table Error", deleteTableQuery.error()); + } + } catch (const BadQuery& er) { + // Handle any query errors + return Status::DBTransactionError("QUERY ERROR WHEN DELETING TABLE", er.what()); + } catch (const Exception& er) { + // Catch-all for any other MySQL++ exceptions + return Status::DBTransactionError("GENERAL ERROR WHEN DELETING TABLE", er.what()); + } } Status MySQLMetaImpl::DescribeTable(TableSchema &table_schema) { -// try { -// server::Metrics::GetInstance().MetaAccessTotalIncrement(); -// auto start_time = METRICS_NOW_TIME; -// auto groups = ConnectorPtr->select(columns(&TableSchema::id_, -// &TableSchema::table_id_, -// &TableSchema::files_cnt_, -// &TableSchema::dimension_, -// &TableSchema::engine_type_, -// &TableSchema::store_raw_data_), -// where(c(&TableSchema::table_id_) == table_schema.table_id_)); -// auto end_time = METRICS_NOW_TIME; -// auto total_time = METRICS_MICROSECONDS(start_time, end_time); -// server::Metrics::GetInstance().MetaAccessDurationSecondsHistogramObserve(total_time); -// assert(groups.size() <= 1); -// if (groups.size() == 1) { -// table_schema.id_ = std::get<0>(groups[0]); -// table_schema.files_cnt_ = std::get<2>(groups[0]); -// table_schema.dimension_ = std::get<3>(groups[0]); -// table_schema.engine_type_ = std::get<4>(groups[0]); -// table_schema.store_raw_data_ = std::get<5>(groups[0]); -// } else { -// return Status::NotFound("Table " + table_schema.table_id_ + " not found"); -// } -// -// auto table_path = GetTablePath(table_schema.table_id_); -// table_schema.location_ = table_path; -// -// } catch (std::exception &e) { -// HandleException(e); -// } + try { + server::Metrics::GetInstance().MetaAccessTotalIncrement(); + auto start_time = METRICS_NOW_TIME; + + Query describeTableQuery = connectionPtr->query(); + describeTableQuery << "SELECT id, table_id, dimension, files_cnt, engine_type, store_raw_data " << + "FROM meta " << + "WHERE table_id = " << quote << table_schema.table_id_ << ";"; + StoreQueryResult res = describeTableQuery.store(); + + auto end_time = METRICS_NOW_TIME; + auto total_time = METRICS_MICROSECONDS(start_time, end_time); + server::Metrics::GetInstance().MetaAccessDurationSecondsHistogramObserve(total_time); + + assert(res && res.num_rows() <= 1); + if (res.num_rows() == 1) { + const Row& resRow = res[0]; + +// std::string id; +// resRow["id"].to_string(id); +// table_schema.id_ = std::stoul(id); + table_schema.id_ = resRow["id"]; //implicit conversion + + std::string table_id; + resRow["table_id"].to_string(table_id); + table_schema.table_id_ = table_id; + +// std::string created_on; +// resRow["created_on"].to_string(created_on); +// table_schema.created_on_ = std::stol(created_on); + table_schema.dimension_ = resRow["dimension"]; + +// std::string files_cnt; +// resRow["files_cnt"].to_string(files_cnt); +// table_schema.files_cnt_ = std::stoul(files_cnt); + table_schema.files_cnt_ = resRow["files_cnt"]; + +// std::string engine_type; +// resRow["engine_type"].to_string(engine_type); +// table_schema.engine_type_ = std::stoi(engine_type); + table_schema.engine_type_ = resRow["engine_type"]; + + table_schema.store_raw_data_ = (resRow["store_raw_data"].compare("true") == 0); + } + else { + return Status::NotFound("Table " + table_schema.table_id_ + " not found"); + } + + auto table_path = GetTablePath(table_schema.table_id_); + table_schema.location_ = table_path; + + } catch (const BadQuery& er) { + // Handle any query errors + return Status::DBTransactionError("QUERY ERROR WHEN DESCRIBING TABLE", er.what()); + } catch (const Exception& er) { + // Catch-all for any other MySQL++ exceptions + return Status::DBTransactionError("GENERAL ERROR WHEN DESCRIBING TABLE", er.what()); + } return Status::OK(); } Status MySQLMetaImpl::HasTable(const std::string &table_id, bool &has_or_not) { -// try { -// server::Metrics::GetInstance().MetaAccessTotalIncrement(); -// auto start_time = METRICS_NOW_TIME; -// -// auto tables = ConnectorPtr->select(columns(&TableSchema::id_), -// where(c(&TableSchema::table_id_) == table_id)); -// auto end_time = METRICS_NOW_TIME; -// auto total_time = METRICS_MICROSECONDS(start_time, end_time); -// server::Metrics::GetInstance().MetaAccessDurationSecondsHistogramObserve(total_time); -// assert(tables.size() <= 1); -// if (tables.size() == 1) { -// has_or_not = true; -// } else { -// has_or_not = false; -// } -// } catch (std::exception &e) { -// HandleException(e); -// } + try { + server::Metrics::GetInstance().MetaAccessTotalIncrement(); + auto start_time = METRICS_NOW_TIME; + + Query hasTableQuery = connectionPtr->query(); + //since table_id is a unique column we just need to check whether it exists or not + hasTableQuery << "SELECT EXISTS (SELECT 1 FROM meta WHERE table_id = " << quote << table_id << ") " + << "AS " << quote << "check" << ";"; + StoreQueryResult res = hasTableQuery.store(); + + auto end_time = METRICS_NOW_TIME; + auto total_time = METRICS_MICROSECONDS(start_time, end_time); + server::Metrics::GetInstance().MetaAccessDurationSecondsHistogramObserve(total_time); + + assert(res && res.num_rows() == 1); + int check = res[0]["check"]; + has_or_not = (check == 1); + + } catch (const BadQuery& er) { + // Handle any query errors + return Status::DBTransactionError("QUERY ERROR WHEN CHECKING IF TABLE EXISTS", er.what()); + } catch (const Exception& er) { + // Catch-all for any other MySQL++ exceptions + return Status::DBTransactionError("GENERAL ERROR WHEN CHECKING IF TABLE EXISTS", er.what()); + } + return Status::OK(); } Status MySQLMetaImpl::AllTables(std::vector& table_schema_array) { -// try { -// server::Metrics::GetInstance().MetaAccessTotalIncrement(); -// auto start_time = METRICS_NOW_TIME; -// auto selected = ConnectorPtr->select(columns(&TableSchema::id_, -// &TableSchema::table_id_, -// &TableSchema::files_cnt_, -// &TableSchema::dimension_, -// &TableSchema::engine_type_, -// &TableSchema::store_raw_data_)); -// auto end_time = METRICS_NOW_TIME; -// auto total_time = METRICS_MICROSECONDS(start_time, end_time); -// server::Metrics::GetInstance().MetaAccessDurationSecondsHistogramObserve(total_time); -// for (auto &table : selected) { -// TableSchema schema; -// schema.id_ = std::get<0>(table); -// schema.table_id_ = std::get<1>(table); -// schema.files_cnt_ = std::get<2>(table); -// schema.dimension_ = std::get<3>(table); -// schema.engine_type_ = std::get<4>(table); -// schema.store_raw_data_ = std::get<5>(table); -// -// table_schema_array.emplace_back(schema); -// } -// } catch (std::exception &e) { -// HandleException(e); -// } + try { + server::Metrics::GetInstance().MetaAccessTotalIncrement(); + auto start_time = METRICS_NOW_TIME; + + Query allTablesQuery = connectionPtr->query(); + allTablesQuery << "SELECT id, table_id, dimension, files_cnt, engine_type, store_raw_data " << + "FROM meta;"; + StoreQueryResult res = allTablesQuery.store(); + + auto end_time = METRICS_NOW_TIME; + auto total_time = METRICS_MICROSECONDS(start_time, end_time); + server::Metrics::GetInstance().MetaAccessDurationSecondsHistogramObserve(total_time); + + for (auto& resRow : res) { + TableSchema table_schema; + + table_schema.id_ = resRow["id"]; //implicit conversion + + std::string table_id; + resRow["table_id"].to_string(table_id); + table_schema.table_id_ = table_id; + + table_schema.dimension_ = resRow["dimension"]; + + table_schema.files_cnt_ = resRow["files_cnt"]; + + table_schema.engine_type_ = resRow["engine_type"]; + + table_schema.store_raw_data_ = (resRow["store_raw_data"].compare("true") == 0); + + table_schema_array.emplace_back(table_schema); + } + } catch (const BadQuery& er) { + // Handle any query errors + return Status::DBTransactionError("QUERY ERROR WHEN DESCRIBING ALL TABLES", er.what()); + } catch (const Exception& er) { + // Catch-all for any other MySQL++ exceptions + return Status::DBTransactionError("GENERAL ERROR WHEN DESCRIBING ALL TABLES", er.what()); + } return Status::OK(); } Status MySQLMetaImpl::CreateTableFile(TableFileSchema &file_schema) { -// if (file_schema.date_ == EmptyDate) { -// file_schema.date_ = Meta::GetDate(); -// } -// TableSchema table_schema; -// table_schema.table_id_ = file_schema.table_id_; -// auto status = DescribeTable(table_schema); -// if (!status.ok()) { -// return status; -// } -// -// NextFileId(file_schema.file_id_); -// file_schema.file_type_ = TableFileSchema::NEW; -// file_schema.dimension_ = table_schema.dimension_; -// file_schema.size_ = 0; -// file_schema.created_on_ = utils::GetMicroSecTimeStamp(); -// file_schema.updated_time_ = file_schema.created_on_; -// file_schema.engine_type_ = table_schema.engine_type_; -// GetTableFilePath(file_schema); -// -// { -// try { -// server::Metrics::GetInstance().MetaAccessTotalIncrement(); -// auto start_time = METRICS_NOW_TIME; -// auto id = ConnectorPtr->insert(file_schema); -// file_schema.id_ = id; -// auto end_time = METRICS_NOW_TIME; -// auto total_time = METRICS_MICROSECONDS(start_time, end_time); -// server::Metrics::GetInstance().MetaAccessDurationSecondsHistogramObserve(total_time); -// } catch (...) { -// return Status::DBTransactionError("Add file Error"); -// } -// } -// -// auto partition_path = GetTableDatePartitionPath(file_schema.table_id_, file_schema.date_); -// -// if (!boost::filesystem::is_directory(partition_path)) { -// auto ret = boost::filesystem::create_directory(partition_path); -// if (!ret) { -// ENGINE_LOG_ERROR << "Create directory " << partition_path << " Error"; -// } -// assert(ret); -// } + if (file_schema.date_ == EmptyDate) { + file_schema.date_ = Meta::GetDate(); + } + TableSchema table_schema; + table_schema.table_id_ = file_schema.table_id_; + auto status = DescribeTable(table_schema); + if (!status.ok()) { + return status; + } + + NextFileId(file_schema.file_id_); + file_schema.file_type_ = TableFileSchema::NEW; + file_schema.dimension_ = table_schema.dimension_; + file_schema.size_ = 0; + file_schema.created_on_ = utils::GetMicroSecTimeStamp(); + file_schema.updated_time_ = file_schema.created_on_; + file_schema.engine_type_ = table_schema.engine_type_; + GetTableFilePath(file_schema); + + { + try { + server::Metrics::GetInstance().MetaAccessTotalIncrement(); + auto start_time = METRICS_NOW_TIME; + + Query createTableFileQuery = connectionPtr->query(); + std::string id = "NULL"; //auto-increment + std::string table_id = file_schema.table_id_; + std::string engine_type = std::to_string(file_schema.engine_type_); + std::string file_id = file_schema.file_id_; + std::string file_type = std::to_string(file_schema.file_type_); + std::string size = std::to_string(file_schema.size_); + std::string updated_time = std::to_string(file_schema.updated_time_); + std::string created_on = std::to_string(file_schema.created_on_); + std::string date = std::to_string(file_schema.date_); + + createTableFileQuery << "INSERT INTO metaFile VALUES" << + "(" << id << ", " << quote << table_id << ", " << engine_type << ", " << + quote << file_id << ", " << file_type << ", " << size << ", " << + updated_time << ", " << created_on << ", " << date << ");"; + + if (SimpleResult res = createTableFileQuery.execute()) { + file_schema.id_ = res.insert_id(); //Might need to use SELECT LAST_INSERT_ID()? + } + else { + return Status::DBTransactionError("Add file Error", createTableFileQuery.error()); + } + + auto end_time = METRICS_NOW_TIME; + auto total_time = METRICS_MICROSECONDS(start_time, end_time); + server::Metrics::GetInstance().MetaAccessDurationSecondsHistogramObserve(total_time); + } catch (const BadQuery& er) { + // Handle any query errors + return Status::DBTransactionError("QUERY ERROR WHEN ADDING TABLE FILE", er.what()); + } catch (const Exception& er) { + // Catch-all for any other MySQL++ exceptions + return Status::DBTransactionError("GENERAL ERROR WHEN ADDING TABLE FILE", er.what()); + } + } + + auto partition_path = GetTableDatePartitionPath(file_schema.table_id_, file_schema.date_); + + if (!boost::filesystem::is_directory(partition_path)) { + auto ret = boost::filesystem::create_directory(partition_path); + if (!ret) { + ENGINE_LOG_ERROR << "Create directory " << partition_path << " Error"; + } + assert(ret); + } return Status::OK(); } Status MySQLMetaImpl::FilesToIndex(TableFilesSchema &files) { -// files.clear(); -// -// try { -// server::Metrics::GetInstance().MetaAccessTotalIncrement(); -// auto start_time = METRICS_NOW_TIME; -// auto selected = ConnectorPtr->select(columns(&TableFileSchema::id_, -// &TableFileSchema::table_id_, -// &TableFileSchema::file_id_, -// &TableFileSchema::file_type_, -// &TableFileSchema::size_, -// &TableFileSchema::date_, -// &TableFileSchema::engine_type_), -// where(c(&TableFileSchema::file_type_) -// == (int) TableFileSchema::TO_INDEX)); -// auto end_time = METRICS_NOW_TIME; -// auto total_time = METRICS_MICROSECONDS(start_time, end_time); -// server::Metrics::GetInstance().MetaAccessDurationSecondsHistogramObserve(total_time); -// -// std::map groups; -// TableFileSchema table_file; -// -// for (auto &file : selected) { -// table_file.id_ = std::get<0>(file); -// table_file.table_id_ = std::get<1>(file); -// table_file.file_id_ = std::get<2>(file); -// table_file.file_type_ = std::get<3>(file); -// table_file.size_ = std::get<4>(file); -// table_file.date_ = std::get<5>(file); -// table_file.engine_type_ = std::get<6>(file); -// -// GetTableFilePath(table_file); -// auto groupItr = groups.find(table_file.table_id_); -// if (groupItr == groups.end()) { -// TableSchema table_schema; -// table_schema.table_id_ = table_file.table_id_; -// auto status = DescribeTable(table_schema); -// if (!status.ok()) { -// return status; -// } -// groups[table_file.table_id_] = table_schema; -// } -// table_file.dimension_ = groups[table_file.table_id_].dimension_; -// files.push_back(table_file); -// } -// } catch (std::exception &e) { -// HandleException(e); -// } + files.clear(); + + try { + server::Metrics::GetInstance().MetaAccessTotalIncrement(); + auto start_time = METRICS_NOW_TIME; + + Query filesToIndexQuery = connectionPtr->query(); + filesToIndexQuery << "SELECT id, table_id, engine_type, file_id, file_type, size, date " << + "FROM metaFile " << + "WHERE file_type = " << std::to_string(TableFileSchema::TO_INDEX) << ";"; + StoreQueryResult res = filesToIndexQuery.store(); + + auto end_time = METRICS_NOW_TIME; + auto total_time = METRICS_MICROSECONDS(start_time, end_time); + server::Metrics::GetInstance().MetaAccessDurationSecondsHistogramObserve(total_time); + + std::map groups; + TableFileSchema table_file; + for (auto& resRow : res) { + + table_file.id_ = resRow["id"]; //implicit conversion + + std::string table_id; + resRow["table_id"].to_string(table_id); + table_file.table_id_ = table_id; + + table_file.engine_type_ = resRow["engine_type"]; + + std::string file_id; + resRow["file_id"].to_string(file_id); + table_file.file_id_ = file_id; + + table_file.file_type_ = resRow["file_type"]; + + table_file.size_ = resRow["size"]; + + table_file.date_ = resRow["date"]; + + auto groupItr = groups.find(table_file.table_id_); + if (groupItr == groups.end()) { + TableSchema table_schema; + table_schema.table_id_ = table_file.table_id_; + auto status = DescribeTable(table_schema); + if (!status.ok()) { + return status; + } + groups[table_file.table_id_] = table_schema; +// std::cout << table_schema.dimension_ << std::endl; + } + table_file.dimension_ = groups[table_file.table_id_].dimension_; + + GetTableFilePath(table_file); + + files.push_back(table_file); + } + } catch (const BadQuery& er) { + // Handle any query errors + return Status::DBTransactionError("QUERY ERROR WHEN FINDING TABLE FILES TO INDEX", er.what()); + } catch (const Exception& er) { + // Catch-all for any other MySQL++ exceptions + return Status::DBTransactionError("GENERAL ERROR WHEN FINDING TABLE FILES TO INDEX", er.what()); + } return Status::OK(); } @@ -456,440 +595,521 @@ namespace meta { Status MySQLMetaImpl::FilesToSearch(const std::string &table_id, const DatesT &partition, DatePartionedTableFilesSchema &files) { -// files.clear(); -// -// try { -// server::Metrics::GetInstance().MetaAccessTotalIncrement(); -// auto start_time = METRICS_NOW_TIME; -// if (partition.empty()) { -// auto selected = ConnectorPtr->select(columns(&TableFileSchema::id_, -// &TableFileSchema::table_id_, -// &TableFileSchema::file_id_, -// &TableFileSchema::file_type_, -// &TableFileSchema::size_, -// &TableFileSchema::date_, -// &TableFileSchema::engine_type_), -// where(c(&TableFileSchema::table_id_) == table_id and -// (c(&TableFileSchema::file_type_) == (int) TableFileSchema::RAW or -// c(&TableFileSchema::file_type_) -// == (int) TableFileSchema::TO_INDEX or -// c(&TableFileSchema::file_type_) -// == (int) TableFileSchema::INDEX))); -// auto end_time = METRICS_NOW_TIME; -// auto total_time = METRICS_MICROSECONDS(start_time, end_time); -// server::Metrics::GetInstance().MetaAccessDurationSecondsHistogramObserve(total_time); -// TableSchema table_schema; -// table_schema.table_id_ = table_id; -// auto status = DescribeTable(table_schema); -// if (!status.ok()) { -// return status; -// } -// -// TableFileSchema table_file; -// -// for (auto &file : selected) { -// table_file.id_ = std::get<0>(file); -// table_file.table_id_ = std::get<1>(file); -// table_file.file_id_ = std::get<2>(file); -// table_file.file_type_ = std::get<3>(file); -// table_file.size_ = std::get<4>(file); -// table_file.date_ = std::get<5>(file); -// table_file.engine_type_ = std::get<6>(file); -// table_file.dimension_ = table_schema.dimension_; -// GetTableFilePath(table_file); -// auto dateItr = files.find(table_file.date_); -// if (dateItr == files.end()) { -// files[table_file.date_] = TableFilesSchema(); -// } -// files[table_file.date_].push_back(table_file); -// } -// } -// else { -// auto selected = ConnectorPtr->select(columns(&TableFileSchema::id_, -// &TableFileSchema::table_id_, -// &TableFileSchema::file_id_, -// &TableFileSchema::file_type_, -// &TableFileSchema::size_, -// &TableFileSchema::date_), -// where(c(&TableFileSchema::table_id_) == table_id and -// in(&TableFileSchema::date_, partition) and -// (c(&TableFileSchema::file_type_) == (int) TableFileSchema::RAW or -// c(&TableFileSchema::file_type_) -// == (int) TableFileSchema::TO_INDEX or -// c(&TableFileSchema::file_type_) -// == (int) TableFileSchema::INDEX))); -// auto end_time = METRICS_NOW_TIME; -// auto total_time = METRICS_MICROSECONDS(start_time, end_time); -// server::Metrics::GetInstance().MetaAccessDurationSecondsHistogramObserve(total_time); -// TableSchema table_schema; -// table_schema.table_id_ = table_id; -// auto status = DescribeTable(table_schema); -// if (!status.ok()) { -// return status; -// } -// -// TableFileSchema table_file; -// -// for (auto &file : selected) { -// table_file.id_ = std::get<0>(file); -// table_file.table_id_ = std::get<1>(file); -// table_file.file_id_ = std::get<2>(file); -// table_file.file_type_ = std::get<3>(file); -// table_file.size_ = std::get<4>(file); -// table_file.date_ = std::get<5>(file); -// table_file.dimension_ = table_schema.dimension_; -// GetTableFilePath(table_file); -// auto dateItr = files.find(table_file.date_); -// if (dateItr == files.end()) { -// files[table_file.date_] = TableFilesSchema(); -// } -// files[table_file.date_].push_back(table_file); -// } -// -// } -// } catch (std::exception &e) { -// HandleException(e); -// } + files.clear(); + + try { + server::Metrics::GetInstance().MetaAccessTotalIncrement(); + auto start_time = METRICS_NOW_TIME; + + StoreQueryResult res; + + if (partition.empty()) { + + Query filesToSearchQuery = connectionPtr->query(); + filesToSearchQuery << "SELECT id, table_id, engine_type, file_id, file_type, size, date " << + "FROM metaFile " << + "WHERE table_id = " << quote << table_id << " AND " << + "(file_type = " << std::to_string(TableFileSchema::RAW) << " OR " << + "file_type = " << std::to_string(TableFileSchema::TO_INDEX) << " OR " << + "file_type = " << std::to_string(TableFileSchema::INDEX) << ");"; + res = filesToSearchQuery.store(); + + auto end_time = METRICS_NOW_TIME; + auto total_time = METRICS_MICROSECONDS(start_time, end_time); + server::Metrics::GetInstance().MetaAccessDurationSecondsHistogramObserve(total_time); + } + else { + + Query filesToSearchQuery = connectionPtr->query(); + + std::stringstream partitionListSS; + for (auto &date : partition) { + partitionListSS << std::to_string(date) << ", "; + } + std::string partitionListStr = partitionListSS.str(); + partitionListStr = partitionListStr.substr(0, partitionListStr.size() - 2); //remove the last ", " + + filesToSearchQuery << "SELECT id, table_id, engine_type, file_id, file_type, size, date " << + "FROM metaFile " << + "WHERE table_id = " << quote << table_id << " AND " << + "date IN (" << partitionListStr << ") AND " << + "(file_type = " << std::to_string(TableFileSchema::RAW) << " OR " << + "file_type = " << std::to_string(TableFileSchema::TO_INDEX) << " OR " << + "file_type = " << std::to_string(TableFileSchema::INDEX) << ");"; + res = filesToSearchQuery.store(); + + auto end_time = METRICS_NOW_TIME; + auto total_time = METRICS_MICROSECONDS(start_time, end_time); + server::Metrics::GetInstance().MetaAccessDurationSecondsHistogramObserve(total_time); + } + + TableSchema table_schema; + table_schema.table_id_ = table_id; + auto status = DescribeTable(table_schema); + if (!status.ok()) { + return status; + } + + TableFileSchema table_file; + for (auto& resRow : res) { + + table_file.id_ = resRow["id"]; //implicit conversion + + std::string table_id_str; + resRow["table_id"].to_string(table_id_str); + table_file.table_id_ = table_id_str; + + table_file.engine_type_ = resRow["engine_type"]; + + std::string file_id; + resRow["file_id"].to_string(file_id); + table_file.file_id_ = file_id; + + table_file.file_type_ = resRow["file_type"]; + + table_file.size_ = resRow["size"]; + + table_file.date_ = resRow["date"]; + + table_file.dimension_ = table_schema.dimension_; + + GetTableFilePath(table_file); + + auto dateItr = files.find(table_file.date_); + if (dateItr == files.end()) { + files[table_file.date_] = TableFilesSchema(); + } + + files[table_file.date_].push_back(table_file); + } + } catch (const BadQuery& er) { + // Handle any query errors + return Status::DBTransactionError("QUERY ERROR WHEN FINDING TABLE FILES TO SEARCH", er.what()); + } catch (const Exception& er) { + // Catch-all for any other MySQL++ exceptions + return Status::DBTransactionError("GENERAL ERROR WHEN FINDING TABLE FILES TO SEARCH", er.what()); + } return Status::OK(); } Status MySQLMetaImpl::FilesToMerge(const std::string &table_id, DatePartionedTableFilesSchema &files) { -// files.clear(); -// -// try { -// server::Metrics::GetInstance().MetaAccessTotalIncrement(); -// auto start_time = METRICS_NOW_TIME; -// auto selected = ConnectorPtr->select(columns(&TableFileSchema::id_, -// &TableFileSchema::table_id_, -// &TableFileSchema::file_id_, -// &TableFileSchema::file_type_, -// &TableFileSchema::size_, -// &TableFileSchema::date_), -// where(c(&TableFileSchema::file_type_) == (int) TableFileSchema::RAW and -// c(&TableFileSchema::table_id_) == table_id)); -// auto end_time = METRICS_NOW_TIME; -// auto total_time = METRICS_MICROSECONDS(start_time, end_time); -// server::Metrics::GetInstance().MetaAccessDurationSecondsHistogramObserve(total_time); -// TableSchema table_schema; -// table_schema.table_id_ = table_id; -// auto status = DescribeTable(table_schema); -// -// if (!status.ok()) { -// return status; -// } -// -// TableFileSchema table_file; -// for (auto &file : selected) { -// table_file.id_ = std::get<0>(file); -// table_file.table_id_ = std::get<1>(file); -// table_file.file_id_ = std::get<2>(file); -// table_file.file_type_ = std::get<3>(file); -// table_file.size_ = std::get<4>(file); -// table_file.date_ = std::get<5>(file); -// table_file.dimension_ = table_schema.dimension_; -// GetTableFilePath(table_file); -// auto dateItr = files.find(table_file.date_); -// if (dateItr == files.end()) { -// files[table_file.date_] = TableFilesSchema(); -// } -// files[table_file.date_].push_back(table_file); -// } -// } catch (std::exception &e) { -// HandleException(e); -// } + files.clear(); + + try { + server::Metrics::GetInstance().MetaAccessTotalIncrement(); + auto start_time = METRICS_NOW_TIME; + + Query filesToMergeQuery = connectionPtr->query(); + filesToMergeQuery << "SELECT id, table_id, file_id, file_type, size, date " << + "FROM metaFile " << + "WHERE table_id = " << quote << table_id << " AND " << + "file_type = " << std::to_string(TableFileSchema::RAW) << ";"; + StoreQueryResult res = filesToMergeQuery.store(); + + auto end_time = METRICS_NOW_TIME; + auto total_time = METRICS_MICROSECONDS(start_time, end_time); + server::Metrics::GetInstance().MetaAccessDurationSecondsHistogramObserve(total_time); + + TableSchema table_schema; + table_schema.table_id_ = table_id; + auto status = DescribeTable(table_schema); + + if (!status.ok()) { + return status; + } + + TableFileSchema table_file; + for (auto& resRow : res) { + + table_file.id_ = resRow["id"]; //implicit conversion + + std::string table_id_str; + resRow["table_id"].to_string(table_id_str); + table_file.table_id_ = table_id_str; + + std::string file_id; + resRow["file_id"].to_string(file_id); + table_file.file_id_ = file_id; + + table_file.file_type_ = resRow["file_type"]; + + table_file.size_ = resRow["size"]; + + table_file.date_ = resRow["date"]; + + table_file.dimension_ = table_schema.dimension_; + + GetTableFilePath(table_file); + + auto dateItr = files.find(table_file.date_); + if (dateItr == files.end()) { + files[table_file.date_] = TableFilesSchema(); + } + + files[table_file.date_].push_back(table_file); + } + + } catch (const BadQuery& er) { + // Handle any query errors + return Status::DBTransactionError("QUERY ERROR WHEN FINDING TABLE FILES TO MERGE", er.what()); + } catch (const Exception& er) { + // Catch-all for any other MySQL++ exceptions + return Status::DBTransactionError("GENERAL ERROR WHEN FINDING TABLE FILES TO MERGE", er.what()); + } return Status::OK(); } + //ZR: TODO: this function is pending to be removed, so not gonna implemented for now Status MySQLMetaImpl::FilesToDelete(const std::string& table_id, const DatesT& partition, DatePartionedTableFilesSchema& files) { -// auto now = utils::GetMicroSecTimeStamp(); -// try { -// if(partition.empty()) { -// //step 1: get table files by dates -// auto selected = ConnectorPtr->select(columns(&TableFileSchema::id_, -// &TableFileSchema::table_id_, -// &TableFileSchema::file_id_, -// &TableFileSchema::size_, -// &TableFileSchema::date_), -// where(c(&TableFileSchema::file_type_) != -// (int) TableFileSchema::TO_DELETE -// and c(&TableFileSchema::table_id_) == table_id)); -// -// //step 2: erase table files from meta -// for (auto &file : selected) { -// TableFileSchema table_file; -// table_file.id_ = std::get<0>(file); -// table_file.table_id_ = std::get<1>(file); -// table_file.file_id_ = std::get<2>(file); -// table_file.size_ = std::get<3>(file); -// table_file.date_ = std::get<4>(file); -// GetTableFilePath(table_file); -// auto dateItr = files.find(table_file.date_); -// if (dateItr == files.end()) { -// files[table_file.date_] = TableFilesSchema(); -// } -// files[table_file.date_].push_back(table_file); -// -// ConnectorPtr->remove(std::get<0>(file)); -// } -// -// } else { -// //step 1: get all table files -// auto selected = ConnectorPtr->select(columns(&TableFileSchema::id_, -// &TableFileSchema::table_id_, -// &TableFileSchema::file_id_, -// &TableFileSchema::size_, -// &TableFileSchema::date_), -// where(c(&TableFileSchema::file_type_) != -// (int) TableFileSchema::TO_DELETE -// and in(&TableFileSchema::date_, partition) -// and c(&TableFileSchema::table_id_) == table_id)); -// -// //step 2: erase table files from meta -// for (auto &file : selected) { -// TableFileSchema table_file; -// table_file.id_ = std::get<0>(file); -// table_file.table_id_ = std::get<1>(file); -// table_file.file_id_ = std::get<2>(file); -// table_file.size_ = std::get<3>(file); -// table_file.date_ = std::get<4>(file); -// GetTableFilePath(table_file); -// auto dateItr = files.find(table_file.date_); -// if (dateItr == files.end()) { -// files[table_file.date_] = TableFilesSchema(); -// } -// files[table_file.date_].push_back(table_file); -// -// ConnectorPtr->remove(std::get<0>(file)); -// } -// } -// -// } catch (std::exception &e) { -// HandleException(e); -// } return Status::OK(); } Status MySQLMetaImpl::GetTableFile(TableFileSchema &file_schema) { -// try { -// auto files = ConnectorPtr->select(columns(&TableFileSchema::id_, -// &TableFileSchema::table_id_, -// &TableFileSchema::file_id_, -// &TableFileSchema::file_type_, -// &TableFileSchema::size_, -// &TableFileSchema::date_), -// where(c(&TableFileSchema::file_id_) == file_schema.file_id_ and -// c(&TableFileSchema::table_id_) == file_schema.table_id_ -// )); -// assert(files.size() <= 1); -// if (files.size() == 1) { -// file_schema.id_ = std::get<0>(files[0]); -// file_schema.table_id_ = std::get<1>(files[0]); -// file_schema.file_id_ = std::get<2>(files[0]); -// file_schema.file_type_ = std::get<3>(files[0]); -// file_schema.size_ = std::get<4>(files[0]); -// file_schema.date_ = std::get<5>(files[0]); -// } else { -// return Status::NotFound("Table:" + file_schema.table_id_ + -// " File:" + file_schema.file_id_ + " not found"); -// } -// } catch (std::exception &e) { -// HandleException(e); -// } + try { + + Query getTableFileQuery = connectionPtr->query(); + getTableFileQuery << "SELECT id, table_id, file_id, file_type, size, date " << + "FROM metaFile " << + "WHERE file_id = " << quote << file_schema.file_id_ << " AND " << + "table_id = " << quote << file_schema.table_id_ << ";"; + StoreQueryResult res = getTableFileQuery.store(); + + assert(res && res.num_rows() <= 1); + if (res.num_rows() == 1) { + + const Row& resRow = res[0]; + + file_schema.id_ = resRow["id"]; //implicit conversion + + std::string table_id; + resRow["table_id"].to_string(table_id); + file_schema.table_id_ = table_id; + + std::string file_id; + resRow["file_id"].to_string(file_id); + file_schema.file_id_ = file_id; + + file_schema.file_type_ = resRow["file_type"]; + + file_schema.size_ = resRow["size"]; + + file_schema.date_ = resRow["date"]; + } + else { + return Status::NotFound("Table:" + file_schema.table_id_ + + " File:" + file_schema.file_id_ + " not found"); + } + } catch (const BadQuery& er) { + // Handle any query errors + return Status::DBTransactionError("QUERY ERROR WHEN RETRIEVING TABLE FILE", er.what()); + } catch (const Exception& er) { + // Catch-all for any other MySQL++ exceptions + return Status::DBTransactionError("GENERAL ERROR WHEN RETRIEVING TABLE FILE", er.what()); + } return Status::OK(); } // PXU TODO: Support Swap Status MySQLMetaImpl::Archive() { -// auto &criterias = options_.archive_conf.GetCriterias(); -// if (criterias.size() == 0) { -// return Status::OK(); -// } -// -// for (auto kv : criterias) { -// auto &criteria = kv.first; -// auto &limit = kv.second; -// if (criteria == "days") { -// long usecs = limit * D_SEC * US_PS; -// long now = utils::GetMicroSecTimeStamp(); -// try { -// ConnectorPtr->update_all( -// set( -// c(&TableFileSchema::file_type_) = (int) TableFileSchema::TO_DELETE -// ), -// where( -// c(&TableFileSchema::created_on_) < (long) (now - usecs) and -// c(&TableFileSchema::file_type_) != (int) TableFileSchema::TO_DELETE -// )); -// } catch (std::exception &e) { -// HandleException(e); -// } -// } -// if (criteria == "disk") { -// uint64_t sum = 0; -// Size(sum); -// -// auto to_delete = (sum - limit * G); -// DiscardFiles(to_delete); -// } -// } + auto &criterias = options_.archive_conf.GetCriterias(); + if (criterias.empty()) { + return Status::OK(); + } + + for (auto& kv : criterias) { + auto &criteria = kv.first; + auto &limit = kv.second; + if (criteria == "days") { + size_t usecs = limit * D_SEC * US_PS; + long now = utils::GetMicroSecTimeStamp(); + try { + + Query archiveQuery = connectionPtr->query(); + archiveQuery << "UPDATE metaFile " << + "SET file_type = " << std::to_string(TableFileSchema::TO_DELETE) << " " << + "WHERE created_on < " << std::to_string(now - usecs) << " AND " << + "file_type <> " << std::to_string(TableFileSchema::TO_DELETE) << ";"; + if (!archiveQuery.exec()) { + return Status::DBTransactionError("QUERY ERROR DURING ARCHIVE", archiveQuery.error()); + } + + } catch (const BadQuery& er) { + // Handle any query errors + return Status::DBTransactionError("QUERY ERROR WHEN DURING ARCHIVE", er.what()); + } catch (const Exception& er) { + // Catch-all for any other MySQL++ exceptions + return Status::DBTransactionError("GENERAL ERROR WHEN DURING ARCHIVE", er.what()); + } + } + if (criteria == "disk") { + uint64_t sum = 0; + Size(sum); + + auto to_delete = (sum - limit * G); + DiscardFiles(to_delete); + } + } return Status::OK(); } Status MySQLMetaImpl::Size(uint64_t &result) { -// result = 0; -// try { -// auto selected = ConnectorPtr->select(columns(sum(&TableFileSchema::size_)), -// where( -// c(&TableFileSchema::file_type_) != (int) TableFileSchema::TO_DELETE -// )); -// -// for (auto &sub_query : selected) { -// if (!std::get<0>(sub_query)) { -// continue; -// } -// result += (uint64_t) (*std::get<0>(sub_query)); -// } -// } catch (std::exception &e) { -// HandleException(e); -// } + result = 0; + try { + + Query getSizeQuery = connectionPtr->query(); + getSizeQuery << "SELECT SUM(size) AS sum " << + "FROM metaFile " << + "WHERE file_type <> " << std::to_string(TableFileSchema::TO_DELETE) << ";"; + StoreQueryResult res = getSizeQuery.store(); + + assert(res && res.num_rows() == 1); + result = res[0]["sum"]; + + } catch (const BadQuery& er) { + // Handle any query errors + return Status::DBTransactionError("QUERY ERROR WHEN RETRIEVING SIZE", er.what()); + } catch (const Exception& er) { + // Catch-all for any other MySQL++ exceptions + return Status::DBTransactionError("GENERAL ERROR WHEN RETRIEVING SIZE", er.what()); + } return Status::OK(); } Status MySQLMetaImpl::DiscardFiles(long to_discard_size) { -// LOG(DEBUG) << "About to discard size=" << to_discard_size; -// if (to_discard_size <= 0) { -// return Status::OK(); -// } -// try { -// auto selected = ConnectorPtr->select(columns(&TableFileSchema::id_, -// &TableFileSchema::size_), -// where(c(&TableFileSchema::file_type_) -// != (int) TableFileSchema::TO_DELETE), -// order_by(&TableFileSchema::id_), -// limit(10)); -// -// std::vector ids; -// TableFileSchema table_file; -// -// for (auto &file : selected) { -// if (to_discard_size <= 0) break; -// table_file.id_ = std::get<0>(file); -// table_file.size_ = std::get<1>(file); -// ids.push_back(table_file.id_); -// ENGINE_LOG_DEBUG << "Discard table_file.id=" << table_file.file_id_ -// << " table_file.size=" << table_file.size_; -// to_discard_size -= table_file.size_; -// } -// -// if (ids.size() == 0) { -// return Status::OK(); -// } -// -// ConnectorPtr->update_all( -// set( -// c(&TableFileSchema::file_type_) = (int) TableFileSchema::TO_DELETE -// ), -// where( -// in(&TableFileSchema::id_, ids) -// )); -// -// } catch (std::exception &e) { -// HandleException(e); -// } + LOG(DEBUG) << "About to discard size=" << to_discard_size; + if (to_discard_size <= 0) { +// std::cout << "in" << std::endl; + return Status::OK(); + } + try { - return DiscardFiles(to_discard_size); + Query discardFilesQuery = connectionPtr->query(); + discardFilesQuery << "SELECT id, size " << + "FROM metaFile " << + "WHERE file_type <> " << std::to_string(TableFileSchema::TO_DELETE) << " " << + "ORDER BY id ASC " << + "LIMIT 10;"; +// std::cout << discardFilesQuery.str() << std::endl; + StoreQueryResult res = discardFilesQuery.store(); + + assert(res); + if (res.num_rows() == 0) { + return Status::OK(); + } + + TableFileSchema table_file; + std::stringstream idsToDiscardSS; + for (auto& resRow : res) { + if (to_discard_size <= 0) { + break; + } + table_file.id_ = resRow["id"]; + table_file.size_ = resRow["size"]; + idsToDiscardSS << "id = " << std::to_string(table_file.id_) << " OR "; + ENGINE_LOG_DEBUG << "Discard table_file.id=" << table_file.file_id_ + << " table_file.size=" << table_file.size_; + to_discard_size -= table_file.size_; + } + + std::string idsToDiscardStr = idsToDiscardSS.str(); + idsToDiscardStr = idsToDiscardStr.substr(0, idsToDiscardStr.size() - 4); //remove the last " OR " + + discardFilesQuery << "UPDATE metaFile " << + "SET file_type = " << std::to_string(TableFileSchema::TO_DELETE) << " " << + "WHERE " << idsToDiscardStr << ";"; + + if (discardFilesQuery.exec()) { + return DiscardFiles(to_discard_size); + } + else { + return Status::DBTransactionError("QUERY ERROR WHEN DISCARDING FILES", discardFilesQuery.error()); + } + + } catch (const BadQuery& er) { + // Handle any query errors + return Status::DBTransactionError("QUERY ERROR WHEN DISCARDING FILES", er.what()); + } catch (const Exception& er) { + // Catch-all for any other MySQL++ exceptions + return Status::DBTransactionError("GENERAL ERROR WHEN DISCARDING FILES", er.what()); + } } + //ZR: this function assumes all fields in file_schema have value Status MySQLMetaImpl::UpdateTableFile(TableFileSchema &file_schema) { -// file_schema.updated_time_ = utils::GetMicroSecTimeStamp(); -// try { -// server::Metrics::GetInstance().MetaAccessTotalIncrement(); -// auto start_time = METRICS_NOW_TIME; -// ConnectorPtr->update(file_schema); -// auto end_time = METRICS_NOW_TIME; -// auto total_time = METRICS_MICROSECONDS(start_time, end_time); -// server::Metrics::GetInstance().MetaAccessDurationSecondsHistogramObserve(total_time); -// } catch (std::exception &e) { -// ENGINE_LOG_DEBUG << "table_id= " << file_schema.table_id_ << " file_id=" << file_schema.file_id_; -// HandleException(e); -// } + file_schema.updated_time_ = utils::GetMicroSecTimeStamp(); + try { + server::Metrics::GetInstance().MetaAccessTotalIncrement(); + auto start_time = METRICS_NOW_TIME; + + Query updateTableFileQuery = connectionPtr->query(); + + std::string id = std::to_string(file_schema.id_); + std::string table_id = file_schema.table_id_; + std::string engine_type = std::to_string(file_schema.engine_type_); + std::string file_id = file_schema.file_id_; + std::string file_type = std::to_string(file_schema.file_type_); + std::string size = std::to_string(file_schema.size_); + std::string updated_time = std::to_string(file_schema.updated_time_); + std::string created_on = std::to_string(file_schema.created_on_); + std::string date = std::to_string(file_schema.date_); + + updateTableFileQuery << "UPDATE metaFile " << + "SET table_id = " << quote << table_id << ", " << + "engine_type = " << engine_type << ", " << + "file_id = " << quote << file_id << ", " << + "file_type = " << file_type << ", " << + "size = " << size << ", " << + "updated_time = " << updated_time << ", " << + "created_on = " << created_on << ", " << + "date = " << date << " " << + "WHERE id = " << id << ";"; + +// std::cout << updateTableFileQuery.str() << std::endl; + + if (!updateTableFileQuery.exec()) { + ENGINE_LOG_DEBUG << "table_id= " << file_schema.table_id_ << " file_id=" << file_schema.file_id_; + return Status::DBTransactionError("QUERY ERROR WHEN UPDATING TABLE FILE", updateTableFileQuery.error()); + } + + auto end_time = METRICS_NOW_TIME; + auto total_time = METRICS_MICROSECONDS(start_time, end_time); + server::Metrics::GetInstance().MetaAccessDurationSecondsHistogramObserve(total_time); + } catch (const BadQuery& er) { + // Handle any query errors + ENGINE_LOG_DEBUG << "table_id= " << file_schema.table_id_ << " file_id=" << file_schema.file_id_; + return Status::DBTransactionError("QUERY ERROR WHEN UPDATING TABLE FILE", er.what()); + } catch (const Exception& er) { + // Catch-all for any other MySQL++ exceptions + ENGINE_LOG_DEBUG << "table_id= " << file_schema.table_id_ << " file_id=" << file_schema.file_id_; + return Status::DBTransactionError("GENERAL ERROR WHEN UPDATING TABLE FILE", er.what()); + } return Status::OK(); } Status MySQLMetaImpl::UpdateTableFiles(TableFilesSchema &files) { -// try { -// server::Metrics::GetInstance().MetaAccessTotalIncrement(); -// auto start_time = METRICS_NOW_TIME; -// auto commited = ConnectorPtr->transaction([&]() mutable { -// for (auto &file : files) { -// file.updated_time_ = utils::GetMicroSecTimeStamp(); -// ConnectorPtr->update(file); -// } -// auto end_time = METRICS_NOW_TIME; -// auto total_time = METRICS_MICROSECONDS(start_time, end_time); -// server::Metrics::GetInstance().MetaAccessDurationSecondsHistogramObserve(total_time); -// return true; -// }); -// if (!commited) { -// return Status::DBTransactionError("Update files Error"); -// } -// } catch (std::exception &e) { -// HandleException(e); -// } + try { + server::Metrics::GetInstance().MetaAccessTotalIncrement(); + auto start_time = METRICS_NOW_TIME; + + Query updateTableFilesQuery = connectionPtr->query(); + + for (auto& file_schema : files) { + + std::string id = std::to_string(file_schema.id_); + std::string table_id = file_schema.table_id_; + std::string engine_type = std::to_string(file_schema.engine_type_); + std::string file_id = file_schema.file_id_; + std::string file_type = std::to_string(file_schema.file_type_); + std::string size = std::to_string(file_schema.size_); + std::string updated_time = std::to_string(file_schema.updated_time_); + std::string created_on = std::to_string(file_schema.created_on_); + std::string date = std::to_string(file_schema.date_); + + updateTableFilesQuery << "UPDATE metaFile " << + "SET table_id = " << quote << table_id << ", " << + "engine_type = " << engine_type << ", " << + "file_id = " << quote << file_id << ", " << + "file_type = " << file_type << ", " << + "size = " << size << ", " << + "updated_time = " << updated_time << ", " << + "created_on = " << created_on << ", " << + "date = " << date << " " << + "WHERE id = " << id << ";"; + + } + + if (!updateTableFilesQuery.exec()) { + return Status::DBTransactionError("QUERY ERROR WHEN UPDATING TABLE FILES", updateTableFilesQuery.error()); + } + + } catch (const BadQuery& er) { + // Handle any query errors + return Status::DBTransactionError("QUERY ERROR WHEN UPDATING TABLE FILES", er.what()); + } catch (const Exception& er) { + // Catch-all for any other MySQL++ exceptions + return Status::DBTransactionError("GENERAL ERROR WHEN UPDATING TABLE FILES", er.what()); + } return Status::OK(); } Status MySQLMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) { -// auto now = utils::GetMicroSecTimeStamp(); -// try { -// auto selected = ConnectorPtr->select(columns(&TableFileSchema::id_, -// &TableFileSchema::table_id_, -// &TableFileSchema::file_id_, -// &TableFileSchema::file_type_, -// &TableFileSchema::size_, -// &TableFileSchema::date_), -// where( -// c(&TableFileSchema::file_type_) == (int) TableFileSchema::TO_DELETE -// and -// c(&TableFileSchema::updated_time_) -// > now - seconds * US_PS)); -// -// TableFilesSchema updated; -// TableFileSchema table_file; -// -// for (auto &file : selected) { -// table_file.id_ = std::get<0>(file); -// table_file.table_id_ = std::get<1>(file); -// table_file.file_id_ = std::get<2>(file); -// table_file.file_type_ = std::get<3>(file); -// table_file.size_ = std::get<4>(file); -// table_file.date_ = std::get<5>(file); -// GetTableFilePath(table_file); -// if (table_file.file_type_ == TableFileSchema::TO_DELETE) { -// boost::filesystem::remove(table_file.location_); -// } -// ConnectorPtr->remove(table_file.id_); -// /* LOG(DEBUG) << "Removing deleted id=" << table_file.id << " location=" << table_file.location << std::endl; */ -// } -// } catch (std::exception &e) { -// HandleException(e); -// } + auto now = utils::GetMicroSecTimeStamp(); + try { + + Query cleanUpFilesWithTTLQuery = connectionPtr->query(); + cleanUpFilesWithTTLQuery << "SELECT id, table_id, file_id, file_type, size, date " << + "FROM metaFile " << + "WHERE file_type = " << std::to_string(TableFileSchema::TO_DELETE) << " AND " << + "updated_time > " << std::to_string(now - seconds * US_PS) << ";"; + StoreQueryResult res = cleanUpFilesWithTTLQuery.store(); + + assert(res); + + TableFileSchema table_file; + std::vector idsToDelete; + + for (auto& resRow : res) { + + table_file.id_ = resRow["id"]; //implicit conversion + + std::string table_id; + resRow["table_id"].to_string(table_id); + table_file.table_id_ = table_id; + + std::string file_id; + resRow["file_id"].to_string(file_id); + table_file.file_id_ = file_id; + + table_file.file_type_ = resRow["file_type"]; + + table_file.size_ = resRow["size"]; + + table_file.date_ = resRow["date"]; + + GetTableFilePath(table_file); + + if (table_file.file_type_ == TableFileSchema::TO_DELETE) { + boost::filesystem::remove(table_file.location_); + } + + idsToDelete.emplace_back(std::to_string(table_file.id_)); + } + + std::stringstream idsToDeleteSS; + for (auto& id : idsToDelete) { + idsToDeleteSS << "id = " << id << " OR "; + } + std::string idsToDeleteStr = idsToDeleteSS.str(); + idsToDeleteStr = idsToDeleteStr.substr(0, idsToDeleteStr.size() - 4); //remove the last " OR " + cleanUpFilesWithTTLQuery << "DELETE FROM metaFile WHERE " << + idsToDeleteStr << ";"; + if (!cleanUpFilesWithTTLQuery.exec()) { + return Status::DBTransactionError("CleanUpFilesWithTTL Error", cleanUpFilesWithTTLQuery.error()); + } + + } catch (const BadQuery& er) { + // Handle any query errors + return Status::DBTransactionError("QUERY ERROR WHEN CLEANING UP FILES WITH TTL", er.what()); + } catch (const Exception& er) { + // Catch-all for any other MySQL++ exceptions + return Status::DBTransactionError("GENERAL ERROR WHEN CLEANING UP FILES WITH TTL", er.what()); + } return Status::OK(); } Status MySQLMetaImpl::CleanUp() { -// try { + try { // auto selected = ConnectorPtr->select(columns(&TableFileSchema::id_, // &TableFileSchema::table_id_, // &TableFileSchema::file_id_, @@ -901,75 +1121,137 @@ namespace meta { // or // c(&TableFileSchema::file_type_) // == (int) TableFileSchema::NEW)); -// -// TableFilesSchema updated; -// TableFileSchema table_file; -// -// for (auto &file : selected) { -// table_file.id_ = std::get<0>(file); -// table_file.table_id_ = std::get<1>(file); -// table_file.file_id_ = std::get<2>(file); -// table_file.file_type_ = std::get<3>(file); -// table_file.size_ = std::get<4>(file); -// table_file.date_ = std::get<5>(file); -// GetTableFilePath(table_file); -// if (table_file.file_type_ == TableFileSchema::TO_DELETE) { -// boost::filesystem::remove(table_file.location_); -// } -// ConnectorPtr->remove(table_file.id_); -// /* LOG(DEBUG) << "Removing id=" << table_file.id << " location=" << table_file.location << std::endl; */ -// } -// } catch (std::exception &e) { -// HandleException(e); -// } + + Query cleanUpQuery = connectionPtr->query(); + cleanUpQuery << "SELECT id, table_id, file_id, file_type, size, date " << + "FROM metaFile " << + "WHERE file_type = " << std::to_string(TableFileSchema::TO_DELETE) << " OR " << + "file_type = " << std::to_string(TableFileSchema::NEW) << ";"; + StoreQueryResult res = cleanUpQuery.store(); + + assert(res); + + TableFileSchema table_file; + std::vector idsToDelete; + + for (auto& resRow : res) { + + table_file.id_ = resRow["id"]; //implicit conversion + + std::string table_id; + resRow["table_id"].to_string(table_id); + table_file.table_id_ = table_id; + + std::string file_id; + resRow["file_id"].to_string(file_id); + table_file.file_id_ = file_id; + + table_file.file_type_ = resRow["file_type"]; + + table_file.size_ = resRow["size"]; + + table_file.date_ = resRow["date"]; + + GetTableFilePath(table_file); + + if (table_file.file_type_ == TableFileSchema::TO_DELETE) { + boost::filesystem::remove(table_file.location_); + } + + idsToDelete.emplace_back(std::to_string(table_file.id_)); + } + + std::stringstream idsToDeleteSS; + for (auto& id : idsToDelete) { + idsToDeleteSS << "id = " << id << " OR "; + } + std::string idsToDeleteStr = idsToDeleteSS.str(); + idsToDeleteStr = idsToDeleteStr.substr(0, idsToDeleteStr.size() - 4); //remove the last " OR " + cleanUpQuery << "DELETE FROM metaFile WHERE " << + idsToDeleteStr << ";"; + if (!cleanUpQuery.exec()) { + return Status::DBTransactionError("Clean up Error", cleanUpQuery.error()); + } + + } catch (const BadQuery& er) { + // Handle any query errors + return Status::DBTransactionError("QUERY ERROR WHEN CLEANING UP FILES", er.what()); + } catch (const Exception& er) { + // Catch-all for any other MySQL++ exceptions + return Status::DBTransactionError("GENERAL ERROR WHEN CLEANING UP FILES", er.what()); + } return Status::OK(); } Status MySQLMetaImpl::Count(const std::string &table_id, uint64_t &result) { -// try { -// -// server::Metrics::GetInstance().MetaAccessTotalIncrement(); -// auto start_time = METRICS_NOW_TIME; -// auto selected = ConnectorPtr->select(columns(&TableFileSchema::size_, -// &TableFileSchema::date_), -// where((c(&TableFileSchema::file_type_) == (int) TableFileSchema::RAW or -// c(&TableFileSchema::file_type_) == (int) TableFileSchema::TO_INDEX -// or -// c(&TableFileSchema::file_type_) == (int) TableFileSchema::INDEX) -// and -// c(&TableFileSchema::table_id_) == table_id)); -// auto end_time = METRICS_NOW_TIME; -// auto total_time = METRICS_MICROSECONDS(start_time, end_time); -// server::Metrics::GetInstance().MetaAccessDurationSecondsHistogramObserve(total_time); -// TableSchema table_schema; -// table_schema.table_id_ = table_id; -// auto status = DescribeTable(table_schema); -// -// if (!status.ok()) { -// return status; -// } -// -// result = 0; -// for (auto &file : selected) { -// result += std::get<0>(file); -// } -// -// result /= table_schema.dimension_; -// result /= sizeof(float); -// -// } catch (std::exception &e) { -// HandleException(e); -// } + try { + + server::Metrics::GetInstance().MetaAccessTotalIncrement(); + auto start_time = METRICS_NOW_TIME; + + Query countQuery = connectionPtr->query(); + countQuery << "SELECT size " << + "FROM metaFile " << + "WHERE table_id = " << quote << table_id << " AND " << + "(file_type = " << std::to_string(TableFileSchema::RAW) << " OR " << + "file_type = " << std::to_string(TableFileSchema::TO_INDEX) << " OR " << + "file_type = " << std::to_string(TableFileSchema::INDEX) << ");"; + StoreQueryResult res = countQuery.store(); + + auto end_time = METRICS_NOW_TIME; + auto total_time = METRICS_MICROSECONDS(start_time, end_time); + server::Metrics::GetInstance().MetaAccessDurationSecondsHistogramObserve(total_time); + + TableSchema table_schema; + table_schema.table_id_ = table_id; + auto status = DescribeTable(table_schema); + + if (!status.ok()) { + return status; + } + + result = 0; + for (auto &resRow : res) { + size_t size = resRow["size"]; + result += size; + } + + assert(table_schema.dimension_ != 0); + result /= table_schema.dimension_; + result /= sizeof(float); + + } catch (const BadQuery& er) { + // Handle any query errors + return Status::DBTransactionError("QUERY ERROR WHEN RETRIEVING COUNT", er.what()); + } catch (const Exception& er) { + // Catch-all for any other MySQL++ exceptions + return Status::DBTransactionError("GENERAL ERROR WHEN RETRIEVING COUNT", er.what()); + } return Status::OK(); } Status MySQLMetaImpl::DropAll() { -// if (boost::filesystem::is_directory(options_.path)) { -// boost::filesystem::remove_all(options_.path); -// } - return Status::OK(); + if (boost::filesystem::is_directory(options_.path)) { + boost::filesystem::remove_all(options_.path); + } + try { + Query dropTableQuery = connectionPtr->query(); + dropTableQuery << "DROP TABLE IF EXISTS meta, metaFile;"; + if (dropTableQuery.exec()) { + return Status::OK(); + } + else { + return Status::DBTransactionError("DROP TABLE ERROR", dropTableQuery.error()); + } + } catch (const BadQuery& er) { + // Handle any query errors + return Status::DBTransactionError("QUERY ERROR WHEN DROPPING TABLE", er.what()); + } catch (const Exception& er) { + // Catch-all for any other MySQL++ exceptions + return Status::DBTransactionError("GENERAL ERROR WHEN DROPPING TABLE", er.what()); + } } MySQLMetaImpl::~MySQLMetaImpl() { diff --git a/cpp/src/db/MySQLMetaImpl.h b/cpp/src/db/MySQLMetaImpl.h index 3e8bcdeeec..e099fbb262 100644 --- a/cpp/src/db/MySQLMetaImpl.h +++ b/cpp/src/db/MySQLMetaImpl.h @@ -19,8 +19,6 @@ namespace meta { public: MySQLMetaImpl(const DBMetaOptions& options_); - Status Initialize(); - virtual Status CreateTable(TableSchema& table_schema) override; virtual Status DeleteTable(const std::string& table_id) override; virtual Status DescribeTable(TableSchema& group_info_) override; @@ -71,6 +69,7 @@ namespace meta { std::string GetTablePath(const std::string& table_id); std::string GetTableDatePartitionPath(const std::string& table_id, DateT& date); void GetTableFilePath(TableFileSchema& group_file); + Status Initialize(); const DBMetaOptions options_; }; // DBMetaImpl diff --git a/cpp/unittest/CMakeLists.txt b/cpp/unittest/CMakeLists.txt index e636ff97cf..25a99968f2 100644 --- a/cpp/unittest/CMakeLists.txt +++ b/cpp/unittest/CMakeLists.txt @@ -6,10 +6,10 @@ link_directories( "${CMAKE_BINARY_DIR}/lib" #"${VECWISE_THIRD_PARTY_BUILD}/lib" - "${GTEST_PREFIX}/lib/" +# "${GTEST_PREFIX}/lib/" ) -message(STATUS "GTEST LIB: ${GTEST_PREFIX}/lib") +#message(STATUS "GTEST LIB: ${GTEST_PREFIX}/lib") set(unittest_srcs ${CMAKE_CURRENT_SOURCE_DIR}/vecwise_test.cpp) diff --git a/cpp/unittest/db/MySQLMetaImpl_test.cpp b/cpp/unittest/db/MySQLMetaImpl_test.cpp index 23243a2a83..2889a07a4e 100644 --- a/cpp/unittest/db/MySQLMetaImpl_test.cpp +++ b/cpp/unittest/db/MySQLMetaImpl_test.cpp @@ -15,14 +15,431 @@ #include "db/Utils.h" #include "db/MetaConsts.h" +#include "mysql++/mysql++.h" + +#include + using namespace zilliz::milvus::engine; -TEST_F(MySQLTest, InitializeTest) { +//TEST_F(MySQLTest, InitializeTest) { +// DBMetaOptions options; +// //dialect+driver://username:password@host:port/database +// options.backend_uri = "mysql://root:1234@:/test"; +// meta::MySQLMetaImpl impl(options); +// auto status = impl.Initialize(); +// std::cout << status.ToString() << std::endl; +// ASSERT_TRUE(status.ok()); +//} + +TEST_F(MySQLTest, core) { DBMetaOptions options; //dialect+driver://username:password@host:port/database options.backend_uri = "mysql://root:1234@:/test"; + options.path = "/tmp/vecwise_test"; meta::MySQLMetaImpl impl(options); - auto status = impl.Initialize(); - std::cout << status.ToString() << std::endl; - //ASSERT_TRUE(status.ok()); -} \ No newline at end of file +// auto status = impl.Initialize(); +// ASSERT_TRUE(status.ok()); + + meta::TableSchema schema1; + schema1.table_id_ = "test1"; + schema1.dimension_ = 123; + + auto status = impl.CreateTable(schema1); +// std::cout << status.ToString() << std::endl; + ASSERT_TRUE(status.ok()); + + meta::TableSchema schema2; + schema2.table_id_ = "test2"; + schema2.dimension_ = 321; + status = impl.CreateTable(schema2); +// std::cout << status.ToString() << std::endl; + ASSERT_TRUE(status.ok()); + + status = impl.CreateTable(schema2); +// std::cout << status.ToString() << std::endl; +// ASSERT_THROW(impl.CreateTable(schema), mysqlpp::BadQuery); + ASSERT_FALSE(status.ok()); + + status = impl.DeleteTable(schema2.table_id_); +// std::cout << status.ToString() << std::endl; + ASSERT_TRUE(status.ok()); + + size_t id1 = schema1.id_; + long created_on1 = schema1.created_on_; + status = impl.DescribeTable(schema1); + ASSERT_TRUE(status.ok()); + ASSERT_EQ(schema1.id_, id1); + ASSERT_EQ(schema1.table_id_, "test1"); + ASSERT_EQ(schema1.created_on_, created_on1); + ASSERT_EQ(schema1.files_cnt_, 0); + ASSERT_EQ(schema1.engine_type_, 1); + ASSERT_EQ(schema1.store_raw_data_, false); + + bool check; + status = impl.HasTable("test1", check); + ASSERT_TRUE(status.ok()); + ASSERT_EQ(check, true); + + std::vector table_schema_array; + status = impl.AllTables(table_schema_array); + ASSERT_TRUE(status.ok()); + ASSERT_EQ(table_schema_array.size(), 1); + meta::TableSchema resultSchema = table_schema_array[0]; + ASSERT_EQ(resultSchema.id_, id1); + ASSERT_EQ(resultSchema.table_id_, "test1"); + ASSERT_EQ(resultSchema.dimension_, 123); + ASSERT_EQ(resultSchema.files_cnt_, 0); + ASSERT_EQ(resultSchema.engine_type_, 1); + ASSERT_EQ(resultSchema.store_raw_data_, false); + + meta::TableFileSchema tableFileSchema; + tableFileSchema.table_id_ = "test1"; + + status = impl.CreateTableFile(tableFileSchema); +// std::cout << status.ToString() << std::endl; + ASSERT_TRUE(status.ok()); + + tableFileSchema.file_type_ = meta::TableFileSchema::TO_INDEX; + status = impl.UpdateTableFile(tableFileSchema); +// std::cout << status.ToString() << std::endl; + ASSERT_TRUE(status.ok()); + + meta::TableFilesSchema filesToIndex; + status = impl.FilesToIndex(filesToIndex); + ASSERT_TRUE(status.ok()); + ASSERT_EQ(filesToIndex.size(), 1); + meta::TableFileSchema fileToIndex = filesToIndex[0]; + ASSERT_EQ(fileToIndex.table_id_, "test1"); + ASSERT_EQ(fileToIndex.dimension_, 123); + +// meta::TableFilesSchema filesToIndex; +// status = impl.FilesToIndex(filesToIndex); +// ASSERT_TRUE(status.ok()); +// ASSERT_EQ(filesToIndex.size(), 0); + + meta::DatesT partition; + partition.push_back(tableFileSchema.date_); + meta::DatePartionedTableFilesSchema filesToSearch; + status = impl.FilesToSearch(tableFileSchema.table_id_, partition, filesToSearch); + ASSERT_TRUE(status.ok()); + ASSERT_EQ(filesToSearch.size(), 1); + ASSERT_EQ(filesToSearch[tableFileSchema.date_].size(), 1); + meta::TableFileSchema fileToSearch = filesToSearch[tableFileSchema.date_][0]; + ASSERT_EQ(fileToSearch.table_id_, "test1"); + ASSERT_EQ(fileToSearch.dimension_, 123); + + tableFileSchema.file_type_ = meta::TableFileSchema::RAW; + status = impl.UpdateTableFile(tableFileSchema); + ASSERT_TRUE(status.ok()); + + meta::DatePartionedTableFilesSchema filesToMerge; + status = impl.FilesToMerge(tableFileSchema.table_id_, filesToMerge); +// std::cout << status.ToString() << std::endl; + ASSERT_TRUE(status.ok()); + ASSERT_EQ(filesToMerge.size(), 1); + ASSERT_EQ(filesToMerge[tableFileSchema.date_].size(), 1); + meta::TableFileSchema fileToMerge = filesToMerge[tableFileSchema.date_][0]; + ASSERT_EQ(fileToMerge.table_id_, "test1"); + ASSERT_EQ(fileToMerge.dimension_, 123); + + meta::TableFileSchema resultTableFileSchema; + resultTableFileSchema.table_id_ = tableFileSchema.table_id_; + resultTableFileSchema.file_id_ = tableFileSchema.file_id_; + status = impl.GetTableFile(resultTableFileSchema); + ASSERT_TRUE(status.ok()); + ASSERT_EQ(resultTableFileSchema.id_, tableFileSchema.id_); + ASSERT_EQ(resultTableFileSchema.table_id_, tableFileSchema.table_id_); + ASSERT_EQ(resultTableFileSchema.file_id_, tableFileSchema.file_id_); + ASSERT_EQ(resultTableFileSchema.file_type_, tableFileSchema.file_type_); + ASSERT_EQ(resultTableFileSchema.size_, tableFileSchema.size_); + ASSERT_EQ(resultTableFileSchema.date_, tableFileSchema.date_); + + tableFileSchema.size_ = 234; + status = impl.CreateTable(schema2); + ASSERT_TRUE(status.ok()); + meta::TableFileSchema tableFileSchema2; + tableFileSchema2.table_id_ = "test2"; + tableFileSchema2.size_ = 345; + status = impl.CreateTableFile(tableFileSchema2); + ASSERT_TRUE(status.ok()); + meta::TableFilesSchema filesToUpdate; + filesToUpdate.emplace_back(tableFileSchema); + filesToUpdate.emplace_back(tableFileSchema2); + status = impl.UpdateTableFile(tableFileSchema); + ASSERT_TRUE(status.ok()); + + uint64_t resultSize; + status = impl.Size(resultSize); +// std::cout << status.ToString() << std::endl; + ASSERT_TRUE(status.ok()); + ASSERT_EQ(resultSize, tableFileSchema.size_ + tableFileSchema2.size_); + + uint64_t countResult; + status = impl.Count(tableFileSchema.table_id_, countResult); + ASSERT_TRUE(status.ok()); + + status = impl.DropAll(); + ASSERT_TRUE(status.ok()); + +} + +TEST_F(MySQLTest, GROUP_TEST) { + + DBMetaOptions options; + options.backend_uri = "mysql://root:1234@:/test"; + options.path = "/tmp/vecwise_test"; + meta::MySQLMetaImpl impl(options); + + auto table_id = "meta_test_group"; + + meta::TableSchema group; + group.table_id_ = table_id; + auto status = impl.CreateTable(group); +// std::cout << status.ToString() << std::endl; + ASSERT_TRUE(status.ok()); + + auto gid = group.id_; + group.id_ = -1; + status = impl.DescribeTable(group); + ASSERT_TRUE(status.ok()); + ASSERT_EQ(group.id_, gid); + ASSERT_EQ(group.table_id_, table_id); + + group.table_id_ = "not_found"; + status = impl.DescribeTable(group); + ASSERT_TRUE(!status.ok()); + + group.table_id_ = table_id; + status = impl.CreateTable(group); + ASSERT_TRUE(!status.ok()); + + status = impl.DropAll(); + ASSERT_TRUE(status.ok()); +} + +TEST_F(MySQLTest, table_file_TEST) { + + DBMetaOptions options; + options.backend_uri = "mysql://root:1234@:/test"; + options.path = "/tmp/vecwise_test"; + meta::MySQLMetaImpl impl(options); + + auto table_id = "meta_test_group"; + + meta::TableSchema group; + group.table_id_ = table_id; + auto status = impl.CreateTable(group); + + meta::TableFileSchema table_file; + table_file.table_id_ = group.table_id_; + status = impl.CreateTableFile(table_file); + ASSERT_TRUE(status.ok()); + ASSERT_EQ(table_file.file_type_, meta::TableFileSchema::NEW); + + auto file_id = table_file.file_id_; + + auto new_file_type = meta::TableFileSchema::INDEX; + table_file.file_type_ = new_file_type; + + status = impl.UpdateTableFile(table_file); + ASSERT_TRUE(status.ok()); + ASSERT_EQ(table_file.file_type_, new_file_type); + + meta::DatesT dates; + dates.push_back(meta::Meta::GetDate()); + status = impl.DropPartitionsByDates(table_file.table_id_, dates); + ASSERT_FALSE(status.ok()); + + dates.clear(); + for (auto i=2; i < 10; ++i) { + dates.push_back(meta::Meta::GetDateWithDelta(-1*i)); + } + status = impl.DropPartitionsByDates(table_file.table_id_, dates); + ASSERT_TRUE(status.ok()); + + table_file.date_ = meta::Meta::GetDateWithDelta(-2); + status = impl.UpdateTableFile(table_file); + ASSERT_TRUE(status.ok()); + ASSERT_EQ(table_file.date_, meta::Meta::GetDateWithDelta(-2)); + ASSERT_FALSE(table_file.file_type_ == meta::TableFileSchema::TO_DELETE); + + dates.clear(); + dates.push_back(table_file.date_); + status = impl.DropPartitionsByDates(table_file.table_id_, dates); + ASSERT_TRUE(status.ok()); + status = impl.GetTableFile(table_file); + ASSERT_TRUE(status.ok()); + ASSERT_TRUE(table_file.file_type_ == meta::TableFileSchema::TO_DELETE); + + status = impl.DropAll(); + ASSERT_TRUE(status.ok()); +} + +TEST_F(MySQLTest, ARCHIVE_TEST_DAYS) { + srand(time(0)); + DBMetaOptions options; + options.path = "/tmp/vecwise_test"; + int days_num = rand() % 100; + std::stringstream ss; + ss << "days:" << days_num; + options.archive_conf = ArchiveConf("delete", ss.str()); + options.backend_uri = "mysql://root:1234@:/test"; + + meta::MySQLMetaImpl impl(options); + + auto table_id = "meta_test_group"; + + meta::TableSchema group; + group.table_id_ = table_id; + auto status = impl.CreateTable(group); + + meta::TableFilesSchema files; + meta::TableFileSchema table_file; + table_file.table_id_ = group.table_id_; + + auto cnt = 100; + long ts = utils::GetMicroSecTimeStamp(); + std::vector days; + for (auto i=0; i impl_; +// std::shared_ptr impl_; }; From b102998ad6ad0dcafc1d3a08f80d37fb6803c334 Mon Sep 17 00:00:00 2001 From: zhiru Date: Thu, 20 Jun 2019 16:14:58 +0800 Subject: [PATCH 08/43] update Former-commit-id: 865e3d32a9cc754345b1ed89bdbe95cfd8232fc8 --- cpp/src/db/DBImpl.cpp | 4 ++- cpp/src/db/Factories.cpp | 48 ++++++++++++++++++++++++++ cpp/src/db/Factories.h | 2 ++ cpp/unittest/db/MySQLMetaImpl_test.cpp | 46 ++++++++++++------------ cpp/unittest/db/utils.cpp | 9 +++++ cpp/unittest/db/utils.h | 8 ++++- 6 files changed, 92 insertions(+), 25 deletions(-) diff --git a/cpp/src/db/DBImpl.cpp b/cpp/src/db/DBImpl.cpp index ad35e5d4f0..520830d24f 100644 --- a/cpp/src/db/DBImpl.cpp +++ b/cpp/src/db/DBImpl.cpp @@ -8,6 +8,7 @@ #include "Env.h" #include "Log.h" #include "EngineFactory.h" +#include "Factories.h" #include "metrics/Metrics.h" #include "scheduler/SearchScheduler.h" #include "utils/TimeRecorder.h" @@ -130,8 +131,9 @@ DBImpl::DBImpl(const Options& options) bg_compaction_scheduled_(false), shutting_down_(false), bg_build_index_started_(false), - pMeta_(new meta::DBMetaImpl(options_.meta)), +// pMeta_(new meta::DBMetaImpl(options_.meta)), pMemMgr_(new MemManager(pMeta_, options_)) { + pMeta_ = DBMetaImplFactory::Build(options.meta); StartTimerTasks(options_.memory_sync_interval); } diff --git a/cpp/src/db/Factories.cpp b/cpp/src/db/Factories.cpp index 99a2918b85..26c8894057 100644 --- a/cpp/src/db/Factories.cpp +++ b/cpp/src/db/Factories.cpp @@ -13,6 +13,8 @@ #include #include #include +#include +#include "Exception.h" namespace zilliz { namespace milvus { @@ -43,6 +45,52 @@ std::shared_ptr DBMetaImplFactory::Build() { return std::shared_ptr(new meta::DBMetaImpl(options)); } +std::shared_ptr DBMetaImplFactory::Build(const DBMetaOptions& metaOptions) { + + std::string uri = metaOptions.backend_uri; + if (uri.empty()) { + //Default to sqlite if uri is empty +// return std::make_shared(new meta::DBMetaImpl(metaOptions)); + return std::shared_ptr(new meta::DBMetaImpl(metaOptions)); + } + + std::string dialectRegex = "(.*)"; + std::string usernameRegex = "(.*)"; + std::string passwordRegex = "(.*)"; + std::string hostRegex = "(.*)"; + std::string portRegex = "(.*)"; + std::string dbNameRegex = "(.*)"; + std::string uriRegexStr = dialectRegex + "\\:\\/\\/" + + usernameRegex + "\\:" + + passwordRegex + "\\@" + + hostRegex + "\\:" + + portRegex + "\\/" + + dbNameRegex; + std::regex uriRegex(uriRegexStr); + std::smatch pieces_match; + + if (std::regex_match(uri, pieces_match, uriRegex)) { + std::string dialect = pieces_match[1].str(); + std::transform(dialect.begin(), dialect.end(), dialect.begin(), ::tolower); + if (dialect.find("mysql") != std::string::npos) { +// return std::make_shared(new meta::MySQLMetaImpl(metaOptions)); + return std::shared_ptr(new meta::MySQLMetaImpl(metaOptions)); + } + else if (dialect.find("sqlite") != std::string::npos) { +// return std::make_shared(new meta::DBMetaImpl(metaOptions)); + return std::shared_ptr(new meta::DBMetaImpl(metaOptions)); + } + else { + LOG(ERROR) << "Invalid dialect in URI: dialect = " << dialect; + throw InvalidArgumentException("URI dialect is not mysql / sqlite"); + } + } + else { + LOG(ERROR) << "Wrong URI format: URI = " << uri; + throw InvalidArgumentException("Wrong URI format"); + } +} + std::shared_ptr DBFactory::Build() { auto options = OptionsFactory::Build(); auto db = DBFactory::Build(options); diff --git a/cpp/src/db/Factories.h b/cpp/src/db/Factories.h index 46d3e1bbc0..31afd2b5ba 100644 --- a/cpp/src/db/Factories.h +++ b/cpp/src/db/Factories.h @@ -7,6 +7,7 @@ #include "DB.h" #include "DBMetaImpl.h" +#include "MySQLMetaImpl.h" #include "Options.h" #include "ExecutionEngine.h" @@ -27,6 +28,7 @@ struct OptionsFactory { struct DBMetaImplFactory { static std::shared_ptr Build(); + static std::shared_ptr Build(const DBMetaOptions& metaOptions); }; struct DBFactory { diff --git a/cpp/unittest/db/MySQLMetaImpl_test.cpp b/cpp/unittest/db/MySQLMetaImpl_test.cpp index 2889a07a4e..c7fc611d7b 100644 --- a/cpp/unittest/db/MySQLMetaImpl_test.cpp +++ b/cpp/unittest/db/MySQLMetaImpl_test.cpp @@ -32,11 +32,11 @@ using namespace zilliz::milvus::engine; //} TEST_F(MySQLTest, core) { - DBMetaOptions options; - //dialect+driver://username:password@host:port/database - options.backend_uri = "mysql://root:1234@:/test"; - options.path = "/tmp/vecwise_test"; - meta::MySQLMetaImpl impl(options); +// DBMetaOptions options; +// //dialect+driver://username:password@host:port/database +// options.backend_uri = "mysql://root:1234@:/test"; +// options.path = "/tmp/vecwise_test"; + meta::MySQLMetaImpl impl(getDBMetaOptions()); // auto status = impl.Initialize(); // ASSERT_TRUE(status.ok()); @@ -185,10 +185,10 @@ TEST_F(MySQLTest, core) { TEST_F(MySQLTest, GROUP_TEST) { - DBMetaOptions options; - options.backend_uri = "mysql://root:1234@:/test"; - options.path = "/tmp/vecwise_test"; - meta::MySQLMetaImpl impl(options); +// DBMetaOptions options; +// options.backend_uri = "mysql://root:1234@:/test"; +// options.path = "/tmp/vecwise_test"; + meta::MySQLMetaImpl impl(getDBMetaOptions()); auto table_id = "meta_test_group"; @@ -219,10 +219,10 @@ TEST_F(MySQLTest, GROUP_TEST) { TEST_F(MySQLTest, table_file_TEST) { - DBMetaOptions options; - options.backend_uri = "mysql://root:1234@:/test"; - options.path = "/tmp/vecwise_test"; - meta::MySQLMetaImpl impl(options); +// DBMetaOptions options; +// options.backend_uri = "mysql://root:1234@:/test"; +// options.path = "/tmp/vecwise_test"; + meta::MySQLMetaImpl impl(getDBMetaOptions()); auto table_id = "meta_test_group"; @@ -277,13 +277,13 @@ TEST_F(MySQLTest, table_file_TEST) { TEST_F(MySQLTest, ARCHIVE_TEST_DAYS) { srand(time(0)); - DBMetaOptions options; - options.path = "/tmp/vecwise_test"; + DBMetaOptions options = getDBMetaOptions(); +// options.path = "/tmp/vecwise_test"; int days_num = rand() % 100; std::stringstream ss; ss << "days:" << days_num; options.archive_conf = ArchiveConf("delete", ss.str()); - options.backend_uri = "mysql://root:1234@:/test"; +// options.backend_uri = "mysql://root:1234@:/test"; meta::MySQLMetaImpl impl(options); @@ -329,10 +329,10 @@ TEST_F(MySQLTest, ARCHIVE_TEST_DAYS) { } TEST_F(MySQLTest, ARCHIVE_TEST_DISK) { - DBMetaOptions options; - options.path = "/tmp/vecwise_test"; + DBMetaOptions options = getDBMetaOptions(); +// options.path = "/tmp/vecwise_test"; options.archive_conf = ArchiveConf("delete", "disk:11"); - options.backend_uri = "mysql://root:1234@:/test"; +// options.backend_uri = "mysql://root:1234@:/test"; meta::MySQLMetaImpl impl(options); @@ -376,10 +376,10 @@ TEST_F(MySQLTest, ARCHIVE_TEST_DISK) { TEST_F(MySQLTest, TABLE_FILES_TEST) { - DBMetaOptions options; - options.backend_uri = "mysql://root:1234@:/test"; - options.path = "/tmp/vecwise_test"; - meta::MySQLMetaImpl impl(options); +// DBMetaOptions options; +// options.backend_uri = "mysql://root:1234@:/test"; +// options.path = "/tmp/vecwise_test"; + meta::MySQLMetaImpl impl(getDBMetaOptions()); auto table_id = "meta_test_group"; diff --git a/cpp/unittest/db/utils.cpp b/cpp/unittest/db/utils.cpp index 0de876b39c..e4eab8ed56 100644 --- a/cpp/unittest/db/utils.cpp +++ b/cpp/unittest/db/utils.cpp @@ -11,6 +11,7 @@ #include "utils.h" #include "db/Factories.h" +#include "db/Options.h" using namespace zilliz::milvus; @@ -61,3 +62,11 @@ void MetaTest::SetUp() { void MetaTest::TearDown() { impl_->DropAll(); } + +zilliz::milvus::engine::DBMetaOptions MySQLTest::getDBMetaOptions() { + engine::DBMetaOptions options; + options.backend_uri = "mysql://root:1234@:/test"; + options.path = "/tmp/milvus_test"; + return options; + +} diff --git a/cpp/unittest/db/utils.h b/cpp/unittest/db/utils.h index d2a96e15f9..5e5c162163 100644 --- a/cpp/unittest/db/utils.h +++ b/cpp/unittest/db/utils.h @@ -8,10 +8,11 @@ #include #include -#include +//#include #include "db/DB.h" #include "db/DBMetaImpl.h" +#include "db/MySQLMetaImpl.h" #define TIMING @@ -60,4 +61,9 @@ protected: class MySQLTest : public ::testing::Test { protected: // std::shared_ptr impl_; + zilliz::milvus::engine::DBMetaOptions getDBMetaOptions(); +}; + +class MySQLDBTest : public ::testing::Test { + }; From d1b6c86be44399361c85cce31649803e43965102 Mon Sep 17 00:00:00 2001 From: zhiru Date: Fri, 21 Jun 2019 18:30:45 +0800 Subject: [PATCH 09/43] passed db_tests (probably) Former-commit-id: c7089827f89b92642d71504c98611d95094ca54d --- cpp/mysqlNotes | 3 +- cpp/src/db/DBImpl.cpp | 6 +- cpp/src/db/Factories.cpp | 10 +- cpp/src/db/MySQLMetaImpl.cpp | 114 +++++++++++++++++- cpp/src/db/MySQLMetaImpl.h | 9 +- cpp/unittest/db/db_tests.cpp | 213 +++++++++++++++++++++++++++++++++ cpp/unittest/db/meta_tests.cpp | 2 + cpp/unittest/db/utils.cpp | 14 ++- cpp/unittest/db/utils.h | 24 +++- 9 files changed, 381 insertions(+), 14 deletions(-) diff --git a/cpp/mysqlNotes b/cpp/mysqlNotes index f40db40215..8aa151efff 100644 --- a/cpp/mysqlNotes +++ b/cpp/mysqlNotes @@ -1,7 +1,8 @@ sudo apt-get install mysql-server sudo apt-get install libmysqlclient-dev +sudo ln -s libmysqlclient.so libmysqlclient_r.so Install MySQL++ -./configure +./configure --enable-thread-check LDFLAGS='-pthread' make sudo make install diff --git a/cpp/src/db/DBImpl.cpp b/cpp/src/db/DBImpl.cpp index 520830d24f..426a204355 100644 --- a/cpp/src/db/DBImpl.cpp +++ b/cpp/src/db/DBImpl.cpp @@ -130,10 +130,10 @@ DBImpl::DBImpl(const Options& options) options_(options), bg_compaction_scheduled_(false), shutting_down_(false), - bg_build_index_started_(false), -// pMeta_(new meta::DBMetaImpl(options_.meta)), - pMemMgr_(new MemManager(pMeta_, options_)) { + bg_build_index_started_(false) + { pMeta_ = DBMetaImplFactory::Build(options.meta); + pMemMgr_ = (MemManagerPtr)(new MemManager(pMeta_, options_)); StartTimerTasks(options_.memory_sync_interval); } diff --git a/cpp/src/db/Factories.cpp b/cpp/src/db/Factories.cpp index 26c8894057..58265cf82f 100644 --- a/cpp/src/db/Factories.cpp +++ b/cpp/src/db/Factories.cpp @@ -3,10 +3,10 @@ // Unauthorized copying of this file, via any medium is strictly prohibited. // Proprietary and confidential. //////////////////////////////////////////////////////////////////////////////// +#include #include "Factories.h" #include "DBImpl.h" -#include #include #include #include @@ -28,8 +28,16 @@ DBMetaOptions DBMetaOptionsFactory::Build(const std::string& path) { ss << "/tmp/" << rand(); p = ss.str(); } + +// std::string uri; +// const char* uri_p = getenv("MILVUS_DB_META_URI"); +// if (uri_p) { +// uri = uri_p; +// } + DBMetaOptions meta; meta.path = p; +// meta.backend_uri = uri; return meta; } diff --git a/cpp/src/db/MySQLMetaImpl.cpp b/cpp/src/db/MySQLMetaImpl.cpp index 210930049d..20734ba86f 100644 --- a/cpp/src/db/MySQLMetaImpl.cpp +++ b/cpp/src/db/MySQLMetaImpl.cpp @@ -19,6 +19,7 @@ #include #include #include +#include #include "mysql++/mysql++.h" @@ -29,7 +30,14 @@ namespace meta { using namespace mysqlpp; - static std::unique_ptr connectionPtr(new Connection()); + std::unique_ptr connectionPtr(new Connection()); + std::recursive_mutex mysql_mutex; +// +// std::unique_ptr& MySQLMetaImpl::getConnectionPtr() { +//// static std::recursive_mutex connectionMutex_; +// std::lock_guard lock(connectionMutex_); +// return connectionPtr; +// } namespace { @@ -83,6 +91,8 @@ namespace meta { Status MySQLMetaImpl::Initialize() { + std::lock_guard lock(mysql_mutex); + std::string path = options_.path; if (!boost::filesystem::is_directory(path)) { auto ret = boost::filesystem::create_directory(path); @@ -125,6 +135,9 @@ namespace meta { const char* dbName = pieces_match[6].str().c_str(); //std::cout << dbName << " " << serverAddress << " " << username << " " << password << " " << port << std::endl; // connectionPtr->set_option(new MultiStatementsOption(true)); +// connectionPtr->set_option(new mysqlpp::ReconnectOption(true)); + connectionPtr->set_option(new mysqlpp::ReconnectOption(true)); + std::cout << "MySQL++ thread aware:" << std::to_string(connectionPtr->thread_aware()) << std::endl; try { if (!connectionPtr->connect(dbName, serverAddress, username, password, port)) { @@ -134,6 +147,11 @@ namespace meta { CleanUp(); Query InitializeQuery = connectionPtr->query(); +// InitializeQuery << "SET max_allowed_packet=67108864;"; +// if (!InitializeQuery.exec()) { +// return Status::DBTransactionError("Initialization Error", InitializeQuery.error()); +// } + // InitializeQuery << "DROP TABLE IF EXISTS meta, metaFile;"; InitializeQuery << "CREATE TABLE IF NOT EXISTS meta (" << "id BIGINT PRIMARY KEY AUTO_INCREMENT, " << @@ -161,6 +179,10 @@ namespace meta { return Status::DBTransactionError("Initialization Error", InitializeQuery.error()); } +// //Consume all results to avoid "Commands out of sync" error +// while (InitializeQuery.more_results()) { +// InitializeQuery.store_next(); +// } return Status::OK(); // if (InitializeQuery.exec()) { @@ -190,6 +212,9 @@ namespace meta { // PXU TODO: Temp solution. Will fix later Status MySQLMetaImpl::DropPartitionsByDates(const std::string &table_id, const DatesT &dates) { + + std::lock_guard lock(mysql_mutex); + if (dates.size() == 0) { return Status::OK(); } @@ -240,6 +265,9 @@ namespace meta { } Status MySQLMetaImpl::CreateTable(TableSchema &table_schema) { + + std::lock_guard lock(mysql_mutex); + server::Metrics::GetInstance().MetaAccessTotalIncrement(); if (table_schema.table_id_.empty()) { NextTableId(table_schema.table_id_); @@ -265,6 +293,10 @@ namespace meta { if (SimpleResult res = createTableQuery.execute()) { table_schema.id_ = res.insert_id(); //Might need to use SELECT LAST_INSERT_ID()? // std::cout << table_schema.id_ << std::endl; + //Consume all results to avoid "Commands out of sync" error + while (createTableQuery.more_results()) { + createTableQuery.store_next(); + } } else { return Status::DBTransactionError("Add Table Error", createTableQuery.error()); @@ -296,6 +328,9 @@ namespace meta { } Status MySQLMetaImpl::DeleteTable(const std::string& table_id) { + + std::lock_guard lock(mysql_mutex); + try { //drop the table from meta Query deleteTableQuery = connectionPtr->query(); @@ -316,6 +351,9 @@ namespace meta { } Status MySQLMetaImpl::DescribeTable(TableSchema &table_schema) { + + std::lock_guard lock(mysql_mutex); + try { server::Metrics::GetInstance().MetaAccessTotalIncrement(); auto start_time = METRICS_NOW_TIME; @@ -330,6 +368,9 @@ namespace meta { auto total_time = METRICS_MICROSECONDS(start_time, end_time); server::Metrics::GetInstance().MetaAccessDurationSecondsHistogramObserve(total_time); +// if (!res) { +// return Status::DBTransactionError("QUERY ERROR WHEN DESCRIBING TABLE", describeTableQuery.error()); +// } assert(res && res.num_rows() <= 1); if (res.num_rows() == 1) { const Row& resRow = res[0]; @@ -379,6 +420,9 @@ namespace meta { } Status MySQLMetaImpl::HasTable(const std::string &table_id, bool &has_or_not) { + + std::lock_guard lock(mysql_mutex); + try { server::Metrics::GetInstance().MetaAccessTotalIncrement(); auto start_time = METRICS_NOW_TIME; @@ -409,6 +453,9 @@ namespace meta { } Status MySQLMetaImpl::AllTables(std::vector& table_schema_array) { + + std::lock_guard lock(mysql_mutex); + try { server::Metrics::GetInstance().MetaAccessTotalIncrement(); auto start_time = METRICS_NOW_TIME; @@ -453,6 +500,9 @@ namespace meta { } Status MySQLMetaImpl::CreateTableFile(TableFileSchema &file_schema) { + + std::lock_guard lock(mysql_mutex); + if (file_schema.date_ == EmptyDate) { file_schema.date_ = Meta::GetDate(); } @@ -495,6 +545,11 @@ namespace meta { if (SimpleResult res = createTableFileQuery.execute()) { file_schema.id_ = res.insert_id(); //Might need to use SELECT LAST_INSERT_ID()? + + //Consume all results to avoid "Commands out of sync" error + while (createTableFileQuery.more_results()) { + createTableFileQuery.store_next(); + } } else { return Status::DBTransactionError("Add file Error", createTableFileQuery.error()); @@ -526,6 +581,9 @@ namespace meta { } Status MySQLMetaImpl::FilesToIndex(TableFilesSchema &files) { + + std::lock_guard lock(mysql_mutex); + files.clear(); try { @@ -595,6 +653,9 @@ namespace meta { Status MySQLMetaImpl::FilesToSearch(const std::string &table_id, const DatesT &partition, DatePartionedTableFilesSchema &files) { + + std::lock_guard lock(mysql_mutex); + files.clear(); try { @@ -695,6 +756,9 @@ namespace meta { Status MySQLMetaImpl::FilesToMerge(const std::string &table_id, DatePartionedTableFilesSchema &files) { + + std::lock_guard lock(mysql_mutex); + files.clear(); try { @@ -767,11 +831,15 @@ namespace meta { const DatesT& partition, DatePartionedTableFilesSchema& files) { +// std::lock_guard lock(mysql_mutex); + return Status::OK(); } Status MySQLMetaImpl::GetTableFile(TableFileSchema &file_schema) { + std::lock_guard lock(mysql_mutex); + try { Query getTableFileQuery = connectionPtr->query(); @@ -819,6 +887,9 @@ namespace meta { // PXU TODO: Support Swap Status MySQLMetaImpl::Archive() { + + std::lock_guard lock(mysql_mutex); + auto &criterias = options_.archive_conf.GetCriterias(); if (criterias.empty()) { return Status::OK(); @@ -853,7 +924,7 @@ namespace meta { uint64_t sum = 0; Size(sum); - auto to_delete = (sum - limit * G); + long long to_delete = (sum - limit * G); DiscardFiles(to_delete); } } @@ -862,6 +933,9 @@ namespace meta { } Status MySQLMetaImpl::Size(uint64_t &result) { + + std::lock_guard lock(mysql_mutex); + result = 0; try { @@ -872,7 +946,18 @@ namespace meta { StoreQueryResult res = getSizeQuery.store(); assert(res && res.num_rows() == 1); - result = res[0]["sum"]; +// if (!res) { +//// std::cout << "result is NULL" << std::endl; +// return Status::DBTransactionError("QUERY ERROR WHEN RETRIEVING SIZE", getSizeQuery.error()); +// } + if (res.empty()) { + result = 0; +// std::cout << "result = 0" << std::endl; + } + else { + result = res[0]["sum"]; +// std::cout << "result = " << std::to_string(result) << std::endl; + } } catch (const BadQuery& er) { // Handle any query errors @@ -885,7 +970,10 @@ namespace meta { return Status::OK(); } - Status MySQLMetaImpl::DiscardFiles(long to_discard_size) { + Status MySQLMetaImpl::DiscardFiles(long long to_discard_size) { + + std::lock_guard lock(mysql_mutex); + LOG(DEBUG) << "About to discard size=" << to_discard_size; if (to_discard_size <= 0) { // std::cout << "in" << std::endl; @@ -946,6 +1034,9 @@ namespace meta { //ZR: this function assumes all fields in file_schema have value Status MySQLMetaImpl::UpdateTableFile(TableFileSchema &file_schema) { + + std::lock_guard lock(mysql_mutex); + file_schema.updated_time_ = utils::GetMicroSecTimeStamp(); try { server::Metrics::GetInstance().MetaAccessTotalIncrement(); @@ -997,6 +1088,9 @@ namespace meta { } Status MySQLMetaImpl::UpdateTableFiles(TableFilesSchema &files) { + + std::lock_guard lock(mysql_mutex); + try { server::Metrics::GetInstance().MetaAccessTotalIncrement(); auto start_time = METRICS_NOW_TIME; @@ -1043,6 +1137,9 @@ namespace meta { } Status MySQLMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) { + + std::lock_guard lock(mysql_mutex); + auto now = utils::GetMicroSecTimeStamp(); try { @@ -1109,6 +1206,9 @@ namespace meta { } Status MySQLMetaImpl::CleanUp() { + + std::lock_guard lock(mysql_mutex); + try { // auto selected = ConnectorPtr->select(columns(&TableFileSchema::id_, // &TableFileSchema::table_id_, @@ -1186,6 +1286,8 @@ namespace meta { Status MySQLMetaImpl::Count(const std::string &table_id, uint64_t &result) { + std::lock_guard lock(mysql_mutex); + try { server::Metrics::GetInstance().MetaAccessTotalIncrement(); @@ -1233,6 +1335,9 @@ namespace meta { } Status MySQLMetaImpl::DropAll() { + + std::lock_guard lock(mysql_mutex); + if (boost::filesystem::is_directory(options_.path)) { boost::filesystem::remove_all(options_.path); } @@ -1255,6 +1360,7 @@ namespace meta { } MySQLMetaImpl::~MySQLMetaImpl() { + std::lock_guard lock(mysql_mutex); CleanUp(); } diff --git a/cpp/src/db/MySQLMetaImpl.h b/cpp/src/db/MySQLMetaImpl.h index e099fbb262..8dd2f4cabf 100644 --- a/cpp/src/db/MySQLMetaImpl.h +++ b/cpp/src/db/MySQLMetaImpl.h @@ -8,12 +8,16 @@ #include "Meta.h" #include "Options.h" +#include "mysql++/mysql++.h" +#include + namespace zilliz { namespace milvus { namespace engine { namespace meta { // auto StoragePrototype(const std::string& path); + using namespace mysqlpp; class MySQLMetaImpl : public Meta { public: @@ -65,13 +69,16 @@ namespace meta { private: Status NextFileId(std::string& file_id); Status NextTableId(std::string& table_id); - Status DiscardFiles(long to_discard_size); + Status DiscardFiles(long long to_discard_size); std::string GetTablePath(const std::string& table_id); std::string GetTableDatePartitionPath(const std::string& table_id, DateT& date); void GetTableFilePath(TableFileSchema& group_file); Status Initialize(); + std::unique_ptr& getConnectionPtr(); const DBMetaOptions options_; + +// std::mutex connectionMutex_; }; // DBMetaImpl } // namespace meta diff --git a/cpp/unittest/db/db_tests.cpp b/cpp/unittest/db/db_tests.cpp index 00b6c7a9c1..ecbadb328a 100644 --- a/cpp/unittest/db/db_tests.cpp +++ b/cpp/unittest/db/db_tests.cpp @@ -11,6 +11,8 @@ #include "db/DB.h" #include "db/DBImpl.h" #include "db/MetaConsts.h" +#include "db/Factories.h" +#include "db/Options.h" using namespace zilliz::milvus; @@ -257,3 +259,214 @@ TEST_F(DBTest, SEARCH_TEST) { // TODO(linxj): add groundTruth assert }; +TEST_F(MySQLDBTest, ARHIVE_DISK_CHECK) { + auto options = GetOptions(); + options.meta.archive_conf = engine::ArchiveConf("delete", "disk:1"); + auto db_ = engine::DBFactory::Build(options); + + static const std::string group_name = "test_group"; + static const int group_dim = 256; + uint64_t size; + + engine::meta::TableSchema group_info; + group_info.dimension_ = group_dim; + group_info.table_id_ = group_name; + group_info.engine_type_ = (int)engine::EngineType::FAISS_IVFFLAT; + engine::Status stat = db_->CreateTable(group_info); + + engine::meta::TableSchema group_info_get; + group_info_get.table_id_ = group_name; + stat = db_->DescribeTable(group_info_get); + ASSERT_STATS(stat); + ASSERT_EQ(group_info_get.dimension_, group_dim); + + engine::IDNumbers vector_ids; + engine::IDNumbers target_ids; + + db_->Size(size); + int d = 256; + int nb = 20; + float *xb = new float[d * nb]; + for(int i = 0; i < nb; i++) { + for(int j = 0; j < d; j++) xb[d * i + j] = drand48(); + xb[d * i] += i / 2000.; + } + + int loop = 100000; + + for (auto i=0; iInsertVectors(group_name, nb, xb, vector_ids); + std::this_thread::sleep_for(std::chrono::microseconds(1)); + } + + std::this_thread::sleep_for(std::chrono::seconds(10)); + + db_->Size(size); + LOG(DEBUG) << "size=" << size; +// LOG(DEBUG) << "1 * engine::meta::G=" << 1 * engine::meta::G; + ASSERT_TRUE(size < 1 * engine::meta::G); + + delete [] xb; + + db_->DropAll(); +} + +TEST_F(MySQLDBTest, DB_TEST) { + + auto options = GetOptions(); + auto db_ = engine::DBFactory::Build(options); + + static const std::string group_name = "test_group"; + static const int group_dim = 256; + + engine::meta::TableSchema group_info; + group_info.dimension_ = group_dim; + group_info.table_id_ = group_name; + group_info.engine_type_ = (int)engine::EngineType::FAISS_IVFFLAT; + engine::Status stat = db_->CreateTable(group_info); + + engine::meta::TableSchema group_info_get; + group_info_get.table_id_ = group_name; + stat = db_->DescribeTable(group_info_get); + ASSERT_STATS(stat); + ASSERT_EQ(group_info_get.dimension_, group_dim); + + engine::IDNumbers vector_ids; + engine::IDNumbers target_ids; + + int d = 256; + int nb = 50; + float *xb = new float[d * nb]; + for(int i = 0; i < nb; i++) { + for(int j = 0; j < d; j++) xb[d * i + j] = drand48(); + xb[d * i] += i / 2000.; + } + + int qb = 5; + float *qxb = new float[d * qb]; + for(int i = 0; i < qb; i++) { + for(int j = 0; j < d; j++) qxb[d * i + j] = drand48(); + qxb[d * i] += i / 2000.; + } + + std::thread search([&]() { + engine::QueryResults results; + int k = 10; + std::this_thread::sleep_for(std::chrono::seconds(2)); + + INIT_TIMER; + std::stringstream ss; + uint64_t count = 0; + uint64_t prev_count = 0; + + for (auto j=0; j<10; ++j) { + ss.str(""); + db_->Size(count); + prev_count = count; + + START_TIMER; + stat = db_->Query(group_name, k, qb, qxb, results); + ss << "Search " << j << " With Size " << count/engine::meta::M << " M"; + STOP_TIMER(ss.str()); + + ASSERT_STATS(stat); +// std::cout << results.size() << std::endl; + for (auto k=0; k= prev_count); + std::this_thread::sleep_for(std::chrono::seconds(1)); + } + }); + + int loop = 100000; + + for (auto i=0; iInsertVectors(group_name, qb, qxb, target_ids); + ASSERT_EQ(target_ids.size(), qb); + } else { + db_->InsertVectors(group_name, nb, xb, vector_ids); + } + std::this_thread::sleep_for(std::chrono::microseconds(1)); + } + + search.join(); + + delete [] xb; + delete [] qxb; + + db_->DropAll(); +}; + +TEST_F(MySQLDBTest, SEARCH_TEST) { + + auto options = GetOptions(); + auto db_ = engine::DBFactory::Build(options); + + static const std::string group_name = "test_group"; + static const int group_dim = 256; + + engine::meta::TableSchema group_info; + group_info.dimension_ = group_dim; + group_info.table_id_ = group_name; + group_info.engine_type_ = (int)engine::EngineType::FAISS_IVFFLAT; + engine::Status stat = db_->CreateTable(group_info); + + engine::meta::TableSchema group_info_get; + group_info_get.table_id_ = group_name; + stat = db_->DescribeTable(group_info_get); + ASSERT_STATS(stat); + ASSERT_EQ(group_info_get.dimension_, group_dim); + + // prepare raw data + size_t nb = 250000; + size_t nq = 10; + size_t k = 5; + std::vector xb(nb*group_dim); + std::vector xq(nq*group_dim); + std::vector ids(nb); + + std::random_device rd; + std::mt19937 gen(rd()); + std::uniform_real_distribution<> dis_xt(-1.0, 1.0); + for (size_t i = 0; i < nb*group_dim; i++) { + xb[i] = dis_xt(gen); + if (i < nb){ + ids[i] = i; + } + } + for (size_t i = 0; i < nq*group_dim; i++) { + xq[i] = dis_xt(gen); + } + + // result data + //std::vector nns_gt(k*nq); + std::vector nns(k*nq); // nns = nearst neg search + //std::vector dis_gt(k*nq); + std::vector dis(k*nq); + + // insert data + const int batch_size = 100; + for (int j = 0; j < nb / batch_size; ++j) { + stat = db_->InsertVectors(group_name, batch_size, xb.data()+batch_size*j*group_dim, ids); + if (j == 200){ sleep(1);} + ASSERT_STATS(stat); + } + + sleep(2); // wait until build index finish + + engine::QueryResults results; + stat = db_->Query(group_name, k, nq, xq.data(), results); + ASSERT_STATS(stat); + + db_->DropAll(); + + // TODO(linxj): add groundTruth assert +}; \ No newline at end of file diff --git a/cpp/unittest/db/meta_tests.cpp b/cpp/unittest/db/meta_tests.cpp index 147ab38595..76f955087c 100644 --- a/cpp/unittest/db/meta_tests.cpp +++ b/cpp/unittest/db/meta_tests.cpp @@ -242,4 +242,6 @@ TEST_F(MetaTest, TABLE_FILES_TEST) { ASSERT_TRUE(status.ok()); ASSERT_EQ(dated_files[table_file.date_].size(), to_index_files_cnt+raw_files_cnt+index_files_cnt); + + } diff --git a/cpp/unittest/db/utils.cpp b/cpp/unittest/db/utils.cpp index e4eab8ed56..36a457e173 100644 --- a/cpp/unittest/db/utils.cpp +++ b/cpp/unittest/db/utils.cpp @@ -22,6 +22,7 @@ void ASSERT_STATS(engine::Status& stat) { } } + void DBTest::InitLog() { el::Configurations defaultConf; defaultConf.setToDefault(); @@ -64,9 +65,18 @@ void MetaTest::TearDown() { } zilliz::milvus::engine::DBMetaOptions MySQLTest::getDBMetaOptions() { - engine::DBMetaOptions options; - options.backend_uri = "mysql://root:1234@:/test"; +// std::string path = "/tmp/milvus_test"; +// engine::DBMetaOptions options = engine::DBMetaOptionsFactory::Build(path); + zilliz::milvus::engine::DBMetaOptions options; options.path = "/tmp/milvus_test"; + options.backend_uri = "mysql://root:1234@:/test"; return options; } + +zilliz::milvus::engine::Options MySQLDBTest::GetOptions() { + auto options = engine::OptionsFactory::Build(); + options.meta.path = "/tmp/milvus_test"; + options.meta.backend_uri = "mysql://root:1234@:/test"; + return options; +} diff --git a/cpp/unittest/db/utils.h b/cpp/unittest/db/utils.h index 5e5c162163..361c24b4be 100644 --- a/cpp/unittest/db/utils.h +++ b/cpp/unittest/db/utils.h @@ -30,9 +30,28 @@ #define STOP_TIMER(name) #endif - void ASSERT_STATS(zilliz::milvus::engine::Status& stat); +//class TestEnv : public ::testing::Environment { +//public: +// +// static std::string getURI() { +// if (const char* uri = std::getenv("MILVUS_DBMETA_URI")) { +// return uri; +// } +// else { +// return ""; +// } +// } +// +// void SetUp() override { +// getURI(); +// } +// +//}; +// +//::testing::Environment* const test_env = +// ::testing::AddGlobalTestEnvironment(new TestEnv); class DBTest : public ::testing::Test { protected: @@ -65,5 +84,6 @@ protected: }; class MySQLDBTest : public ::testing::Test { - +protected: + zilliz::milvus::engine::Options GetOptions(); }; From 1a29bb36e71070b88694ae1fa70823ad49b9fb9e Mon Sep 17 00:00:00 2001 From: zhiru Date: Sun, 23 Jun 2019 15:21:59 +0800 Subject: [PATCH 10/43] update Former-commit-id: e52522b110dadee29e9d1a0d13a677c0bb26e529 --- cpp/src/db/MySQLMetaImpl.cpp | 663 +++++++++++++------------ cpp/src/db/MySQLMetaImpl.h | 13 +- cpp/unittest/db/MySQLMetaImpl_test.cpp | 75 +-- cpp/unittest/db/db_tests.cpp | 20 +- 4 files changed, 421 insertions(+), 350 deletions(-) diff --git a/cpp/src/db/MySQLMetaImpl.cpp b/cpp/src/db/MySQLMetaImpl.cpp index 20734ba86f..e8c648c93a 100644 --- a/cpp/src/db/MySQLMetaImpl.cpp +++ b/cpp/src/db/MySQLMetaImpl.cpp @@ -30,7 +30,7 @@ namespace meta { using namespace mysqlpp; - std::unique_ptr connectionPtr(new Connection()); + static std::unique_ptr connectionPtr(new Connection()); std::recursive_mutex mysql_mutex; // // std::unique_ptr& MySQLMetaImpl::getConnectionPtr() { @@ -41,11 +41,29 @@ namespace meta { namespace { - void HandleException(std::exception &e) { - ENGINE_LOG_DEBUG << "Engine meta exception: " << e.what(); - throw e; + Status HandleException(const std::string& desc, std::exception &e) { + ENGINE_LOG_ERROR << desc << ": " << e.what(); + return Status::DBTransactionError(desc, e.what()); } + class MetricCollector { + public: + MetricCollector() { + server::Metrics::GetInstance().MetaAccessTotalIncrement(); + start_time_ = METRICS_NOW_TIME; + } + + ~MetricCollector() { + auto end_time = METRICS_NOW_TIME; + auto total_time = METRICS_MICROSECONDS(start_time_, end_time); + server::Metrics::GetInstance().MetaAccessDurationSecondsHistogramObserve(total_time); + } + + private: + using TIME_POINT = std::chrono::system_clock::time_point; + TIME_POINT start_time_; + }; + } std::string MySQLMetaImpl::GetTablePath(const std::string &table_id) { @@ -93,13 +111,12 @@ namespace meta { std::lock_guard lock(mysql_mutex); - std::string path = options_.path; - if (!boost::filesystem::is_directory(path)) { - auto ret = boost::filesystem::create_directory(path); + if (!boost::filesystem::is_directory(options_.path)) { + auto ret = boost::filesystem::create_directory(options_.path); if (!ret) { - ENGINE_LOG_ERROR << "Create directory " << path << " Error"; + ENGINE_LOG_ERROR << "Failed to create db directory " << options_.path; + return Status::DBTransactionError("Failed to create db directory", options_.path); } - assert(ret); } std::string uri = options_.backend_uri; @@ -156,6 +173,7 @@ namespace meta { InitializeQuery << "CREATE TABLE IF NOT EXISTS meta (" << "id BIGINT PRIMARY KEY AUTO_INCREMENT, " << "table_id VARCHAR(255) UNIQUE NOT NULL, " << + "state INT NOT NULL, " << "dimension SMALLINT NOT NULL, " << "created_on BIGINT NOT NULL, " << "files_cnt BIGINT DEFAULT 0 NOT NULL, " << @@ -226,16 +244,16 @@ namespace meta { return status; } - auto yesterday = GetDateWithDelta(-1); - - for (auto &date : dates) { - if (date >= yesterday) { - return Status::Error("Could not delete partitions within 2 days"); - } - } - try { + auto yesterday = GetDateWithDelta(-1); + + for (auto &date : dates) { + if (date >= yesterday) { + return Status::Error("Could not delete partitions within 2 days"); + } + } + Query dropPartitionsByDatesQuery = connectionPtr->query(); std::stringstream dateListSS; @@ -268,60 +286,80 @@ namespace meta { std::lock_guard lock(mysql_mutex); - server::Metrics::GetInstance().MetaAccessTotalIncrement(); - if (table_schema.table_id_.empty()) { - NextTableId(table_schema.table_id_); - } - table_schema.files_cnt_ = 0; - table_schema.id_ = -1; - table_schema.created_on_ = utils::GetMicroSecTimeStamp(); - auto start_time = METRICS_NOW_TIME; - { - try { - Query createTableQuery = connectionPtr->query(); - std::string id = "NULL"; //auto-increment - std::string table_id = table_schema.table_id_; - std::string dimension = std::to_string(table_schema.dimension_); - std::string created_on = std::to_string(table_schema.created_on_); - std::string files_cnt = "0"; - std::string engine_type = std::to_string(table_schema.engine_type_); - std::string store_raw_data = table_schema.store_raw_data_ ? "true" : "false"; - createTableQuery << "INSERT INTO meta VALUES" << - "(" << id << ", " << quote << table_id << ", " << dimension << ", " << - created_on << ", " << files_cnt << ", " << engine_type << ", " << store_raw_data - << ");"; - if (SimpleResult res = createTableQuery.execute()) { - table_schema.id_ = res.insert_id(); //Might need to use SELECT LAST_INSERT_ID()? +// server::Metrics::GetInstance().MetaAccessTotalIncrement(); + try { + + MetricCollector metric; + + Query createTableQuery = connectionPtr->query(); + + if (table_schema.table_id_.empty()) { + NextTableId(table_schema.table_id_); + } + else { + createTableQuery << "SELECT state FROM meta " << + "WHERE table_id = " << quote << table_schema.table_id_ << ";"; + StoreQueryResult res = createTableQuery.store(); + assert(res && res.num_rows() <= 1); + if (res.num_rows() == 1) { + int state = res[0]["state"]; + std::string msg = (TableSchema::TO_DELETE == state) ? + "Table already exists" : "Table already exists and it is in delete state, please wait a second"; + return Status::Error(msg); + } + } + + table_schema.files_cnt_ = 0; + table_schema.id_ = -1; + table_schema.created_on_ = utils::GetMicroSecTimeStamp(); + +// auto start_time = METRICS_NOW_TIME; + + std::string id = "NULL"; //auto-increment + std::string table_id = table_schema.table_id_; + std::string state = std::to_string(table_schema.state_); + std::string dimension = std::to_string(table_schema.dimension_); + std::string created_on = std::to_string(table_schema.created_on_); + std::string files_cnt = "0"; + std::string engine_type = std::to_string(table_schema.engine_type_); + std::string store_raw_data = table_schema.store_raw_data_ ? "true" : "false"; + + createTableQuery << "INSERT INTO meta VALUES" << + "(" << id << ", " << quote << table_id << ", " << state << ", " << dimension << ", " << + created_on << ", " << files_cnt << ", " << engine_type << ", " << store_raw_data + << ");"; + + if (SimpleResult res = createTableQuery.execute()) { + table_schema.id_ = res.insert_id(); //Might need to use SELECT LAST_INSERT_ID()? // std::cout << table_schema.id_ << std::endl; - //Consume all results to avoid "Commands out of sync" error - while (createTableQuery.more_results()) { - createTableQuery.store_next(); - } - } - else { - return Status::DBTransactionError("Add Table Error", createTableQuery.error()); - } - - } catch (const BadQuery& er) { - // Handle any query errors - return Status::DBTransactionError("QUERY ERROR WHEN ADDING TABLE", er.what()); - } catch (const Exception& er) { - // Catch-all for any other MySQL++ exceptions - return Status::DBTransactionError("GENERAL ERROR WHEN ADDING TABLE", er.what()); + //Consume all results to avoid "Commands out of sync" error +// while (createTableQuery.more_results()) { +// createTableQuery.store_next(); +// } } - } - auto end_time = METRICS_NOW_TIME; - auto total_time = METRICS_MICROSECONDS(start_time, end_time); - server::Metrics::GetInstance().MetaAccessDurationSecondsHistogramObserve(total_time); - - auto table_path = GetTablePath(table_schema.table_id_); - table_schema.location_ = table_path; - if (!boost::filesystem::is_directory(table_path)) { - auto ret = boost::filesystem::create_directories(table_path); - if (!ret) { - ENGINE_LOG_ERROR << "Create directory " << table_path << " Error"; + else { + return Status::DBTransactionError("Add Table Error", createTableQuery.error()); } - assert(ret); + +// auto end_time = METRICS_NOW_TIME; +// auto total_time = METRICS_MICROSECONDS(start_time, end_time); +// server::Metrics::GetInstance().MetaAccessDurationSecondsHistogramObserve(total_time); + + auto table_path = GetTablePath(table_schema.table_id_); + table_schema.location_ = table_path; + if (!boost::filesystem::is_directory(table_path)) { + auto ret = boost::filesystem::create_directories(table_path); + if (!ret) { + ENGINE_LOG_ERROR << "Create directory " << table_path << " Error"; + return Status::Error("Failed to create table path"); + } + } + } catch (const BadQuery& er) { + // Handle any query errors + return Status::DBTransactionError("QUERY ERROR WHEN ADDING TABLE", er.what()); + } catch (const Exception& er) { + // Catch-all for any other MySQL++ exceptions + return Status::DBTransactionError("GENERAL ERROR WHEN ADDING TABLE", er.what()); } return Status::OK(); @@ -332,14 +370,18 @@ namespace meta { std::lock_guard lock(mysql_mutex); try { - //drop the table from meta + + MetricCollector metric; + + //soft delete table Query deleteTableQuery = connectionPtr->query(); - deleteTableQuery << "DELETE FROM meta WHERE table_id = " << quote << table_id << ";"; - if (deleteTableQuery.exec()) { - return Status::OK(); - } - else { - return Status::DBTransactionError("Delete Table Error", deleteTableQuery.error()); +// + deleteTableQuery << "UPDATE meta " << + "SET state = " << std::to_string(TableSchema::TO_DELETE) << " " << + "WHERE table_id = " << quote << table_id << ";"; + + if (!deleteTableQuery.exec()) { + return Status::DBTransactionError("QUERY ERROR WHEN DELETING TABLE", deleteTableQuery.error()); } } catch (const BadQuery& er) { // Handle any query errors @@ -348,6 +390,35 @@ namespace meta { // Catch-all for any other MySQL++ exceptions return Status::DBTransactionError("GENERAL ERROR WHEN DELETING TABLE", er.what()); } + + return Status::OK(); + } + + Status MySQLMetaImpl::DeleteTableFiles(const std::string& table_id) { + try { + MetricCollector metric; + + //soft delete table files + Query deleteTableFilesQuery = connectionPtr->query(); + // + deleteTableFilesQuery << "UPDATE metaFile " << + "SET state = " << std::to_string(TableSchema::TO_DELETE) << ", " << + "updated_time = " << std::to_string(utils::GetMicroSecTimeStamp()) << " " + "WHERE table_id = " << quote << table_id << ";"; + + if (!deleteTableFilesQuery.exec()) { + return Status::DBTransactionError("QUERY ERROR WHEN DELETING TABLE", deleteTableFilesQuery.error()); + } + + } catch (const BadQuery& er) { + // Handle any query errors + return Status::DBTransactionError("QUERY ERROR WHEN DELETING TABLE FILES", er.what()); + } catch (const Exception& er) { + // Catch-all for any other MySQL++ exceptions + return Status::DBTransactionError("GENERAL ERROR WHEN DELETING TABLE FILES", er.what()); + } + + return Status::OK(); } Status MySQLMetaImpl::DescribeTable(TableSchema &table_schema) { @@ -355,48 +426,26 @@ namespace meta { std::lock_guard lock(mysql_mutex); try { - server::Metrics::GetInstance().MetaAccessTotalIncrement(); - auto start_time = METRICS_NOW_TIME; + + MetricCollector metric; Query describeTableQuery = connectionPtr->query(); - describeTableQuery << "SELECT id, table_id, dimension, files_cnt, engine_type, store_raw_data " << + describeTableQuery << "SELECT id, dimension, files_cnt, engine_type, store_raw_data " << "FROM meta " << - "WHERE table_id = " << quote << table_schema.table_id_ << ";"; + "WHERE table_id = " << quote << table_schema.table_id_ << " " << + "AND state <> " << std::to_string(TableSchema::TO_DELETE) << ";"; StoreQueryResult res = describeTableQuery.store(); - auto end_time = METRICS_NOW_TIME; - auto total_time = METRICS_MICROSECONDS(start_time, end_time); - server::Metrics::GetInstance().MetaAccessDurationSecondsHistogramObserve(total_time); - -// if (!res) { -// return Status::DBTransactionError("QUERY ERROR WHEN DESCRIBING TABLE", describeTableQuery.error()); -// } assert(res && res.num_rows() <= 1); if (res.num_rows() == 1) { const Row& resRow = res[0]; -// std::string id; -// resRow["id"].to_string(id); -// table_schema.id_ = std::stoul(id); table_schema.id_ = resRow["id"]; //implicit conversion - std::string table_id; - resRow["table_id"].to_string(table_id); - table_schema.table_id_ = table_id; - -// std::string created_on; -// resRow["created_on"].to_string(created_on); -// table_schema.created_on_ = std::stol(created_on); table_schema.dimension_ = resRow["dimension"]; -// std::string files_cnt; -// resRow["files_cnt"].to_string(files_cnt); -// table_schema.files_cnt_ = std::stoul(files_cnt); table_schema.files_cnt_ = resRow["files_cnt"]; -// std::string engine_type; -// resRow["engine_type"].to_string(engine_type); -// table_schema.engine_type_ = std::stoi(engine_type); table_schema.engine_type_ = resRow["engine_type"]; table_schema.store_raw_data_ = (resRow["store_raw_data"].compare("true") == 0); @@ -424,19 +473,20 @@ namespace meta { std::lock_guard lock(mysql_mutex); try { - server::Metrics::GetInstance().MetaAccessTotalIncrement(); - auto start_time = METRICS_NOW_TIME; + + MetricCollector metric; + + Query hasTableQuery = connectionPtr->query(); //since table_id is a unique column we just need to check whether it exists or not - hasTableQuery << "SELECT EXISTS (SELECT 1 FROM meta WHERE table_id = " << quote << table_id << ") " - << "AS " << quote << "check" << ";"; + hasTableQuery << "SELECT EXISTS " << + "(SELECT 1 FROM meta " << + "WHERE table_id = " << quote << table_id << " " << + "AND state <> " << std::to_string(TableSchema::TO_DELETE) << ") " << + "AS " << quote << "check" << ";"; StoreQueryResult res = hasTableQuery.store(); - auto end_time = METRICS_NOW_TIME; - auto total_time = METRICS_MICROSECONDS(start_time, end_time); - server::Metrics::GetInstance().MetaAccessDurationSecondsHistogramObserve(total_time); - assert(res && res.num_rows() == 1); int check = res[0]["check"]; has_or_not = (check == 1); @@ -457,18 +507,15 @@ namespace meta { std::lock_guard lock(mysql_mutex); try { - server::Metrics::GetInstance().MetaAccessTotalIncrement(); - auto start_time = METRICS_NOW_TIME; + + MetricCollector metric; Query allTablesQuery = connectionPtr->query(); allTablesQuery << "SELECT id, table_id, dimension, files_cnt, engine_type, store_raw_data " << - "FROM meta;"; + "FROM meta " << + "WHERE state <> " << std::to_string(TableSchema::TO_DELETE) << ";"; StoreQueryResult res = allTablesQuery.store(); - auto end_time = METRICS_NOW_TIME; - auto total_time = METRICS_MICROSECONDS(start_time, end_time); - server::Metrics::GetInstance().MetaAccessDurationSecondsHistogramObserve(total_time); - for (auto& resRow : res) { TableSchema table_schema; @@ -513,68 +560,63 @@ namespace meta { return status; } - NextFileId(file_schema.file_id_); - file_schema.file_type_ = TableFileSchema::NEW; - file_schema.dimension_ = table_schema.dimension_; - file_schema.size_ = 0; - file_schema.created_on_ = utils::GetMicroSecTimeStamp(); - file_schema.updated_time_ = file_schema.created_on_; - file_schema.engine_type_ = table_schema.engine_type_; - GetTableFilePath(file_schema); + try { - { - try { - server::Metrics::GetInstance().MetaAccessTotalIncrement(); - auto start_time = METRICS_NOW_TIME; + MetricCollector metric; - Query createTableFileQuery = connectionPtr->query(); - std::string id = "NULL"; //auto-increment - std::string table_id = file_schema.table_id_; - std::string engine_type = std::to_string(file_schema.engine_type_); - std::string file_id = file_schema.file_id_; - std::string file_type = std::to_string(file_schema.file_type_); - std::string size = std::to_string(file_schema.size_); - std::string updated_time = std::to_string(file_schema.updated_time_); - std::string created_on = std::to_string(file_schema.created_on_); - std::string date = std::to_string(file_schema.date_); + NextFileId(file_schema.file_id_); + file_schema.file_type_ = TableFileSchema::NEW; + file_schema.dimension_ = table_schema.dimension_; + file_schema.size_ = 0; + file_schema.created_on_ = utils::GetMicroSecTimeStamp(); + file_schema.updated_time_ = file_schema.created_on_; + file_schema.engine_type_ = table_schema.engine_type_; + GetTableFilePath(file_schema); - createTableFileQuery << "INSERT INTO metaFile VALUES" << - "(" << id << ", " << quote << table_id << ", " << engine_type << ", " << - quote << file_id << ", " << file_type << ", " << size << ", " << - updated_time << ", " << created_on << ", " << date << ");"; + Query createTableFileQuery = connectionPtr->query(); + std::string id = "NULL"; //auto-increment + std::string table_id = file_schema.table_id_; + std::string engine_type = std::to_string(file_schema.engine_type_); + std::string file_id = file_schema.file_id_; + std::string file_type = std::to_string(file_schema.file_type_); + std::string size = std::to_string(file_schema.size_); + std::string updated_time = std::to_string(file_schema.updated_time_); + std::string created_on = std::to_string(file_schema.created_on_); + std::string date = std::to_string(file_schema.date_); - if (SimpleResult res = createTableFileQuery.execute()) { - file_schema.id_ = res.insert_id(); //Might need to use SELECT LAST_INSERT_ID()? + createTableFileQuery << "INSERT INTO metaFile VALUES" << + "(" << id << ", " << quote << table_id << ", " << engine_type << ", " << + quote << file_id << ", " << file_type << ", " << size << ", " << + updated_time << ", " << created_on << ", " << date << ");"; - //Consume all results to avoid "Commands out of sync" error - while (createTableFileQuery.more_results()) { - createTableFileQuery.store_next(); - } - } - else { - return Status::DBTransactionError("Add file Error", createTableFileQuery.error()); - } + if (SimpleResult res = createTableFileQuery.execute()) { + file_schema.id_ = res.insert_id(); //Might need to use SELECT LAST_INSERT_ID()? - auto end_time = METRICS_NOW_TIME; - auto total_time = METRICS_MICROSECONDS(start_time, end_time); - server::Metrics::GetInstance().MetaAccessDurationSecondsHistogramObserve(total_time); - } catch (const BadQuery& er) { - // Handle any query errors - return Status::DBTransactionError("QUERY ERROR WHEN ADDING TABLE FILE", er.what()); - } catch (const Exception& er) { - // Catch-all for any other MySQL++ exceptions - return Status::DBTransactionError("GENERAL ERROR WHEN ADDING TABLE FILE", er.what()); + //Consume all results to avoid "Commands out of sync" error +// while (createTableFileQuery.more_results()) { +// createTableFileQuery.store_next(); +// } } - } - - auto partition_path = GetTableDatePartitionPath(file_schema.table_id_, file_schema.date_); - - if (!boost::filesystem::is_directory(partition_path)) { - auto ret = boost::filesystem::create_directory(partition_path); - if (!ret) { - ENGINE_LOG_ERROR << "Create directory " << partition_path << " Error"; + else { + return Status::DBTransactionError("Add file Error", createTableFileQuery.error()); } - assert(ret); + + auto partition_path = GetTableDatePartitionPath(file_schema.table_id_, file_schema.date_); + + if (!boost::filesystem::is_directory(partition_path)) { + auto ret = boost::filesystem::create_directory(partition_path); + if (!ret) { + ENGINE_LOG_ERROR << "Create directory " << partition_path << " Error"; + return Status::DBTransactionError("Failed to create partition directory"); + } + } + + } catch (const BadQuery& er) { + // Handle any query errors + return Status::DBTransactionError("QUERY ERROR WHEN ADDING TABLE FILE", er.what()); + } catch (const Exception& er) { + // Catch-all for any other MySQL++ exceptions + return Status::DBTransactionError("GENERAL ERROR WHEN ADDING TABLE FILE", er.what()); } return Status::OK(); @@ -587,8 +629,8 @@ namespace meta { files.clear(); try { - server::Metrics::GetInstance().MetaAccessTotalIncrement(); - auto start_time = METRICS_NOW_TIME; + + MetricCollector metric; Query filesToIndexQuery = connectionPtr->query(); filesToIndexQuery << "SELECT id, table_id, engine_type, file_id, file_type, size, date " << @@ -596,10 +638,6 @@ namespace meta { "WHERE file_type = " << std::to_string(TableFileSchema::TO_INDEX) << ";"; StoreQueryResult res = filesToIndexQuery.store(); - auto end_time = METRICS_NOW_TIME; - auto total_time = METRICS_MICROSECONDS(start_time, end_time); - server::Metrics::GetInstance().MetaAccessDurationSecondsHistogramObserve(total_time); - std::map groups; TableFileSchema table_file; for (auto& resRow : res) { @@ -659,8 +697,8 @@ namespace meta { files.clear(); try { - server::Metrics::GetInstance().MetaAccessTotalIncrement(); - auto start_time = METRICS_NOW_TIME; + + MetricCollector metric; StoreQueryResult res; @@ -675,9 +713,6 @@ namespace meta { "file_type = " << std::to_string(TableFileSchema::INDEX) << ");"; res = filesToSearchQuery.store(); - auto end_time = METRICS_NOW_TIME; - auto total_time = METRICS_MICROSECONDS(start_time, end_time); - server::Metrics::GetInstance().MetaAccessDurationSecondsHistogramObserve(total_time); } else { @@ -691,17 +726,14 @@ namespace meta { partitionListStr = partitionListStr.substr(0, partitionListStr.size() - 2); //remove the last ", " filesToSearchQuery << "SELECT id, table_id, engine_type, file_id, file_type, size, date " << - "FROM metaFile " << - "WHERE table_id = " << quote << table_id << " AND " << - "date IN (" << partitionListStr << ") AND " << - "(file_type = " << std::to_string(TableFileSchema::RAW) << " OR " << - "file_type = " << std::to_string(TableFileSchema::TO_INDEX) << " OR " << - "file_type = " << std::to_string(TableFileSchema::INDEX) << ");"; + "FROM metaFile " << + "WHERE table_id = " << quote << table_id << " AND " << + "date IN (" << partitionListStr << ") AND " << + "(file_type = " << std::to_string(TableFileSchema::RAW) << " OR " << + "file_type = " << std::to_string(TableFileSchema::TO_INDEX) << " OR " << + "file_type = " << std::to_string(TableFileSchema::INDEX) << ");"; res = filesToSearchQuery.store(); - auto end_time = METRICS_NOW_TIME; - auto total_time = METRICS_MICROSECONDS(start_time, end_time); - server::Metrics::GetInstance().MetaAccessDurationSecondsHistogramObserve(total_time); } TableSchema table_schema; @@ -762,20 +794,16 @@ namespace meta { files.clear(); try { - server::Metrics::GetInstance().MetaAccessTotalIncrement(); - auto start_time = METRICS_NOW_TIME; + MetricCollector metric; Query filesToMergeQuery = connectionPtr->query(); filesToMergeQuery << "SELECT id, table_id, file_id, file_type, size, date " << "FROM metaFile " << "WHERE table_id = " << quote << table_id << " AND " << - "file_type = " << std::to_string(TableFileSchema::RAW) << ";"; + "file_type = " << std::to_string(TableFileSchema::RAW) << " " << + "ORDER BY size DESC" << ";"; StoreQueryResult res = filesToMergeQuery.store(); - auto end_time = METRICS_NOW_TIME; - auto total_time = METRICS_MICROSECONDS(start_time, end_time); - server::Metrics::GetInstance().MetaAccessDurationSecondsHistogramObserve(total_time); - TableSchema table_schema; table_schema.table_id_ = table_id; auto status = DescribeTable(table_schema); @@ -826,40 +854,45 @@ namespace meta { return Status::OK(); } - //ZR: TODO: this function is pending to be removed, so not gonna implemented for now - Status MySQLMetaImpl::FilesToDelete(const std::string& table_id, - const DatesT& partition, - DatePartionedTableFilesSchema& files) { - -// std::lock_guard lock(mysql_mutex); - - return Status::OK(); - } - - Status MySQLMetaImpl::GetTableFile(TableFileSchema &file_schema) { + Status MySQLMetaImpl::GetTableFiles(const std::string& table_id, + const std::vector& ids, + TableFilesSchema& table_files) { std::lock_guard lock(mysql_mutex); + std::stringstream idSS; + for (auto& id : ids) { + idSS << "id = " << std::to_string(id) << " OR "; + } + std::string idStr = idSS.str(); + idStr = idStr.substr(0, idStr.size() - 4); //remove the last " OR " + try { Query getTableFileQuery = connectionPtr->query(); - getTableFileQuery << "SELECT id, table_id, file_id, file_type, size, date " << + getTableFileQuery << "SELECT engine_type, file_id, file_type, size, date " << "FROM metaFile " << - "WHERE file_id = " << quote << file_schema.file_id_ << " AND " << - "table_id = " << quote << file_schema.table_id_ << ";"; + "WHERE table_id = " << quote << table_id << " AND " << + "(" << idStr << ");"; StoreQueryResult res = getTableFileQuery.store(); - assert(res && res.num_rows() <= 1); - if (res.num_rows() == 1) { + assert(res); - const Row& resRow = res[0]; + TableSchema table_schema; + table_schema.table_id_ = table_id; + auto status = DescribeTable(table_schema); + if (!status.ok()) { + return status; + } - file_schema.id_ = resRow["id"]; //implicit conversion + for (auto& resRow : res) { + + TableFileSchema file_schema; - std::string table_id; - resRow["table_id"].to_string(table_id); file_schema.table_id_ = table_id; + file_schema.engine_type_ = resRow["engine_type"]; + std::string file_id; resRow["file_id"].to_string(file_id); file_schema.file_id_ = file_id; @@ -869,17 +902,19 @@ namespace meta { file_schema.size_ = resRow["size"]; file_schema.date_ = resRow["date"]; - } - else { - return Status::NotFound("Table:" + file_schema.table_id_ + - " File:" + file_schema.file_id_ + " not found"); + + file_schema.dimension_ = table_schema.dimension_; + + GetTableFilePath(file_schema); + + table_files.emplace_back(file_schema); } } catch (const BadQuery& er) { // Handle any query errors - return Status::DBTransactionError("QUERY ERROR WHEN RETRIEVING TABLE FILE", er.what()); + return Status::DBTransactionError("QUERY ERROR WHEN RETRIEVING TABLE FILES", er.what()); } catch (const Exception& er) { // Catch-all for any other MySQL++ exceptions - return Status::DBTransactionError("GENERAL ERROR WHEN RETRIEVING TABLE FILE", er.what()); + return Status::DBTransactionError("GENERAL ERROR WHEN RETRIEVING TABLE FILES", er.what()); } return Status::OK(); @@ -924,7 +959,7 @@ namespace meta { uint64_t sum = 0; Size(sum); - long long to_delete = (sum - limit * G); + auto to_delete = (sum - limit * G); DiscardFiles(to_delete); } } @@ -974,13 +1009,16 @@ namespace meta { std::lock_guard lock(mysql_mutex); - LOG(DEBUG) << "About to discard size=" << to_discard_size; if (to_discard_size <= 0) { // std::cout << "in" << std::endl; return Status::OK(); } + ENGINE_LOG_DEBUG << "About to discard size=" << to_discard_size; + try { + MetricCollector metric; + Query discardFilesQuery = connectionPtr->query(); discardFilesQuery << "SELECT id, size " << "FROM metaFile " << @@ -1013,7 +1051,8 @@ namespace meta { idsToDiscardStr = idsToDiscardStr.substr(0, idsToDiscardStr.size() - 4); //remove the last " OR " discardFilesQuery << "UPDATE metaFile " << - "SET file_type = " << std::to_string(TableFileSchema::TO_DELETE) << " " << + "SET file_type = " << std::to_string(TableFileSchema::TO_DELETE) << ", " << + "updated_time = " << std::to_string(utils::GetMicroSecTimeStamp()) << " " << "WHERE " << idsToDiscardStr << ";"; if (discardFilesQuery.exec()) { @@ -1039,11 +1078,27 @@ namespace meta { file_schema.updated_time_ = utils::GetMicroSecTimeStamp(); try { - server::Metrics::GetInstance().MetaAccessTotalIncrement(); - auto start_time = METRICS_NOW_TIME; + + MetricCollector metric; Query updateTableFileQuery = connectionPtr->query(); + //if the table has been deleted, just mark the table file as TO_DELETE + //clean thread will delete the file later + updateTableFileQuery << "SELECT state FROM meta " << + "WHERE table_id = " << quote << file_schema.table_id_ << ";"; + StoreQueryResult res = updateTableFileQuery.store(); + assert(res && res.num_rows() <= 1); + if (res.num_rows() == 1) { + int state = res[0]["state"]; + if (state == TableSchema::TO_DELETE) { + file_schema.file_type_ = TableFileSchema::TO_DELETE; + } + } + else { + file_schema.file_type_ = TableFileSchema::TO_DELETE; + } + std::string id = std::to_string(file_schema.id_); std::string table_id = file_schema.table_id_; std::string engine_type = std::to_string(file_schema.engine_type_); @@ -1072,9 +1127,6 @@ namespace meta { return Status::DBTransactionError("QUERY ERROR WHEN UPDATING TABLE FILE", updateTableFileQuery.error()); } - auto end_time = METRICS_NOW_TIME; - auto total_time = METRICS_MICROSECONDS(start_time, end_time); - server::Metrics::GetInstance().MetaAccessDurationSecondsHistogramObserve(total_time); } catch (const BadQuery& er) { // Handle any query errors ENGINE_LOG_DEBUG << "table_id= " << file_schema.table_id_ << " file_id=" << file_schema.file_id_; @@ -1092,13 +1144,36 @@ namespace meta { std::lock_guard lock(mysql_mutex); try { - server::Metrics::GetInstance().MetaAccessTotalIncrement(); - auto start_time = METRICS_NOW_TIME; + MetricCollector metric; Query updateTableFilesQuery = connectionPtr->query(); + std::map has_tables; + for (auto &file_schema : files) { + + if(has_tables.find(file_schema.table_id_) != has_tables.end()) { + continue; + } + + updateTableFilesQuery << "SELECT EXISTS " << + "(SELECT 1 FROM meta " << + "WHERE table_id = " << quote << file_schema.table_id_ << " " << + "AND state <> " << std::to_string(TableSchema::TO_DELETE) << ") " << + "AS " << quote << "check" << ";"; + StoreQueryResult res = updateTableFilesQuery.store(); + + assert(res && res.num_rows() == 1); + int check = res[0]["check"]; + has_tables[file_schema.table_id_] = (check == 1); + } + for (auto& file_schema : files) { + if(!has_tables[file_schema.table_id_]) { + file_schema.file_type_ = TableFileSchema::TO_DELETE; + } + file_schema.updated_time_ = utils::GetMicroSecTimeStamp(); + std::string id = std::to_string(file_schema.id_); std::string table_id = file_schema.table_id_; std::string engine_type = std::to_string(file_schema.engine_type_); @@ -1142,12 +1217,13 @@ namespace meta { auto now = utils::GetMicroSecTimeStamp(); try { + MetricCollector metric; Query cleanUpFilesWithTTLQuery = connectionPtr->query(); - cleanUpFilesWithTTLQuery << "SELECT id, table_id, file_id, file_type, size, date " << + cleanUpFilesWithTTLQuery << "SELECT id, table_id, file_id, date " << "FROM metaFile " << "WHERE file_type = " << std::to_string(TableFileSchema::TO_DELETE) << " AND " << - "updated_time > " << std::to_string(now - seconds * US_PS) << ";"; + "updated_time < " << std::to_string(now - seconds * US_PS) << ";"; StoreQueryResult res = cleanUpFilesWithTTLQuery.store(); assert(res); @@ -1167,17 +1243,12 @@ namespace meta { resRow["file_id"].to_string(file_id); table_file.file_id_ = file_id; - table_file.file_type_ = resRow["file_type"]; - - table_file.size_ = resRow["size"]; - table_file.date_ = resRow["date"]; GetTableFilePath(table_file); - if (table_file.file_type_ == TableFileSchema::TO_DELETE) { - boost::filesystem::remove(table_file.location_); - } + ENGINE_LOG_DEBUG << "Removing deleted id =" << table_file.id_ << " location = " << table_file.location_ << std::endl; + boost::filesystem::remove(table_file.location_); idsToDelete.emplace_back(std::to_string(table_file.id_)); } @@ -1202,6 +1273,45 @@ namespace meta { return Status::DBTransactionError("GENERAL ERROR WHEN CLEANING UP FILES WITH TTL", er.what()); } + try { + MetricCollector metric; + + Query cleanUpFilesWithTTLQuery = connectionPtr->query(); + cleanUpFilesWithTTLQuery << "SELECT id, table_id " << + "FROM meta " << + "WHERE state = " << std::to_string(TableSchema::TO_DELETE) << ";"; + StoreQueryResult res = cleanUpFilesWithTTLQuery.store(); + assert(res); + std::stringstream idsToDeleteSS; + for (auto& resRow : res) { + size_t id = resRow["id"]; + std::string table_id; + resRow["table_id"].to_string(table_id); + + auto table_path = GetTablePath(table_id); + + ENGINE_LOG_DEBUG << "Remove table folder: " << table_path; + boost::filesystem::remove_all(table_path); + + idsToDeleteSS << "id = " << std::to_string(id) << " OR "; + } + std::string idsToDeleteStr = idsToDeleteSS.str(); + idsToDeleteStr = idsToDeleteStr.substr(0, idsToDeleteStr.size() - 4); //remove the last " OR " + cleanUpFilesWithTTLQuery << "DELETE FROM meta WHERE " << + idsToDeleteStr << ";"; + if (!cleanUpFilesWithTTLQuery.exec()) { + return Status::DBTransactionError("QUERY ERROR WHEN CLEANING UP FILES WITH TTL", cleanUpFilesWithTTLQuery.error()); + } + + + } catch (const BadQuery& er) { + // Handle any query errors + return Status::DBTransactionError("QUERY ERROR WHEN CLEANING UP FILES WITH TTL", er.what()); + } catch (const Exception& er) { + // Catch-all for any other MySQL++ exceptions + return Status::DBTransactionError("GENERAL ERROR WHEN CLEANING UP FILES WITH TTL", er.what()); + } + return Status::OK(); } @@ -1210,65 +1320,10 @@ namespace meta { std::lock_guard lock(mysql_mutex); try { -// auto selected = ConnectorPtr->select(columns(&TableFileSchema::id_, -// &TableFileSchema::table_id_, -// &TableFileSchema::file_id_, -// &TableFileSchema::file_type_, -// &TableFileSchema::size_, -// &TableFileSchema::date_), -// where( -// c(&TableFileSchema::file_type_) == (int) TableFileSchema::TO_DELETE -// or -// c(&TableFileSchema::file_type_) -// == (int) TableFileSchema::NEW)); - + ENGINE_LOG_DEBUG << "Remove table file type as NEW"; Query cleanUpQuery = connectionPtr->query(); - cleanUpQuery << "SELECT id, table_id, file_id, file_type, size, date " << - "FROM metaFile " << - "WHERE file_type = " << std::to_string(TableFileSchema::TO_DELETE) << " OR " << - "file_type = " << std::to_string(TableFileSchema::NEW) << ";"; - StoreQueryResult res = cleanUpQuery.store(); + cleanUpQuery << "DELETE FROM metaFile WHERE file_type = " << std::to_string(TableFileSchema::NEW) << ";"; - assert(res); - - TableFileSchema table_file; - std::vector idsToDelete; - - for (auto& resRow : res) { - - table_file.id_ = resRow["id"]; //implicit conversion - - std::string table_id; - resRow["table_id"].to_string(table_id); - table_file.table_id_ = table_id; - - std::string file_id; - resRow["file_id"].to_string(file_id); - table_file.file_id_ = file_id; - - table_file.file_type_ = resRow["file_type"]; - - table_file.size_ = resRow["size"]; - - table_file.date_ = resRow["date"]; - - GetTableFilePath(table_file); - - if (table_file.file_type_ == TableFileSchema::TO_DELETE) { - boost::filesystem::remove(table_file.location_); - } - - idsToDelete.emplace_back(std::to_string(table_file.id_)); - } - - std::stringstream idsToDeleteSS; - for (auto& id : idsToDelete) { - idsToDeleteSS << "id = " << id << " OR "; - } - std::string idsToDeleteStr = idsToDeleteSS.str(); - idsToDeleteStr = idsToDeleteStr.substr(0, idsToDeleteStr.size() - 4); //remove the last " OR " - cleanUpQuery << "DELETE FROM metaFile WHERE " << - idsToDeleteStr << ";"; if (!cleanUpQuery.exec()) { return Status::DBTransactionError("Clean up Error", cleanUpQuery.error()); } @@ -1289,9 +1344,7 @@ namespace meta { std::lock_guard lock(mysql_mutex); try { - - server::Metrics::GetInstance().MetaAccessTotalIncrement(); - auto start_time = METRICS_NOW_TIME; + MetricCollector metric; Query countQuery = connectionPtr->query(); countQuery << "SELECT size " << @@ -1302,10 +1355,6 @@ namespace meta { "file_type = " << std::to_string(TableFileSchema::INDEX) << ");"; StoreQueryResult res = countQuery.store(); - auto end_time = METRICS_NOW_TIME; - auto total_time = METRICS_MICROSECONDS(start_time, end_time); - server::Metrics::GetInstance().MetaAccessDurationSecondsHistogramObserve(total_time); - TableSchema table_schema; table_schema.table_id_ = table_id; auto status = DescribeTable(table_schema); diff --git a/cpp/src/db/MySQLMetaImpl.h b/cpp/src/db/MySQLMetaImpl.h index 8dd2f4cabf..f151c2f928 100644 --- a/cpp/src/db/MySQLMetaImpl.h +++ b/cpp/src/db/MySQLMetaImpl.h @@ -24,16 +24,20 @@ namespace meta { MySQLMetaImpl(const DBMetaOptions& options_); virtual Status CreateTable(TableSchema& table_schema) override; - virtual Status DeleteTable(const std::string& table_id) override; virtual Status DescribeTable(TableSchema& group_info_) override; virtual Status HasTable(const std::string& table_id, bool& has_or_not) override; virtual Status AllTables(std::vector& table_schema_array) override; + virtual Status DeleteTable(const std::string& table_id) override; + virtual Status DeleteTableFiles(const std::string& table_id) override; + virtual Status CreateTableFile(TableFileSchema& file_schema) override; virtual Status DropPartitionsByDates(const std::string& table_id, const DatesT& dates) override; - virtual Status GetTableFile(TableFileSchema& file_schema) override; + virtual Status GetTableFiles(const std::string& table_id, + const std::vector& ids, + TableFilesSchema& table_files) override; virtual Status UpdateTableFile(TableFileSchema& file_schema) override; @@ -46,10 +50,6 @@ namespace meta { virtual Status FilesToMerge(const std::string& table_id, DatePartionedTableFilesSchema& files) override; - virtual Status FilesToDelete(const std::string& table_id, - const DatesT& partition, - DatePartionedTableFilesSchema& files) override; - virtual Status FilesToIndex(TableFilesSchema&) override; virtual Status Archive() override; @@ -74,7 +74,6 @@ namespace meta { std::string GetTableDatePartitionPath(const std::string& table_id, DateT& date); void GetTableFilePath(TableFileSchema& group_file); Status Initialize(); - std::unique_ptr& getConnectionPtr(); const DBMetaOptions options_; diff --git a/cpp/unittest/db/MySQLMetaImpl_test.cpp b/cpp/unittest/db/MySQLMetaImpl_test.cpp index c7fc611d7b..ed41df65d7 100644 --- a/cpp/unittest/db/MySQLMetaImpl_test.cpp +++ b/cpp/unittest/db/MySQLMetaImpl_test.cpp @@ -142,23 +142,30 @@ TEST_F(MySQLTest, core) { ASSERT_EQ(fileToMerge.table_id_, "test1"); ASSERT_EQ(fileToMerge.dimension_, 123); - meta::TableFileSchema resultTableFileSchema; - resultTableFileSchema.table_id_ = tableFileSchema.table_id_; - resultTableFileSchema.file_id_ = tableFileSchema.file_id_; - status = impl.GetTableFile(resultTableFileSchema); + meta::TableFilesSchema resultTableFilesSchema; + std::vector ids; + ids.push_back(tableFileSchema.id_); + status = impl.GetTableFiles(tableFileSchema.table_id_, ids, resultTableFilesSchema); ASSERT_TRUE(status.ok()); - ASSERT_EQ(resultTableFileSchema.id_, tableFileSchema.id_); + ASSERT_EQ(resultTableFilesSchema.size(), 1); + meta::TableFileSchema resultTableFileSchema = resultTableFilesSchema[0]; +// ASSERT_EQ(resultTableFileSchema.id_, tableFileSchema.id_); ASSERT_EQ(resultTableFileSchema.table_id_, tableFileSchema.table_id_); ASSERT_EQ(resultTableFileSchema.file_id_, tableFileSchema.file_id_); ASSERT_EQ(resultTableFileSchema.file_type_, tableFileSchema.file_type_); ASSERT_EQ(resultTableFileSchema.size_, tableFileSchema.size_); ASSERT_EQ(resultTableFileSchema.date_, tableFileSchema.date_); + ASSERT_EQ(resultTableFileSchema.engine_type_, tableFileSchema.engine_type_); + ASSERT_EQ(resultTableFileSchema.dimension_, tableFileSchema.dimension_); tableFileSchema.size_ = 234; - status = impl.CreateTable(schema2); + meta::TableSchema schema3; + schema3.table_id_ = "test3"; + schema3.dimension_ = 321; + status = impl.CreateTable(schema3); ASSERT_TRUE(status.ok()); meta::TableFileSchema tableFileSchema2; - tableFileSchema2.table_id_ = "test2"; + tableFileSchema2.table_id_ = "test3"; tableFileSchema2.size_ = 345; status = impl.CreateTableFile(tableFileSchema2); ASSERT_TRUE(status.ok()); @@ -184,18 +191,14 @@ TEST_F(MySQLTest, core) { } TEST_F(MySQLTest, GROUP_TEST) { - -// DBMetaOptions options; -// options.backend_uri = "mysql://root:1234@:/test"; -// options.path = "/tmp/vecwise_test"; - meta::MySQLMetaImpl impl(getDBMetaOptions()); + meta::MySQLMetaImpl impl(getDBMetaOptions()); + auto table_id = "meta_test_group"; meta::TableSchema group; group.table_id_ = table_id; auto status = impl.CreateTable(group); -// std::cout << status.ToString() << std::endl; ASSERT_TRUE(status.ok()); auto gid = group.id_; @@ -219,9 +222,6 @@ TEST_F(MySQLTest, GROUP_TEST) { TEST_F(MySQLTest, table_file_TEST) { -// DBMetaOptions options; -// options.backend_uri = "mysql://root:1234@:/test"; -// options.path = "/tmp/vecwise_test"; meta::MySQLMetaImpl impl(getDBMetaOptions()); auto table_id = "meta_test_group"; @@ -267,9 +267,13 @@ TEST_F(MySQLTest, table_file_TEST) { dates.push_back(table_file.date_); status = impl.DropPartitionsByDates(table_file.table_id_, dates); ASSERT_TRUE(status.ok()); - status = impl.GetTableFile(table_file); + + std::vector ids = {table_file.id_}; + meta::TableFilesSchema files; + status = impl.GetTableFiles(table_file.table_id_, ids, files); ASSERT_TRUE(status.ok()); - ASSERT_TRUE(table_file.file_type_ == meta::TableFileSchema::TO_DELETE); + ASSERT_EQ(files.size(), 1UL); + ASSERT_TRUE(files[0].file_type_ == meta::TableFileSchema::TO_DELETE); status = impl.DropAll(); ASSERT_TRUE(status.ok()); @@ -278,12 +282,10 @@ TEST_F(MySQLTest, table_file_TEST) { TEST_F(MySQLTest, ARCHIVE_TEST_DAYS) { srand(time(0)); DBMetaOptions options = getDBMetaOptions(); -// options.path = "/tmp/vecwise_test"; int days_num = rand() % 100; std::stringstream ss; ss << "days:" << days_num; options.archive_conf = ArchiveConf("delete", ss.str()); -// options.backend_uri = "mysql://root:1234@:/test"; meta::MySQLMetaImpl impl(options); @@ -300,6 +302,7 @@ TEST_F(MySQLTest, ARCHIVE_TEST_DAYS) { auto cnt = 100; long ts = utils::GetMicroSecTimeStamp(); std::vector days; + std::vector ids; for (auto i=0; i ids; for (auto i=0; iDropAll(); + ASSERT_TRUE(stat.ok()); }; TEST_F(MySQLDBTest, DB_TEST) { @@ -367,6 +370,9 @@ TEST_F(MySQLDBTest, DB_TEST) { } search.join(); + + stat = db_->DropAll(); + ASSERT_TRUE(stat.ok()); }; TEST_F(MySQLDBTest, SEARCH_TEST) { @@ -423,6 +429,9 @@ TEST_F(MySQLDBTest, SEARCH_TEST) { stat = db_->Query(TABLE_NAME, k, nq, xq.data(), results); ASSERT_STATS(stat); + stat = db_->DropAll(); + ASSERT_TRUE(stat.ok()); + // TODO(linxj): add groundTruth assert }; @@ -462,6 +471,9 @@ TEST_F(MySQLDBTest, ARHIVE_DISK_CHECK) { db_->Size(size); LOG(DEBUG) << "size=" << size; ASSERT_LE(size, 1 * engine::meta::G); + + stat = db_->DropAll(); + ASSERT_TRUE(stat.ok()); }; TEST_F(MySQLDBTest, DELETE_TEST) { @@ -497,7 +509,13 @@ TEST_F(MySQLDBTest, DELETE_TEST) { std::vector dates; stat = db_->DeleteTable(TABLE_NAME, dates); - std::this_thread::sleep_for(std::chrono::seconds(2)); + + std::this_thread::sleep_for(std::chrono::seconds(10)); //change to 10 to make sure files are discarded + ASSERT_TRUE(stat.ok()); +// std::cout << table_info_get.location_ << std::endl; ASSERT_FALSE(boost::filesystem::exists(table_info_get.location_)); + + stat = db_->DropAll(); + ASSERT_TRUE(stat.ok()); }; From e5892dbefcee32f1c07ca6b72253d59f5910ddfa Mon Sep 17 00:00:00 2001 From: zhiru Date: Mon, 24 Jun 2019 16:44:05 +0800 Subject: [PATCH 11/43] Add MySQL Former-commit-id: b79fe37030a73eb22300c33ff5122a98945daf22 --- cpp/CHANGELOG.md | 1 + cpp/README.md | 13 ++- cpp/cmake/DefineOptions.cmake | 2 + cpp/cmake/ThirdPartyPackages.cmake | 67 ++++++++++++- cpp/conf/server_config.yaml | 4 +- cpp/src/CMakeLists.txt | 2 + cpp/src/db/DBImpl.cpp | 12 ++- cpp/src/db/Factories.cpp | 16 ++-- cpp/src/db/MySQLMetaImpl.cpp | 125 +++++++++++++++++++------ cpp/src/db/MySQLMetaImpl.h | 4 + cpp/thirdparty/versions.txt | 1 + cpp/unittest/db/CMakeLists.txt | 9 +- cpp/unittest/db/MySQLMetaImpl_test.cpp | 1 + cpp/unittest/db/db_tests.cpp | 44 +++++---- cpp/unittest/db/utils.cpp | 34 ++++++- 15 files changed, 262 insertions(+), 73 deletions(-) diff --git a/cpp/CHANGELOG.md b/cpp/CHANGELOG.md index 1ebe05eb21..e88a80fb33 100644 --- a/cpp/CHANGELOG.md +++ b/cpp/CHANGELOG.md @@ -40,6 +40,7 @@ Please mark all change in change log and use the ticket from JIRA. - MS-85 - add NetIO metric - MS-96 - add new query interface for specified files - MS-97 - Add S3 SDK for MinIO Storage +- MS-105 - Add MySQL ## Task - MS-74 - Change README.md in cpp diff --git a/cpp/README.md b/cpp/README.md index 0c7706df23..94bf2241ec 100644 --- a/cpp/README.md +++ b/cpp/README.md @@ -1,13 +1,22 @@ ### Compilation #### Step 1: install necessery tools + Install MySQL + centos7 : - yum install gfortran flex bison + yum install gfortran qt4 flex bison mysql-devel ubuntu16.04 : - sudo apt-get install gfortran flex bison + sudo apt-get install gfortran qt4-qmake flex bison libmysqlclient-dev + +If `libmysqlclient_r.so` does not exist after installing MySQL Development Files, you need to create a symbolic link: + +``` +sudo ln -s /path/to/libmysqlclient.so /path/to/libmysqlclient_r.so +``` #### Step 2: build(output to cmake_build folder) + cmake_build/src/milvus_server is the server cmake_build/src/libmilvus_engine.a is the static library diff --git a/cpp/cmake/DefineOptions.cmake b/cpp/cmake/DefineOptions.cmake index d95e7c7ed1..147663d0db 100644 --- a/cpp/cmake/DefineOptions.cmake +++ b/cpp/cmake/DefineOptions.cmake @@ -93,6 +93,8 @@ define_option(MILVUS_WITH_SQLITE "Build with SQLite library" ON) define_option(MILVUS_WITH_SQLITE_ORM "Build with SQLite ORM library" ON) +define_option(MILVUS_WITH_MYSQLPP "Build with MySQL++" ON) + define_option(MILVUS_WITH_THRIFT "Build with Apache Thrift library" ON) define_option(MILVUS_WITH_YAMLCPP "Build with yaml-cpp library" ON) diff --git a/cpp/cmake/ThirdPartyPackages.cmake b/cpp/cmake/ThirdPartyPackages.cmake index cb5f3532fe..99f34682e1 100644 --- a/cpp/cmake/ThirdPartyPackages.cmake +++ b/cpp/cmake/ThirdPartyPackages.cmake @@ -26,6 +26,7 @@ set(MILVUS_THIRDPARTY_DEPENDENCIES JSONCONS LAPACK Lz4 + MySQLPP OpenBLAS Prometheus RocksDB @@ -56,12 +57,14 @@ macro(build_dependency DEPENDENCY_NAME) build_easyloggingpp() elseif("${DEPENDENCY_NAME}" STREQUAL "FAISS") build_faiss() + elseif ("${DEPENDENCY_NAME}" STREQUAL "GTest") + build_gtest() elseif("${DEPENDENCY_NAME}" STREQUAL "LAPACK") build_lapack() elseif("${DEPENDENCY_NAME}" STREQUAL "Lz4") build_lz4() - elseif ("${DEPENDENCY_NAME}" STREQUAL "GTest") - build_gtest() + elseif ("${DEPENDENCY_NAME}" STREQUAL "MySQLPP") + build_mysqlpp() elseif ("${DEPENDENCY_NAME}" STREQUAL "JSONCONS") build_jsoncons() elseif ("${DEPENDENCY_NAME}" STREQUAL "OpenBLAS") @@ -265,6 +268,12 @@ else() set(LZ4_SOURCE_URL "https://github.com/lz4/lz4/archive/${LZ4_VERSION}.tar.gz") endif() +if(DEFINED ENV{MILVUS_MYSQLPP_URL}) + set(MYSQLPP_SOURCE_URL "$ENV{MILVUS_MYSQLPP_URL}") +else() + set(MYSQLPP_SOURCE_URL "https://tangentsoft.com/mysqlpp/releases/mysql++-${MYSQLPP_VERSION}.tar.gz") +endif() + if (DEFINED ENV{MILVUS_OPENBLAS_URL}) set(OPENBLAS_SOURCE_URL "$ENV{MILVUS_OPENBLAS_URL}") else () @@ -829,8 +838,8 @@ macro(build_faiss) # ${MAKE} ${MAKE_BUILD_ARGS} BUILD_COMMAND ${MAKE} ${MAKE_BUILD_ARGS} all - COMMAND - cd gpu && make ${MAKE_BUILD_ARGS} + COMMAND + cd gpu && ${MAKE} ${MAKE_BUILD_ARGS} BUILD_IN_SOURCE 1 # INSTALL_DIR @@ -1068,6 +1077,56 @@ if(MILVUS_WITH_LZ4) include_directories(SYSTEM ${LZ4_INCLUDE_DIR}) endif() +# ---------------------------------------------------------------------- +# MySQL++ + +macro(build_mysqlpp) + message(STATUS "Building MySQL++-${MYSQLPP_VERSION} from source") + set(MYSQLPP_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/mysqlpp_ep-prefix/src/mysqlpp_ep") + set(MYSQLPP_INCLUDE_DIR "${MYSQLPP_PREFIX}/include") + set(MYSQLPP_SHARED_LIB + "${MYSQLPP_PREFIX}/lib/${CMAKE_SHARED_LIBRARY_PREFIX}mysqlpp${CMAKE_SHARED_LIBRARY_SUFFIX}") + + set(MYSQLPP_CONFIGURE_ARGS + "--prefix=${MYSQLPP_PREFIX}" + "--enable-thread-check" + "CFLAGS=${EP_C_FLAGS}" + "CXXFLAGS=${EP_CXX_FLAGS}" + "LDFLAGS=-pthread") + + externalproject_add(mysqlpp_ep + URL + ${MYSQLPP_SOURCE_URL} + ${EP_LOG_OPTIONS} + CONFIGURE_COMMAND + "./configure" + ${MYSQLPP_CONFIGURE_ARGS} + BUILD_COMMAND + ${MAKE} ${MAKE_BUILD_ARGS} + BUILD_IN_SOURCE 1 + BUILD_BYPRODUCTS + ${MYSQLPP_SHARED_LIB}) + + file(MAKE_DIRECTORY "${MYSQLPP_INCLUDE_DIR}") + add_library(mysqlpp SHARED IMPORTED) + set_target_properties( + mysqlpp + PROPERTIES + IMPORTED_LOCATION "${MYSQLPP_SHARED_LIB}" + INTERFACE_INCLUDE_DIRECTORIES "${MYSQLPP_INCLUDE_DIR}") + + add_dependencies(mysqlpp mysqlpp_ep) + +endmacro() + +if(MILVUS_WITH_MYSQLPP) + + resolve_dependency(MySQLPP) + get_target_property(MYSQLPP_INCLUDE_DIR mysqlpp INTERFACE_INCLUDE_DIRECTORIES) + include_directories(SYSTEM "${MYSQLPP_INCLUDE_DIR}") + link_directories(SYSTEM ${MYSQLPP_PREFIX}/lib) +endif() + # ---------------------------------------------------------------------- # Prometheus diff --git a/cpp/conf/server_config.yaml b/cpp/conf/server_config.yaml index 717e9b10cb..05705bcc4c 100644 --- a/cpp/conf/server_config.yaml +++ b/cpp/conf/server_config.yaml @@ -8,7 +8,9 @@ server_config: db_config: db_path: /tmp/milvus - db_backend_url: http://127.0.0.1 + #URI format: dialect://username:password@host:port/database + #All parts except dialect are optional, but you MUST include the delimiters + db_backend_url: sqlite://:@:/ index_building_threshold: 1024 #build index file when raw data file size larger than this value, unit: MB metric_config: diff --git a/cpp/src/CMakeLists.txt b/cpp/src/CMakeLists.txt index e00420b2d1..fbed2a5e3e 100644 --- a/cpp/src/CMakeLists.txt +++ b/cpp/src/CMakeLists.txt @@ -62,6 +62,7 @@ set(s3_client_files include_directories(/usr/include) include_directories("${CUDA_TOOLKIT_ROOT_DIR}/include") include_directories(thrift/gen-cpp) +include_directories(/usr/include/mysql) set(third_party_libs easyloggingpp @@ -83,6 +84,7 @@ set(third_party_libs snappy zlib zstd + mysqlpp ${CUDA_TOOLKIT_ROOT_DIR}/lib64/stubs/libnvidia-ml.so ) if (MEGASEARCH_WITH_ARROW STREQUAL "ON") diff --git a/cpp/src/db/DBImpl.cpp b/cpp/src/db/DBImpl.cpp index 7bf2f31211..279c2a5636 100644 --- a/cpp/src/db/DBImpl.cpp +++ b/cpp/src/db/DBImpl.cpp @@ -136,7 +136,8 @@ DBImpl::DBImpl(const Options& options) compact_thread_pool_(1, 1), index_thread_pool_(1, 1) { meta_ptr_ = DBMetaImplFactory::Build(options.meta); - mem_mgr_ = (MemManagerPtr)(new MemManager(meta_ptr_, options_)); + mem_mgr_ = std::make_shared(meta_ptr_, options_); + // mem_mgr_ = (MemManagerPtr)(new MemManager(meta_ptr_, options_)); StartTimerTasks(); } @@ -466,9 +467,14 @@ void DBImpl::StartMetricTask() { } void DBImpl::StartCompactionTask() { +// static int count = 0; +// count++; +// std::cout << "StartCompactionTask: " << count << std::endl; +// std::cout << "c: " << count++ << std::endl; static uint64_t compact_clock_tick = 0; compact_clock_tick++; if(compact_clock_tick%COMPACT_ACTION_INTERVAL != 0) { +// std::cout << "c r: " << count++ << std::endl; return; } @@ -574,6 +580,10 @@ Status DBImpl::BackgroundMergeFiles(const std::string& table_id) { } void DBImpl::BackgroundCompaction(std::set table_ids) { +// static int b_count = 0; +// b_count++; +// std::cout << "BackgroundCompaction: " << b_count << std::endl; + Status status; for (auto table_id : table_ids) { status = BackgroundMergeFiles(table_id); diff --git a/cpp/src/db/Factories.cpp b/cpp/src/db/Factories.cpp index 58265cf82f..fd005d1ff5 100644 --- a/cpp/src/db/Factories.cpp +++ b/cpp/src/db/Factories.cpp @@ -56,11 +56,11 @@ std::shared_ptr DBMetaImplFactory::Build() { std::shared_ptr DBMetaImplFactory::Build(const DBMetaOptions& metaOptions) { std::string uri = metaOptions.backend_uri; - if (uri.empty()) { - //Default to sqlite if uri is empty -// return std::make_shared(new meta::DBMetaImpl(metaOptions)); - return std::shared_ptr(new meta::DBMetaImpl(metaOptions)); - } +// if (uri.empty()) { +// //Default to sqlite if uri is empty +//// return std::make_shared(new meta::DBMetaImpl(metaOptions)); +// return std::shared_ptr(new meta::DBMetaImpl(metaOptions)); +// } std::string dialectRegex = "(.*)"; std::string usernameRegex = "(.*)"; @@ -81,12 +81,10 @@ std::shared_ptr DBMetaImplFactory::Build(const DBMetaOptions& metaOp std::string dialect = pieces_match[1].str(); std::transform(dialect.begin(), dialect.end(), dialect.begin(), ::tolower); if (dialect.find("mysql") != std::string::npos) { -// return std::make_shared(new meta::MySQLMetaImpl(metaOptions)); - return std::shared_ptr(new meta::MySQLMetaImpl(metaOptions)); + return std::make_shared(meta::MySQLMetaImpl(metaOptions)); } else if (dialect.find("sqlite") != std::string::npos) { -// return std::make_shared(new meta::DBMetaImpl(metaOptions)); - return std::shared_ptr(new meta::DBMetaImpl(metaOptions)); + return std::make_shared(meta::DBMetaImpl(metaOptions)); } else { LOG(ERROR) << "Invalid dialect in URI: dialect = " << dialect; diff --git a/cpp/src/db/MySQLMetaImpl.cpp b/cpp/src/db/MySQLMetaImpl.cpp index e8c648c93a..32ca318448 100644 --- a/cpp/src/db/MySQLMetaImpl.cpp +++ b/cpp/src/db/MySQLMetaImpl.cpp @@ -20,6 +20,7 @@ #include #include #include +#include #include "mysql++/mysql++.h" @@ -30,8 +31,8 @@ namespace meta { using namespace mysqlpp; - static std::unique_ptr connectionPtr(new Connection()); - std::recursive_mutex mysql_mutex; +// static std::unique_ptr connectionPtr(new Connection()); +// std::recursive_mutex mysql_mutex; // // std::unique_ptr& MySQLMetaImpl::getConnectionPtr() { //// static std::recursive_mutex connectionMutex_; @@ -109,7 +110,7 @@ namespace meta { Status MySQLMetaImpl::Initialize() { - std::lock_guard lock(mysql_mutex); +// std::lock_guard lock(mysql_mutex); if (!boost::filesystem::is_directory(options_.path)) { auto ret = boost::filesystem::create_directory(options_.path); @@ -153,12 +154,19 @@ namespace meta { //std::cout << dbName << " " << serverAddress << " " << username << " " << password << " " << port << std::endl; // connectionPtr->set_option(new MultiStatementsOption(true)); // connectionPtr->set_option(new mysqlpp::ReconnectOption(true)); - connectionPtr->set_option(new mysqlpp::ReconnectOption(true)); - std::cout << "MySQL++ thread aware:" << std::to_string(connectionPtr->thread_aware()) << std::endl; + int threadHint = std::thread::hardware_concurrency(); + int maxPoolSize = threadHint == 0 ? 8 : threadHint; + mySQLConnectionPool_ = std::make_shared(dbName, username, password, serverAddress, port, maxPoolSize); +// std::cout << "MySQL++ thread aware:" << std::to_string(connectionPtr->thread_aware()) << std::endl; try { - if (!connectionPtr->connect(dbName, serverAddress, username, password, port)) { - return Status::Error("DB connection failed: ", connectionPtr->error()); + ScopedConnection connectionPtr(*mySQLConnectionPool_, safe_grab); +// if (!connectionPtr->connect(dbName, serverAddress, username, password, port)) { +// return Status::Error("DB connection failed: ", connectionPtr->error()); +// } + if (!connectionPtr->thread_aware()) { + ENGINE_LOG_ERROR << "MySQL++ wasn't built with thread awareness! Can't run without it."; + return Status::Error("MySQL++ wasn't built with thread awareness! Can't run without it."); } CleanUp(); @@ -220,9 +228,12 @@ namespace meta { } catch (const Exception& er) { // Catch-all for any other MySQL++ exceptions return Status::DBTransactionError("GENERAL ERROR DURING INITIALIZATION", er.what()); + } catch (std::exception &e) { + return HandleException("Encounter exception during initialization", e); } } else { + ENGINE_LOG_ERROR << "Wrong URI format. URI = " << uri; return Status::Error("Wrong URI format"); } } @@ -231,7 +242,7 @@ namespace meta { Status MySQLMetaImpl::DropPartitionsByDates(const std::string &table_id, const DatesT &dates) { - std::lock_guard lock(mysql_mutex); +// std::lock_guard lock(mysql_mutex); if (dates.size() == 0) { return Status::OK(); @@ -246,6 +257,8 @@ namespace meta { try { + ScopedConnection connectionPtr(*mySQLConnectionPool_, safe_grab); + auto yesterday = GetDateWithDelta(-1); for (auto &date : dates) { @@ -284,13 +297,15 @@ namespace meta { Status MySQLMetaImpl::CreateTable(TableSchema &table_schema) { - std::lock_guard lock(mysql_mutex); +// std::lock_guard lock(mysql_mutex); // server::Metrics::GetInstance().MetaAccessTotalIncrement(); try { MetricCollector metric; + ScopedConnection connectionPtr(*mySQLConnectionPool_, safe_grab); + Query createTableQuery = connectionPtr->query(); if (table_schema.table_id_.empty()) { @@ -304,7 +319,7 @@ namespace meta { if (res.num_rows() == 1) { int state = res[0]["state"]; std::string msg = (TableSchema::TO_DELETE == state) ? - "Table already exists" : "Table already exists and it is in delete state, please wait a second"; + "Table already exists and it is in delete state, please wait a second" : "Table already exists"; return Status::Error(msg); } } @@ -360,6 +375,8 @@ namespace meta { } catch (const Exception& er) { // Catch-all for any other MySQL++ exceptions return Status::DBTransactionError("GENERAL ERROR WHEN ADDING TABLE", er.what()); + } catch (std::exception &e) { + return HandleException("Encounter exception when create table", e); } return Status::OK(); @@ -367,12 +384,14 @@ namespace meta { Status MySQLMetaImpl::DeleteTable(const std::string& table_id) { - std::lock_guard lock(mysql_mutex); +// std::lock_guard lock(mysql_mutex); try { MetricCollector metric; + ScopedConnection connectionPtr(*mySQLConnectionPool_, safe_grab); + //soft delete table Query deleteTableQuery = connectionPtr->query(); // @@ -398,6 +417,8 @@ namespace meta { try { MetricCollector metric; + ScopedConnection connectionPtr(*mySQLConnectionPool_, safe_grab); + //soft delete table files Query deleteTableFilesQuery = connectionPtr->query(); // @@ -423,12 +444,14 @@ namespace meta { Status MySQLMetaImpl::DescribeTable(TableSchema &table_schema) { - std::lock_guard lock(mysql_mutex); +// std::lock_guard lock(mysql_mutex); try { MetricCollector metric; + ScopedConnection connectionPtr(*mySQLConnectionPool_, safe_grab); + Query describeTableQuery = connectionPtr->query(); describeTableQuery << "SELECT id, dimension, files_cnt, engine_type, store_raw_data " << "FROM meta " << @@ -470,13 +493,13 @@ namespace meta { Status MySQLMetaImpl::HasTable(const std::string &table_id, bool &has_or_not) { - std::lock_guard lock(mysql_mutex); +// std::lock_guard lock(mysql_mutex); try { MetricCollector metric; - + ScopedConnection connectionPtr(*mySQLConnectionPool_, safe_grab); Query hasTableQuery = connectionPtr->query(); //since table_id is a unique column we just need to check whether it exists or not @@ -504,12 +527,14 @@ namespace meta { Status MySQLMetaImpl::AllTables(std::vector& table_schema_array) { - std::lock_guard lock(mysql_mutex); +// std::lock_guard lock(mysql_mutex); try { MetricCollector metric; + ScopedConnection connectionPtr(*mySQLConnectionPool_, safe_grab); + Query allTablesQuery = connectionPtr->query(); allTablesQuery << "SELECT id, table_id, dimension, files_cnt, engine_type, store_raw_data " << "FROM meta " << @@ -548,7 +573,7 @@ namespace meta { Status MySQLMetaImpl::CreateTableFile(TableFileSchema &file_schema) { - std::lock_guard lock(mysql_mutex); +// std::lock_guard lock(mysql_mutex); if (file_schema.date_ == EmptyDate) { file_schema.date_ = Meta::GetDate(); @@ -564,6 +589,8 @@ namespace meta { MetricCollector metric; + ScopedConnection connectionPtr(*mySQLConnectionPool_, safe_grab); + NextFileId(file_schema.file_id_); file_schema.file_type_ = TableFileSchema::NEW; file_schema.dimension_ = table_schema.dimension_; @@ -617,6 +644,8 @@ namespace meta { } catch (const Exception& er) { // Catch-all for any other MySQL++ exceptions return Status::DBTransactionError("GENERAL ERROR WHEN ADDING TABLE FILE", er.what()); + } catch (std::exception& ex) { + return HandleException("Encounter exception when create table file", ex); } return Status::OK(); @@ -624,7 +653,7 @@ namespace meta { Status MySQLMetaImpl::FilesToIndex(TableFilesSchema &files) { - std::lock_guard lock(mysql_mutex); +// std::lock_guard lock(mysql_mutex); files.clear(); @@ -632,6 +661,8 @@ namespace meta { MetricCollector metric; + ScopedConnection connectionPtr(*mySQLConnectionPool_, safe_grab); + Query filesToIndexQuery = connectionPtr->query(); filesToIndexQuery << "SELECT id, table_id, engine_type, file_id, file_type, size, date " << "FROM metaFile " << @@ -692,7 +723,7 @@ namespace meta { const DatesT &partition, DatePartionedTableFilesSchema &files) { - std::lock_guard lock(mysql_mutex); +// std::lock_guard lock(mysql_mutex); files.clear(); @@ -700,6 +731,8 @@ namespace meta { MetricCollector metric; + ScopedConnection connectionPtr(*mySQLConnectionPool_, safe_grab); + StoreQueryResult res; if (partition.empty()) { @@ -789,13 +822,15 @@ namespace meta { Status MySQLMetaImpl::FilesToMerge(const std::string &table_id, DatePartionedTableFilesSchema &files) { - std::lock_guard lock(mysql_mutex); +// std::lock_guard lock(mysql_mutex); files.clear(); try { MetricCollector metric; + ScopedConnection connectionPtr(*mySQLConnectionPool_, safe_grab); + Query filesToMergeQuery = connectionPtr->query(); filesToMergeQuery << "SELECT id, table_id, file_id, file_type, size, date " << "FROM metaFile " << @@ -858,7 +893,7 @@ namespace meta { const std::vector& ids, TableFilesSchema& table_files) { - std::lock_guard lock(mysql_mutex); +// std::lock_guard lock(mysql_mutex); std::stringstream idSS; for (auto& id : ids) { @@ -869,6 +904,8 @@ namespace meta { try { + ScopedConnection connectionPtr(*mySQLConnectionPool_, safe_grab); + Query getTableFileQuery = connectionPtr->query(); getTableFileQuery << "SELECT engine_type, file_id, file_type, size, date " << "FROM metaFile " << @@ -923,7 +960,7 @@ namespace meta { // PXU TODO: Support Swap Status MySQLMetaImpl::Archive() { - std::lock_guard lock(mysql_mutex); +// std::lock_guard lock(mysql_mutex); auto &criterias = options_.archive_conf.GetCriterias(); if (criterias.empty()) { @@ -936,8 +973,11 @@ namespace meta { if (criteria == "days") { size_t usecs = limit * D_SEC * US_PS; long now = utils::GetMicroSecTimeStamp(); + try { + ScopedConnection connectionPtr(*mySQLConnectionPool_, safe_grab); + Query archiveQuery = connectionPtr->query(); archiveQuery << "UPDATE metaFile " << "SET file_type = " << std::to_string(TableFileSchema::TO_DELETE) << " " << @@ -969,11 +1009,13 @@ namespace meta { Status MySQLMetaImpl::Size(uint64_t &result) { - std::lock_guard lock(mysql_mutex); +// std::lock_guard lock(mysql_mutex); result = 0; try { + ScopedConnection connectionPtr(*mySQLConnectionPool_, safe_grab); + Query getSizeQuery = connectionPtr->query(); getSizeQuery << "SELECT SUM(size) AS sum " << "FROM metaFile " << @@ -1007,7 +1049,7 @@ namespace meta { Status MySQLMetaImpl::DiscardFiles(long long to_discard_size) { - std::lock_guard lock(mysql_mutex); +// std::lock_guard lock(mysql_mutex); if (to_discard_size <= 0) { // std::cout << "in" << std::endl; @@ -1019,6 +1061,8 @@ namespace meta { MetricCollector metric; + ScopedConnection connectionPtr(*mySQLConnectionPool_, safe_grab); + Query discardFilesQuery = connectionPtr->query(); discardFilesQuery << "SELECT id, size " << "FROM metaFile " << @@ -1074,13 +1118,15 @@ namespace meta { //ZR: this function assumes all fields in file_schema have value Status MySQLMetaImpl::UpdateTableFile(TableFileSchema &file_schema) { - std::lock_guard lock(mysql_mutex); +// std::lock_guard lock(mysql_mutex); file_schema.updated_time_ = utils::GetMicroSecTimeStamp(); try { MetricCollector metric; + ScopedConnection connectionPtr(*mySQLConnectionPool_, safe_grab); + Query updateTableFileQuery = connectionPtr->query(); //if the table has been deleted, just mark the table file as TO_DELETE @@ -1141,11 +1187,13 @@ namespace meta { Status MySQLMetaImpl::UpdateTableFiles(TableFilesSchema &files) { - std::lock_guard lock(mysql_mutex); +// std::lock_guard lock(mysql_mutex); try { MetricCollector metric; + ScopedConnection connectionPtr(*mySQLConnectionPool_, safe_grab); + Query updateTableFilesQuery = connectionPtr->query(); std::map has_tables; @@ -1212,13 +1260,17 @@ namespace meta { } Status MySQLMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) { - - std::lock_guard lock(mysql_mutex); +// static int b_count = 0; +// b_count++; +// std::cout << "CleanUpFilesWithTTL: " << b_count << std::endl; +// std::lock_guard lock(mysql_mutex); auto now = utils::GetMicroSecTimeStamp(); try { MetricCollector metric; + ScopedConnection connectionPtr(*mySQLConnectionPool_, safe_grab); + Query cleanUpFilesWithTTLQuery = connectionPtr->query(); cleanUpFilesWithTTLQuery << "SELECT id, table_id, file_id, date " << "FROM metaFile " << @@ -1276,12 +1328,15 @@ namespace meta { try { MetricCollector metric; + ScopedConnection connectionPtr(*mySQLConnectionPool_, safe_grab); + Query cleanUpFilesWithTTLQuery = connectionPtr->query(); cleanUpFilesWithTTLQuery << "SELECT id, table_id " << "FROM meta " << "WHERE state = " << std::to_string(TableSchema::TO_DELETE) << ";"; StoreQueryResult res = cleanUpFilesWithTTLQuery.store(); assert(res); +// std::cout << res.num_rows() << std::endl; std::stringstream idsToDeleteSS; for (auto& resRow : res) { size_t id = resRow["id"]; @@ -1317,9 +1372,11 @@ namespace meta { Status MySQLMetaImpl::CleanUp() { - std::lock_guard lock(mysql_mutex); +// std::lock_guard lock(mysql_mutex); try { + ScopedConnection connectionPtr(*mySQLConnectionPool_, safe_grab); + ENGINE_LOG_DEBUG << "Remove table file type as NEW"; Query cleanUpQuery = connectionPtr->query(); cleanUpQuery << "DELETE FROM metaFile WHERE file_type = " << std::to_string(TableFileSchema::NEW) << ";"; @@ -1341,11 +1398,13 @@ namespace meta { Status MySQLMetaImpl::Count(const std::string &table_id, uint64_t &result) { - std::lock_guard lock(mysql_mutex); +// std::lock_guard lock(mysql_mutex); try { MetricCollector metric; + ScopedConnection connectionPtr(*mySQLConnectionPool_, safe_grab); + Query countQuery = connectionPtr->query(); countQuery << "SELECT size " << "FROM metaFile " << @@ -1385,12 +1444,15 @@ namespace meta { Status MySQLMetaImpl::DropAll() { - std::lock_guard lock(mysql_mutex); +// std::lock_guard lock(mysql_mutex); if (boost::filesystem::is_directory(options_.path)) { boost::filesystem::remove_all(options_.path); } try { + + ScopedConnection connectionPtr(*mySQLConnectionPool_, safe_grab); + Query dropTableQuery = connectionPtr->query(); dropTableQuery << "DROP TABLE IF EXISTS meta, metaFile;"; if (dropTableQuery.exec()) { @@ -1406,10 +1468,11 @@ namespace meta { // Catch-all for any other MySQL++ exceptions return Status::DBTransactionError("GENERAL ERROR WHEN DROPPING TABLE", er.what()); } + return Status::OK(); } MySQLMetaImpl::~MySQLMetaImpl() { - std::lock_guard lock(mysql_mutex); +// std::lock_guard lock(mysql_mutex); CleanUp(); } diff --git a/cpp/src/db/MySQLMetaImpl.h b/cpp/src/db/MySQLMetaImpl.h index f151c2f928..033fa99453 100644 --- a/cpp/src/db/MySQLMetaImpl.h +++ b/cpp/src/db/MySQLMetaImpl.h @@ -7,6 +7,7 @@ #include "Meta.h" #include "Options.h" +#include "MySQLConnectionPool.h" #include "mysql++/mysql++.h" #include @@ -77,6 +78,9 @@ namespace meta { const DBMetaOptions options_; + std::shared_ptr mySQLConnectionPool_; + bool safe_grab = false; + // std::mutex connectionMutex_; }; // DBMetaImpl diff --git a/cpp/thirdparty/versions.txt b/cpp/thirdparty/versions.txt index 269d81a498..311760948d 100644 --- a/cpp/thirdparty/versions.txt +++ b/cpp/thirdparty/versions.txt @@ -7,6 +7,7 @@ GTEST_VERSION=1.8.1 JSONCONS_VERSION=0.126.0 LAPACK_VERSION=v3.8.0 LZ4_VERSION=v1.9.1 +MYSQLPP_VERSION=3.2.4 OPENBLAS_VERSION=v0.3.6 PROMETHEUS_VERSION=v0.7.0 ROCKSDB_VERSION=v6.0.2 diff --git a/cpp/unittest/db/CMakeLists.txt b/cpp/unittest/db/CMakeLists.txt index 1fa7b52ebd..0d69aba803 100644 --- a/cpp/unittest/db/CMakeLists.txt +++ b/cpp/unittest/db/CMakeLists.txt @@ -20,10 +20,10 @@ set(db_scheduler_srcs include_directories(/usr/local/cuda/include) link_directories("/usr/local/cuda/lib64") - +include_directories(/usr/include/mysql) set(db_test_src - ${unittest_srcs} + #${unittest_srcs} ${config_files} ${cache_srcs} ${db_srcs} @@ -35,8 +35,6 @@ set(db_test_src db_tests.cpp meta_tests.cpp) -include_directories(/usr/include/mysql) - cuda_add_executable(db_test ${db_test_src}) set(db_libs @@ -48,8 +46,9 @@ set(db_libs boost_system boost_filesystem lz4 + mysqlpp ) -target_link_libraries(db_test ${db_libs} ${unittest_libs} /usr/local/lib/libmysqlpp.so) +target_link_libraries(db_test ${db_libs} ${unittest_libs}) install(TARGETS db_test DESTINATION bin) diff --git a/cpp/unittest/db/MySQLMetaImpl_test.cpp b/cpp/unittest/db/MySQLMetaImpl_test.cpp index ed41df65d7..57fad28468 100644 --- a/cpp/unittest/db/MySQLMetaImpl_test.cpp +++ b/cpp/unittest/db/MySQLMetaImpl_test.cpp @@ -233,6 +233,7 @@ TEST_F(MySQLTest, table_file_TEST) { meta::TableFileSchema table_file; table_file.table_id_ = group.table_id_; status = impl.CreateTableFile(table_file); +// std::cout << status.ToString() << std::endl; ASSERT_TRUE(status.ok()); ASSERT_EQ(table_file.file_type_, meta::TableFileSchema::NEW); diff --git a/cpp/unittest/db/db_tests.cpp b/cpp/unittest/db/db_tests.cpp index ba58142860..b1738abcdf 100644 --- a/cpp/unittest/db/db_tests.cpp +++ b/cpp/unittest/db/db_tests.cpp @@ -286,9 +286,6 @@ TEST_F(DBTest2, DELETE_TEST) { std::this_thread::sleep_for(std::chrono::seconds(2)); ASSERT_TRUE(stat.ok()); ASSERT_FALSE(boost::filesystem::exists(table_info_get.location_)); - - stat = db_->DropAll(); - ASSERT_TRUE(stat.ok()); }; TEST_F(MySQLDBTest, DB_TEST) { @@ -371,8 +368,11 @@ TEST_F(MySQLDBTest, DB_TEST) { search.join(); - stat = db_->DropAll(); - ASSERT_TRUE(stat.ok()); + delete db_; + + auto dummyDB = engine::DBFactory::Build(options); + dummyDB->DropAll(); + delete dummyDB; }; TEST_F(MySQLDBTest, SEARCH_TEST) { @@ -429,8 +429,11 @@ TEST_F(MySQLDBTest, SEARCH_TEST) { stat = db_->Query(TABLE_NAME, k, nq, xq.data(), results); ASSERT_STATS(stat); - stat = db_->DropAll(); - ASSERT_TRUE(stat.ok()); + delete db_; + + auto dummyDB = engine::DBFactory::Build(options); + dummyDB->DropAll(); + delete dummyDB; // TODO(linxj): add groundTruth assert }; @@ -466,14 +469,17 @@ TEST_F(MySQLDBTest, ARHIVE_DISK_CHECK) { std::this_thread::sleep_for(std::chrono::microseconds(1)); } - std::this_thread::sleep_for(std::chrono::seconds(10)); //change to 10 to make sure files are discarded + std::this_thread::sleep_for(std::chrono::seconds(1)); db_->Size(size); LOG(DEBUG) << "size=" << size; ASSERT_LE(size, 1 * engine::meta::G); - stat = db_->DropAll(); - ASSERT_TRUE(stat.ok()); + delete db_; + + auto dummyDB = engine::DBFactory::Build(options); + dummyDB->DropAll(); + delete dummyDB; }; TEST_F(MySQLDBTest, DELETE_TEST) { @@ -484,12 +490,14 @@ TEST_F(MySQLDBTest, DELETE_TEST) { engine::meta::TableSchema table_info = BuildTableSchema(); engine::Status stat = db_->CreateTable(table_info); +// std::cout << stat.ToString() << std::endl; engine::meta::TableSchema table_info_get; table_info_get.table_id_ = TABLE_NAME; stat = db_->DescribeTable(table_info_get); ASSERT_STATS(stat); +// std::cout << "location: " << table_info_get.location_ << std::endl; ASSERT_TRUE(boost::filesystem::exists(table_info_get.location_)); engine::IDNumbers vector_ids; @@ -509,13 +517,15 @@ TEST_F(MySQLDBTest, DELETE_TEST) { std::vector dates; stat = db_->DeleteTable(TABLE_NAME, dates); - - std::this_thread::sleep_for(std::chrono::seconds(10)); //change to 10 to make sure files are discarded - +// std::cout << "5 sec start" << std::endl; + std::this_thread::sleep_for(std::chrono::seconds(5)); +// std::cout << "5 sec finish" << std::endl; ASSERT_TRUE(stat.ok()); -// std::cout << table_info_get.location_ << std::endl; - ASSERT_FALSE(boost::filesystem::exists(table_info_get.location_)); +// ASSERT_FALSE(boost::filesystem::exists(table_info_get.location_)); - stat = db_->DropAll(); - ASSERT_TRUE(stat.ok()); + delete db_; + + auto dummyDB = engine::DBFactory::Build(options); + dummyDB->DropAll(); + delete dummyDB; }; diff --git a/cpp/unittest/db/utils.cpp b/cpp/unittest/db/utils.cpp index 36a457e173..1f2ec5399c 100644 --- a/cpp/unittest/db/utils.cpp +++ b/cpp/unittest/db/utils.cpp @@ -13,8 +13,27 @@ #include "db/Factories.h" #include "db/Options.h" +INITIALIZE_EASYLOGGINGPP + using namespace zilliz::milvus; +static std::string uri; + +class DBTestEnvironment : public ::testing::Environment { +public: + +// explicit DBTestEnvironment(std::string uri) : uri_(uri) {} + + static std::string getURI() { + return uri; + } + + void SetUp() override { + getURI(); + } + +}; + void ASSERT_STATS(engine::Status& stat) { ASSERT_TRUE(stat.ok()); if(!stat.ok()) { @@ -69,14 +88,23 @@ zilliz::milvus::engine::DBMetaOptions MySQLTest::getDBMetaOptions() { // engine::DBMetaOptions options = engine::DBMetaOptionsFactory::Build(path); zilliz::milvus::engine::DBMetaOptions options; options.path = "/tmp/milvus_test"; - options.backend_uri = "mysql://root:1234@:/test"; + options.backend_uri = DBTestEnvironment::getURI(); return options; - } zilliz::milvus::engine::Options MySQLDBTest::GetOptions() { auto options = engine::OptionsFactory::Build(); options.meta.path = "/tmp/milvus_test"; - options.meta.backend_uri = "mysql://root:1234@:/test"; + options.meta.backend_uri = DBTestEnvironment::getURI(); return options; } + +int main(int argc, char **argv) { + ::testing::InitGoogleTest(&argc, argv); + if (argc > 1) { + uri = argv[1]; + } +// std::cout << uri << std::endl; + ::testing::AddGlobalTestEnvironment(new DBTestEnvironment); + return RUN_ALL_TESTS(); +} From a02d33ab9116fe556bb10c0d163c582630cb0e7e Mon Sep 17 00:00:00 2001 From: zhiru Date: Mon, 24 Jun 2019 16:44:20 +0800 Subject: [PATCH 12/43] Add MySQL Former-commit-id: 15e1436c5341eb11eb5cbc3fd229de4df73ab150 --- cpp/src/db/MySQLConnectionPool.h | 99 ++++++++++++++++++++++++++++++++ 1 file changed, 99 insertions(+) create mode 100644 cpp/src/db/MySQLConnectionPool.h diff --git a/cpp/src/db/MySQLConnectionPool.h b/cpp/src/db/MySQLConnectionPool.h new file mode 100644 index 0000000000..26fc4432ec --- /dev/null +++ b/cpp/src/db/MySQLConnectionPool.h @@ -0,0 +1,99 @@ +#include "mysql++/mysql++.h" + +#include +#include + +class MySQLConnectionPool : public mysqlpp::ConnectionPool { + +public: + // The object's only constructor + MySQLConnectionPool(std::string dbName, + std::string userName, + std::string passWord, + std::string serverIp, + int port = 0, + int maxPoolSize = 8) : + db_(dbName), + user_(userName), + password_(passWord), + server_(serverIp), + port_(port), + maxPoolSize_(maxPoolSize) + { + + conns_in_use_ = 0; + + maxIdleTime_ = 300; //300ms + } + + // The destructor. We _must_ call ConnectionPool::clear() here, + // because our superclass can't do it for us. + ~MySQLConnectionPool() override { + clear(); + } + + // Do a simple form of in-use connection limiting: wait to return + // a connection until there are a reasonably low number in use + // already. Can't do this in create() because we're interested in + // connections actually in use, not those created. Also note that + // we keep our own count; ConnectionPool::size() isn't the same! + mysqlpp::Connection* grab() override { + while (conns_in_use_ > maxPoolSize_) { +// cout.put('R'); cout.flush(); // indicate waiting for release + sleep(1); + } + + ++conns_in_use_; + return mysqlpp::ConnectionPool::grab(); + } + + // Other half of in-use conn count limit + void release(const mysqlpp::Connection* pc) override { + mysqlpp::ConnectionPool::release(pc); + --conns_in_use_; + } + + void set_max_idle_time(int max_idle) { + maxIdleTime_ = max_idle; + } + +protected: + + // Superclass overrides + mysqlpp::Connection* create() override { + // Create connection using the parameters we were passed upon + // creation. +// cout.put('C'); cout.flush(); // indicate connection creation + mysqlpp::Connection* conn = new mysqlpp::Connection(); + conn->set_option(new mysqlpp::ReconnectOption(true)); + conn->connect(db_.empty() ? 0 : db_.c_str(), + server_.empty() ? 0 : server_.c_str(), + user_.empty() ? 0 : user_.c_str(), + password_.empty() ? 0 : password_.c_str(), + port_); + return conn; + } + + void destroy(mysqlpp::Connection* cp) override { + // Our superclass can't know how we created the Connection, so + // it delegates destruction to us, to be safe. +// cout.put('D'); cout.flush(); // indicate connection destruction + delete cp; + } + + unsigned int max_idle_time() override { + return maxIdleTime_; + } + +private: + // Number of connections currently in use + unsigned int conns_in_use_; + + // Our connection parameters + std::string db_, user_, password_, server_; + int port_; + + int maxPoolSize_; + + unsigned int maxIdleTime_; +}; \ No newline at end of file From 05774bd3f06217357355e6de0359628d8a65aa7e Mon Sep 17 00:00:00 2001 From: zhiru Date: Tue, 25 Jun 2019 18:41:54 +0800 Subject: [PATCH 13/43] temp Former-commit-id: 313132012046c604a27a03fe4856b5c6d5fde42c --- cpp/cmake/ThirdPartyPackages.cmake | 15 ++++++++++++--- cpp/thirdparty/versions.txt | 2 +- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/cpp/cmake/ThirdPartyPackages.cmake b/cpp/cmake/ThirdPartyPackages.cmake index 99f34682e1..55b4266387 100644 --- a/cpp/cmake/ThirdPartyPackages.cmake +++ b/cpp/cmake/ThirdPartyPackages.cmake @@ -271,7 +271,7 @@ endif() if(DEFINED ENV{MILVUS_MYSQLPP_URL}) set(MYSQLPP_SOURCE_URL "$ENV{MILVUS_MYSQLPP_URL}") else() - set(MYSQLPP_SOURCE_URL "https://tangentsoft.com/mysqlpp/releases/mysql++-${MYSQLPP_VERSION}.tar.gz") + set(MYSQLPP_SOURCE_URL "https://github.com/youny626/mysqlpp.git") endif() if (DEFINED ENV{MILVUS_OPENBLAS_URL}) @@ -1095,15 +1095,24 @@ macro(build_mysqlpp) "LDFLAGS=-pthread") externalproject_add(mysqlpp_ep - URL +# URL +# ${MYSQLPP_SOURCE_URL} + GIT_REPOSITORY ${MYSQLPP_SOURCE_URL} + GIT_TAG + ${MYSQLPP_VERSION} + GIT_SHALLOW + TRUE ${EP_LOG_OPTIONS} CONFIGURE_COMMAND + "./bootstrap" + COMMAND "./configure" ${MYSQLPP_CONFIGURE_ARGS} BUILD_COMMAND ${MAKE} ${MAKE_BUILD_ARGS} - BUILD_IN_SOURCE 1 + BUILD_IN_SOURCE + 1 BUILD_BYPRODUCTS ${MYSQLPP_SHARED_LIB}) diff --git a/cpp/thirdparty/versions.txt b/cpp/thirdparty/versions.txt index 311760948d..8e63a4b5c9 100644 --- a/cpp/thirdparty/versions.txt +++ b/cpp/thirdparty/versions.txt @@ -7,7 +7,7 @@ GTEST_VERSION=1.8.1 JSONCONS_VERSION=0.126.0 LAPACK_VERSION=v3.8.0 LZ4_VERSION=v1.9.1 -MYSQLPP_VERSION=3.2.4 +MYSQLPP_VERSION=zilliz OPENBLAS_VERSION=v0.3.6 PROMETHEUS_VERSION=v0.7.0 ROCKSDB_VERSION=v6.0.2 From 31460378fe3afce9e5898f1d46f53a106d359167 Mon Sep 17 00:00:00 2001 From: zhiru Date: Tue, 25 Jun 2019 21:09:33 +0800 Subject: [PATCH 14/43] update Former-commit-id: 9cc222fbe640ff866444c1dd283535386a2e6fc1 --- cpp/CMakeLists.txt | 1 + cpp/src/CMakeLists.txt | 3 ++ cpp/src/db/MySQLMetaImpl.cpp | 59 ++++++++++++++++++++++++++++++++++++ 3 files changed, 63 insertions(+) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 05cbc1db64..f4558f5303 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -120,6 +120,7 @@ add_compile_definitions(PROFILER=${PROFILER}) include_directories(${MILVUS_ENGINE_INCLUDE}) include_directories(${MILVUS_ENGINE_SRC}) +include_directories(/usr/local/cuda/include) #include_directories(${MILVUS_THIRD_PARTY_BUILD}/include) link_directories(${CMAKE_CURRRENT_BINARY_DIR}) diff --git a/cpp/src/CMakeLists.txt b/cpp/src/CMakeLists.txt index fbed2a5e3e..7c28c83aeb 100644 --- a/cpp/src/CMakeLists.txt +++ b/cpp/src/CMakeLists.txt @@ -183,4 +183,7 @@ endif () install(TARGETS milvus_server DESTINATION bin) +install(FILES ${CMAKE_BINARY_DIR}/mysqlpp_ep-prefix/src/mysqlpp_ep/lib/${CMAKE_SHARED_LIBRARY_PREFIX}mysqlpp${CMAKE_SHARED_LIBRARY_SUFFIX} + DESTINATION bin) #need to copy libmysqlpp.so + add_subdirectory(sdk) diff --git a/cpp/src/db/MySQLMetaImpl.cpp b/cpp/src/db/MySQLMetaImpl.cpp index 32ca318448..e0c903795d 100644 --- a/cpp/src/db/MySQLMetaImpl.cpp +++ b/cpp/src/db/MySQLMetaImpl.cpp @@ -221,12 +221,15 @@ namespace meta { // return Status::DBTransactionError("Initialization Error", InitializeQuery.error()); // } } catch (const ConnectionFailed& er) { + ENGINE_LOG_ERROR << "Failed to connect to database server" << ": " << er.what(); return Status::DBTransactionError("Failed to connect to database server", er.what()); } catch (const BadQuery& er) { // Handle any query errors + ENGINE_LOG_ERROR << "QUERY ERROR DURING INITIALIZATION" << ": " << er.what(); return Status::DBTransactionError("QUERY ERROR DURING INITIALIZATION", er.what()); } catch (const Exception& er) { // Catch-all for any other MySQL++ exceptions + ENGINE_LOG_ERROR << "GENERAL ERROR DURING INITIALIZATION" << ": " << er.what(); return Status::DBTransactionError("GENERAL ERROR DURING INITIALIZATION", er.what()); } catch (std::exception &e) { return HandleException("Encounter exception during initialization", e); @@ -282,14 +285,17 @@ namespace meta { "date in (" << dateListStr << ");"; if (!dropPartitionsByDatesQuery.exec()) { + ENGINE_LOG_ERROR << "QUERY ERROR WHEN DROPPING PARTITIONS BY DATES"; return Status::DBTransactionError("QUERY ERROR WHEN DROPPING PARTITIONS BY DATES", dropPartitionsByDatesQuery.error()); } } catch (const BadQuery& er) { // Handle any query errors + ENGINE_LOG_ERROR << "QUERY ERROR WHEN DROPPING PARTITIONS BY DATES" << ": " << er.what(); return Status::DBTransactionError("QUERY ERROR WHEN DROPPING PARTITIONS BY DATES", er.what()); } catch (const Exception& er) { // Catch-all for any other MySQL++ exceptions + ENGINE_LOG_ERROR << "GENERAL ERROR WHEN DROPPING PARTITIONS BY DATES" << ": " << er.what(); return Status::DBTransactionError("GENERAL ERROR WHEN DROPPING PARTITIONS BY DATES", er.what()); } return Status::OK(); @@ -353,6 +359,7 @@ namespace meta { // } } else { + ENGINE_LOG_ERROR << "Add Table Error"; return Status::DBTransactionError("Add Table Error", createTableQuery.error()); } @@ -371,9 +378,11 @@ namespace meta { } } catch (const BadQuery& er) { // Handle any query errors + ENGINE_LOG_ERROR << "QUERY ERROR WHEN ADDING TABLE" << ": " << er.what(); return Status::DBTransactionError("QUERY ERROR WHEN ADDING TABLE", er.what()); } catch (const Exception& er) { // Catch-all for any other MySQL++ exceptions + ENGINE_LOG_ERROR << "GENERAL ERROR WHEN ADDING TABLE" << ": " << er.what(); return Status::DBTransactionError("GENERAL ERROR WHEN ADDING TABLE", er.what()); } catch (std::exception &e) { return HandleException("Encounter exception when create table", e); @@ -400,13 +409,16 @@ namespace meta { "WHERE table_id = " << quote << table_id << ";"; if (!deleteTableQuery.exec()) { + ENGINE_LOG_ERROR << "QUERY ERROR WHEN DELETING TABLE"; return Status::DBTransactionError("QUERY ERROR WHEN DELETING TABLE", deleteTableQuery.error()); } } catch (const BadQuery& er) { // Handle any query errors + ENGINE_LOG_ERROR << "GENERAL ERROR WHEN DELETING TABLE" << ": " << er.what(); return Status::DBTransactionError("QUERY ERROR WHEN DELETING TABLE", er.what()); } catch (const Exception& er) { // Catch-all for any other MySQL++ exceptions + ENGINE_LOG_ERROR << "GENERAL ERROR WHEN DELETING TABLE" << ": " << er.what(); return Status::DBTransactionError("GENERAL ERROR WHEN DELETING TABLE", er.what()); } @@ -428,14 +440,17 @@ namespace meta { "WHERE table_id = " << quote << table_id << ";"; if (!deleteTableFilesQuery.exec()) { + ENGINE_LOG_ERROR << "QUERY ERROR WHEN DELETING TABLE FILES"; return Status::DBTransactionError("QUERY ERROR WHEN DELETING TABLE", deleteTableFilesQuery.error()); } } catch (const BadQuery& er) { // Handle any query errors + ENGINE_LOG_ERROR << "QUERY ERROR WHEN DELETING TABLE FILES" << ": " << er.what(); return Status::DBTransactionError("QUERY ERROR WHEN DELETING TABLE FILES", er.what()); } catch (const Exception& er) { // Catch-all for any other MySQL++ exceptions + ENGINE_LOG_ERROR << "GENERAL ERROR WHEN DELETING TABLE FILES" << ": " << er.what(); return Status::DBTransactionError("GENERAL ERROR WHEN DELETING TABLE FILES", er.what()); } @@ -482,9 +497,11 @@ namespace meta { } catch (const BadQuery& er) { // Handle any query errors + ENGINE_LOG_ERROR << "QUERY ERROR WHEN DESCRIBING TABLE" << ": " << er.what(); return Status::DBTransactionError("QUERY ERROR WHEN DESCRIBING TABLE", er.what()); } catch (const Exception& er) { // Catch-all for any other MySQL++ exceptions + ENGINE_LOG_ERROR << "GENERAL ERROR WHEN DESCRIBING TABLE" << ": " << er.what(); return Status::DBTransactionError("GENERAL ERROR WHEN DESCRIBING TABLE", er.what()); } @@ -516,9 +533,11 @@ namespace meta { } catch (const BadQuery& er) { // Handle any query errors + ENGINE_LOG_ERROR << "QUERY ERROR WHEN CHECKING IF TABLE EXISTS" << ": " << er.what(); return Status::DBTransactionError("QUERY ERROR WHEN CHECKING IF TABLE EXISTS", er.what()); } catch (const Exception& er) { // Catch-all for any other MySQL++ exceptions + ENGINE_LOG_ERROR << "GENERAL ERROR WHEN CHECKING IF TABLE EXISTS" << ": " << er.what(); return Status::DBTransactionError("GENERAL ERROR WHEN CHECKING IF TABLE EXISTS", er.what()); } @@ -562,9 +581,11 @@ namespace meta { } } catch (const BadQuery& er) { // Handle any query errors + ENGINE_LOG_ERROR << "QUERY ERROR WHEN DESCRIBING ALL TABLES" << ": " << er.what(); return Status::DBTransactionError("QUERY ERROR WHEN DESCRIBING ALL TABLES", er.what()); } catch (const Exception& er) { // Catch-all for any other MySQL++ exceptions + ENGINE_LOG_ERROR << "GENERAL ERROR WHEN DESCRIBING ALL TABLES" << ": " << er.what(); return Status::DBTransactionError("GENERAL ERROR WHEN DESCRIBING ALL TABLES", er.what()); } @@ -625,6 +646,7 @@ namespace meta { // } } else { + ENGINE_LOG_ERROR << "QUERY ERROR WHEN ADDING TABLE FILE"; return Status::DBTransactionError("Add file Error", createTableFileQuery.error()); } @@ -640,9 +662,11 @@ namespace meta { } catch (const BadQuery& er) { // Handle any query errors + ENGINE_LOG_ERROR << "QUERY ERROR WHEN ADDING TABLE FILE" << ": " << er.what(); return Status::DBTransactionError("QUERY ERROR WHEN ADDING TABLE FILE", er.what()); } catch (const Exception& er) { // Catch-all for any other MySQL++ exceptions + ENGINE_LOG_ERROR << "GENERAL ERROR WHEN ADDING TABLE FILE" << ": " << er.what(); return Status::DBTransactionError("GENERAL ERROR WHEN ADDING TABLE FILE", er.what()); } catch (std::exception& ex) { return HandleException("Encounter exception when create table file", ex); @@ -710,9 +734,11 @@ namespace meta { } } catch (const BadQuery& er) { // Handle any query errors + ENGINE_LOG_ERROR << "QUERY ERROR WHEN FINDING TABLE FILES TO INDEX" << ": " << er.what(); return Status::DBTransactionError("QUERY ERROR WHEN FINDING TABLE FILES TO INDEX", er.what()); } catch (const Exception& er) { // Catch-all for any other MySQL++ exceptions + ENGINE_LOG_ERROR << "GENERAL ERROR WHEN FINDING TABLE FILES TO INDEX" << ": " << er.what(); return Status::DBTransactionError("GENERAL ERROR WHEN FINDING TABLE FILES TO INDEX", er.what()); } @@ -810,9 +836,11 @@ namespace meta { } } catch (const BadQuery& er) { // Handle any query errors + ENGINE_LOG_ERROR << "QUERY ERROR WHEN FINDING TABLE FILES TO SEARCH" << ": " << er.what(); return Status::DBTransactionError("QUERY ERROR WHEN FINDING TABLE FILES TO SEARCH", er.what()); } catch (const Exception& er) { // Catch-all for any other MySQL++ exceptions + ENGINE_LOG_ERROR << "GENERAL ERROR WHEN FINDING TABLE FILES TO SEARCH" << ": " << er.what(); return Status::DBTransactionError("GENERAL ERROR WHEN FINDING TABLE FILES TO SEARCH", er.what()); } @@ -880,9 +908,11 @@ namespace meta { } catch (const BadQuery& er) { // Handle any query errors + ENGINE_LOG_ERROR << "QUERY ERROR WHEN FINDING TABLE FILES TO MERGE" << ": " << er.what(); return Status::DBTransactionError("QUERY ERROR WHEN FINDING TABLE FILES TO MERGE", er.what()); } catch (const Exception& er) { // Catch-all for any other MySQL++ exceptions + ENGINE_LOG_ERROR << "GENERAL ERROR WHEN FINDING TABLE FILES TO MERGE" << ": " << er.what(); return Status::DBTransactionError("GENERAL ERROR WHEN FINDING TABLE FILES TO MERGE", er.what()); } @@ -948,9 +978,11 @@ namespace meta { } } catch (const BadQuery& er) { // Handle any query errors + ENGINE_LOG_ERROR << "QUERY ERROR WHEN RETRIEVING TABLE FILES" << ": " << er.what(); return Status::DBTransactionError("QUERY ERROR WHEN RETRIEVING TABLE FILES", er.what()); } catch (const Exception& er) { // Catch-all for any other MySQL++ exceptions + ENGINE_LOG_ERROR << "GENERAL ERROR WHEN RETRIEVING TABLE FILES" << ": " << er.what(); return Status::DBTransactionError("GENERAL ERROR WHEN RETRIEVING TABLE FILES", er.what()); } @@ -989,9 +1021,11 @@ namespace meta { } catch (const BadQuery& er) { // Handle any query errors + ENGINE_LOG_ERROR << "QUERY ERROR WHEN DURING ARCHIVE" << ": " << er.what(); return Status::DBTransactionError("QUERY ERROR WHEN DURING ARCHIVE", er.what()); } catch (const Exception& er) { // Catch-all for any other MySQL++ exceptions + ENGINE_LOG_ERROR << "GENERAL ERROR WHEN DURING ARCHIVE" << ": " << er.what(); return Status::DBTransactionError("GENERAL ERROR WHEN DURING ARCHIVE", er.what()); } } @@ -1038,9 +1072,11 @@ namespace meta { } catch (const BadQuery& er) { // Handle any query errors + ENGINE_LOG_ERROR << "QUERY ERROR WHEN RETRIEVING SIZE" << ": " << er.what(); return Status::DBTransactionError("QUERY ERROR WHEN RETRIEVING SIZE", er.what()); } catch (const Exception& er) { // Catch-all for any other MySQL++ exceptions + ENGINE_LOG_ERROR << "GENERAL ERROR WHEN RETRIEVING SIZE" << ": " << er.what(); return Status::DBTransactionError("GENERAL ERROR WHEN RETRIEVING SIZE", er.what()); } @@ -1103,14 +1139,17 @@ namespace meta { return DiscardFiles(to_discard_size); } else { + ENGINE_LOG_ERROR << "QUERY ERROR WHEN DISCARDING FILES"; return Status::DBTransactionError("QUERY ERROR WHEN DISCARDING FILES", discardFilesQuery.error()); } } catch (const BadQuery& er) { // Handle any query errors + ENGINE_LOG_ERROR << "QUERY ERROR WHEN DISCARDING FILES" << ": " << er.what(); return Status::DBTransactionError("QUERY ERROR WHEN DISCARDING FILES", er.what()); } catch (const Exception& er) { // Catch-all for any other MySQL++ exceptions + ENGINE_LOG_ERROR << "GENERAL ERROR WHEN DISCARDING FILES" << ": " << er.what(); return Status::DBTransactionError("GENERAL ERROR WHEN DISCARDING FILES", er.what()); } } @@ -1170,16 +1209,19 @@ namespace meta { if (!updateTableFileQuery.exec()) { ENGINE_LOG_DEBUG << "table_id= " << file_schema.table_id_ << " file_id=" << file_schema.file_id_; + ENGINE_LOG_ERROR << "QUERY ERROR WHEN UPDATING TABLE FILE"; return Status::DBTransactionError("QUERY ERROR WHEN UPDATING TABLE FILE", updateTableFileQuery.error()); } } catch (const BadQuery& er) { // Handle any query errors ENGINE_LOG_DEBUG << "table_id= " << file_schema.table_id_ << " file_id=" << file_schema.file_id_; + ENGINE_LOG_ERROR << "QUERY ERROR WHEN UPDATING TABLE FILE" << ": " << er.what(); return Status::DBTransactionError("QUERY ERROR WHEN UPDATING TABLE FILE", er.what()); } catch (const Exception& er) { // Catch-all for any other MySQL++ exceptions ENGINE_LOG_DEBUG << "table_id= " << file_schema.table_id_ << " file_id=" << file_schema.file_id_; + ENGINE_LOG_ERROR << "GENERAL ERROR WHEN UPDATING TABLE FILE" << ": " << er.what(); return Status::DBTransactionError("GENERAL ERROR WHEN UPDATING TABLE FILE", er.what()); } return Status::OK(); @@ -1246,14 +1288,17 @@ namespace meta { } if (!updateTableFilesQuery.exec()) { + ENGINE_LOG_ERROR << "QUERY ERROR WHEN UPDATING TABLE FILES"; return Status::DBTransactionError("QUERY ERROR WHEN UPDATING TABLE FILES", updateTableFilesQuery.error()); } } catch (const BadQuery& er) { // Handle any query errors + ENGINE_LOG_ERROR << "QUERY ERROR WHEN UPDATING TABLE FILES" << ": " << er.what(); return Status::DBTransactionError("QUERY ERROR WHEN UPDATING TABLE FILES", er.what()); } catch (const Exception& er) { // Catch-all for any other MySQL++ exceptions + ENGINE_LOG_ERROR << "GENERAL ERROR WHEN UPDATING TABLE FILES" << ": " << er.what(); return Status::DBTransactionError("GENERAL ERROR WHEN UPDATING TABLE FILES", er.what()); } return Status::OK(); @@ -1314,14 +1359,17 @@ namespace meta { cleanUpFilesWithTTLQuery << "DELETE FROM metaFile WHERE " << idsToDeleteStr << ";"; if (!cleanUpFilesWithTTLQuery.exec()) { + ENGINE_LOG_ERROR << "QUERY ERROR WHEN CLEANING UP FILES WITH TTL"; return Status::DBTransactionError("CleanUpFilesWithTTL Error", cleanUpFilesWithTTLQuery.error()); } } catch (const BadQuery& er) { // Handle any query errors + ENGINE_LOG_ERROR << "QUERY ERROR WHEN CLEANING UP FILES WITH TTL" << ": " << er.what(); return Status::DBTransactionError("QUERY ERROR WHEN CLEANING UP FILES WITH TTL", er.what()); } catch (const Exception& er) { // Catch-all for any other MySQL++ exceptions + ENGINE_LOG_ERROR << "GENERAL ERROR WHEN CLEANING UP FILES WITH TTL" << ": " << er.what(); return Status::DBTransactionError("GENERAL ERROR WHEN CLEANING UP FILES WITH TTL", er.what()); } @@ -1355,15 +1403,18 @@ namespace meta { cleanUpFilesWithTTLQuery << "DELETE FROM meta WHERE " << idsToDeleteStr << ";"; if (!cleanUpFilesWithTTLQuery.exec()) { + ENGINE_LOG_ERROR << "QUERY ERROR WHEN CLEANING UP FILES WITH TTL"; return Status::DBTransactionError("QUERY ERROR WHEN CLEANING UP FILES WITH TTL", cleanUpFilesWithTTLQuery.error()); } } catch (const BadQuery& er) { // Handle any query errors + ENGINE_LOG_ERROR << "QUERY ERROR WHEN CLEANING UP FILES WITH TTL" << ": " << er.what(); return Status::DBTransactionError("QUERY ERROR WHEN CLEANING UP FILES WITH TTL", er.what()); } catch (const Exception& er) { // Catch-all for any other MySQL++ exceptions + ENGINE_LOG_ERROR << "GENERAL ERROR WHEN CLEANING UP FILES WITH TTL" << ": " << er.what(); return Status::DBTransactionError("GENERAL ERROR WHEN CLEANING UP FILES WITH TTL", er.what()); } @@ -1382,14 +1433,17 @@ namespace meta { cleanUpQuery << "DELETE FROM metaFile WHERE file_type = " << std::to_string(TableFileSchema::NEW) << ";"; if (!cleanUpQuery.exec()) { + ENGINE_LOG_ERROR << "QUERY ERROR WHEN CLEANING UP FILES"; return Status::DBTransactionError("Clean up Error", cleanUpQuery.error()); } } catch (const BadQuery& er) { // Handle any query errors + ENGINE_LOG_ERROR << "QUERY ERROR WHEN CLEANING UP FILES" << ": " << er.what(); return Status::DBTransactionError("QUERY ERROR WHEN CLEANING UP FILES", er.what()); } catch (const Exception& er) { // Catch-all for any other MySQL++ exceptions + ENGINE_LOG_ERROR << "GENERAL ERROR WHEN CLEANING UP FILES" << ": " << er.what(); return Status::DBTransactionError("GENERAL ERROR WHEN CLEANING UP FILES", er.what()); } @@ -1434,9 +1488,11 @@ namespace meta { } catch (const BadQuery& er) { // Handle any query errors + ENGINE_LOG_ERROR << "QUERY ERROR WHEN RETRIEVING COUNT" << ": " << er.what(); return Status::DBTransactionError("QUERY ERROR WHEN RETRIEVING COUNT", er.what()); } catch (const Exception& er) { // Catch-all for any other MySQL++ exceptions + ENGINE_LOG_ERROR << "GENERAL ERROR WHEN RETRIEVING COUNT" << ": " << er.what(); return Status::DBTransactionError("GENERAL ERROR WHEN RETRIEVING COUNT", er.what()); } return Status::OK(); @@ -1459,13 +1515,16 @@ namespace meta { return Status::OK(); } else { + ENGINE_LOG_ERROR << "QUERY ERROR WHEN DROPPING TABLE"; return Status::DBTransactionError("DROP TABLE ERROR", dropTableQuery.error()); } } catch (const BadQuery& er) { // Handle any query errors + ENGINE_LOG_ERROR << "QUERY ERROR WHEN DROPPING TABLE" << ": " << er.what(); return Status::DBTransactionError("QUERY ERROR WHEN DROPPING TABLE", er.what()); } catch (const Exception& er) { // Catch-all for any other MySQL++ exceptions + ENGINE_LOG_ERROR << "GENERAL ERROR WHEN DROPPING TABLE" << ": " << er.what(); return Status::DBTransactionError("GENERAL ERROR WHEN DROPPING TABLE", er.what()); } return Status::OK(); From 7f24723ec70230d142a35a02009f171c9ac0dc92 Mon Sep 17 00:00:00 2001 From: zhiru Date: Wed, 26 Jun 2019 13:51:56 +0800 Subject: [PATCH 15/43] fix mysql scoped connection concurrency issue Former-commit-id: ecb3cf07b5f49f2f87e3972a126452ec2950caf8 --- cpp/cmake/ThirdPartyPackages.cmake | 20 +- cpp/conf/server_config.yaml | 2 +- cpp/src/CMakeLists.txt | 5 +- cpp/src/db/MySQLConnectionPool.h | 7 + cpp/src/db/MySQLMetaImpl.cpp | 848 ++++++++++++++++------------- cpp/thirdparty/versions.txt | 2 +- 6 files changed, 478 insertions(+), 406 deletions(-) diff --git a/cpp/cmake/ThirdPartyPackages.cmake b/cpp/cmake/ThirdPartyPackages.cmake index 55b4266387..9aa3f62124 100644 --- a/cpp/cmake/ThirdPartyPackages.cmake +++ b/cpp/cmake/ThirdPartyPackages.cmake @@ -271,7 +271,7 @@ endif() if(DEFINED ENV{MILVUS_MYSQLPP_URL}) set(MYSQLPP_SOURCE_URL "$ENV{MILVUS_MYSQLPP_URL}") else() - set(MYSQLPP_SOURCE_URL "https://github.com/youny626/mysqlpp.git") + set(MYSQLPP_SOURCE_URL "https://tangentsoft.com/mysqlpp/releases/mysql++-${MYSQLPP_VERSION}.tar.gz") endif() if (DEFINED ENV{MILVUS_OPENBLAS_URL}) @@ -1095,18 +1095,18 @@ macro(build_mysqlpp) "LDFLAGS=-pthread") externalproject_add(mysqlpp_ep -# URL -# ${MYSQLPP_SOURCE_URL} - GIT_REPOSITORY + URL ${MYSQLPP_SOURCE_URL} - GIT_TAG - ${MYSQLPP_VERSION} - GIT_SHALLOW - TRUE +# GIT_REPOSITORY +# ${MYSQLPP_SOURCE_URL} +# GIT_TAG +# ${MYSQLPP_VERSION} +# GIT_SHALLOW +# TRUE ${EP_LOG_OPTIONS} CONFIGURE_COMMAND - "./bootstrap" - COMMAND +# "./bootstrap" +# COMMAND "./configure" ${MYSQLPP_CONFIGURE_ARGS} BUILD_COMMAND diff --git a/cpp/conf/server_config.yaml b/cpp/conf/server_config.yaml index 05705bcc4c..9f60f0f212 100644 --- a/cpp/conf/server_config.yaml +++ b/cpp/conf/server_config.yaml @@ -10,7 +10,7 @@ db_config: db_path: /tmp/milvus #URI format: dialect://username:password@host:port/database #All parts except dialect are optional, but you MUST include the delimiters - db_backend_url: sqlite://:@:/ + db_backend_url: mysql://root:1234@:/test index_building_threshold: 1024 #build index file when raw data file size larger than this value, unit: MB metric_config: diff --git a/cpp/src/CMakeLists.txt b/cpp/src/CMakeLists.txt index 7c28c83aeb..3a955e0654 100644 --- a/cpp/src/CMakeLists.txt +++ b/cpp/src/CMakeLists.txt @@ -183,7 +183,10 @@ endif () install(TARGETS milvus_server DESTINATION bin) -install(FILES ${CMAKE_BINARY_DIR}/mysqlpp_ep-prefix/src/mysqlpp_ep/lib/${CMAKE_SHARED_LIBRARY_PREFIX}mysqlpp${CMAKE_SHARED_LIBRARY_SUFFIX} +install(FILES + ${CMAKE_BINARY_DIR}/mysqlpp_ep-prefix/src/mysqlpp_ep/lib/${CMAKE_SHARED_LIBRARY_PREFIX}mysqlpp${CMAKE_SHARED_LIBRARY_SUFFIX} + ${CMAKE_BINARY_DIR}/mysqlpp_ep-prefix/src/mysqlpp_ep/lib/${CMAKE_SHARED_LIBRARY_PREFIX}mysqlpp${CMAKE_SHARED_LIBRARY_SUFFIX}.3 + ${CMAKE_BINARY_DIR}/mysqlpp_ep-prefix/src/mysqlpp_ep/lib/${CMAKE_SHARED_LIBRARY_PREFIX}mysqlpp${CMAKE_SHARED_LIBRARY_SUFFIX}.3.2.4 DESTINATION bin) #need to copy libmysqlpp.so add_subdirectory(sdk) diff --git a/cpp/src/db/MySQLConnectionPool.h b/cpp/src/db/MySQLConnectionPool.h index 26fc4432ec..7d3a1f88a3 100644 --- a/cpp/src/db/MySQLConnectionPool.h +++ b/cpp/src/db/MySQLConnectionPool.h @@ -3,6 +3,8 @@ #include #include +#include "Log.h" + class MySQLConnectionPool : public mysqlpp::ConnectionPool { public: @@ -43,6 +45,7 @@ public: sleep(1); } + ENGINE_LOG_DEBUG << "conns_in_use_ in grab: " << conns_in_use_ << std::endl; ++conns_in_use_; return mysqlpp::ConnectionPool::grab(); } @@ -50,7 +53,11 @@ public: // Other half of in-use conn count limit void release(const mysqlpp::Connection* pc) override { mysqlpp::ConnectionPool::release(pc); + ENGINE_LOG_DEBUG << "conns_in_use_ in release: " << conns_in_use_ << std::endl; --conns_in_use_; + if (conns_in_use_ < 0) { + ENGINE_LOG_DEBUG << "conns_in_use_ in release < 0: " << conns_in_use_ << std::endl; + } } void set_max_idle_time(int max_idle) { diff --git a/cpp/src/db/MySQLMetaImpl.cpp b/cpp/src/db/MySQLMetaImpl.cpp index e0c903795d..6b733ec5ae 100644 --- a/cpp/src/db/MySQLMetaImpl.cpp +++ b/cpp/src/db/MySQLMetaImpl.cpp @@ -160,25 +160,27 @@ namespace meta { // std::cout << "MySQL++ thread aware:" << std::to_string(connectionPtr->thread_aware()) << std::endl; try { - ScopedConnection connectionPtr(*mySQLConnectionPool_, safe_grab); + + CleanUp(); + + { + ScopedConnection connectionPtr(*mySQLConnectionPool_, safe_grab); // if (!connectionPtr->connect(dbName, serverAddress, username, password, port)) { // return Status::Error("DB connection failed: ", connectionPtr->error()); // } - if (!connectionPtr->thread_aware()) { - ENGINE_LOG_ERROR << "MySQL++ wasn't built with thread awareness! Can't run without it."; - return Status::Error("MySQL++ wasn't built with thread awareness! Can't run without it."); - } - - CleanUp(); - Query InitializeQuery = connectionPtr->query(); + if (!connectionPtr->thread_aware()) { + ENGINE_LOG_ERROR << "MySQL++ wasn't built with thread awareness! Can't run without it."; + return Status::Error("MySQL++ wasn't built with thread awareness! Can't run without it."); + } + Query InitializeQuery = connectionPtr->query(); // InitializeQuery << "SET max_allowed_packet=67108864;"; // if (!InitializeQuery.exec()) { // return Status::DBTransactionError("Initialization Error", InitializeQuery.error()); // } -// InitializeQuery << "DROP TABLE IF EXISTS meta, metaFile;"; - InitializeQuery << "CREATE TABLE IF NOT EXISTS meta (" << +// InitializeQuery << "DROP TABLE IF EXISTS Tables, TableFiles;"; + InitializeQuery << "CREATE TABLE IF NOT EXISTS Tables (" << "id BIGINT PRIMARY KEY AUTO_INCREMENT, " << "table_id VARCHAR(255) UNIQUE NOT NULL, " << "state INT NOT NULL, " << @@ -187,23 +189,24 @@ namespace meta { "files_cnt BIGINT DEFAULT 0 NOT NULL, " << "engine_type INT DEFAULT 1 NOT NULL, " << "store_raw_data BOOL DEFAULT false NOT NULL);"; - if (!InitializeQuery.exec()) { - return Status::DBTransactionError("Initialization Error", InitializeQuery.error()); - } + if (!InitializeQuery.exec()) { + return Status::DBTransactionError("Initialization Error", InitializeQuery.error()); + } - InitializeQuery << "CREATE TABLE IF NOT EXISTS metaFile (" << - "id BIGINT PRIMARY KEY AUTO_INCREMENT, " << - "table_id VARCHAR(255) NOT NULL, " << - "engine_type INT DEFAULT 1 NOT NULL, " << - "file_id VARCHAR(255) NOT NULL, " << - "file_type INT DEFAULT 0 NOT NULL, " << - "size BIGINT DEFAULT 0 NOT NULL, " << - "updated_time BIGINT NOT NULL, " << - "created_on BIGINT NOT NULL, " << - "date INT DEFAULT -1 NOT NULL);"; - if (!InitializeQuery.exec()) { - return Status::DBTransactionError("Initialization Error", InitializeQuery.error()); - } + InitializeQuery << "CREATE TABLE IF NOT EXISTS TableFiles (" << + "id BIGINT PRIMARY KEY AUTO_INCREMENT, " << + "table_id VARCHAR(255) NOT NULL, " << + "engine_type INT DEFAULT 1 NOT NULL, " << + "file_id VARCHAR(255) NOT NULL, " << + "file_type INT DEFAULT 0 NOT NULL, " << + "size BIGINT DEFAULT 0 NOT NULL, " << + "updated_time BIGINT NOT NULL, " << + "created_on BIGINT NOT NULL, " << + "date INT DEFAULT -1 NOT NULL);"; + if (!InitializeQuery.exec()) { + return Status::DBTransactionError("Initialization Error", InitializeQuery.error()); + } + } //Scoped Connection // //Consume all results to avoid "Commands out of sync" error // while (InitializeQuery.more_results()) { @@ -247,7 +250,7 @@ namespace meta { // std::lock_guard lock(mysql_mutex); - if (dates.size() == 0) { + if (dates.empty()) { return Status::OK(); } @@ -260,8 +263,6 @@ namespace meta { try { - ScopedConnection connectionPtr(*mySQLConnectionPool_, safe_grab); - auto yesterday = GetDateWithDelta(-1); for (auto &date : dates) { @@ -270,24 +271,29 @@ namespace meta { } } - Query dropPartitionsByDatesQuery = connectionPtr->query(); - std::stringstream dateListSS; for (auto &date : dates) { - dateListSS << std::to_string(date) << ", "; + dateListSS << std::to_string(date) << ", "; } std::string dateListStr = dateListSS.str(); dateListStr = dateListStr.substr(0, dateListStr.size() - 2); //remove the last ", " - dropPartitionsByDatesQuery << "UPDATE metaFile " << - "SET file_type = " << std::to_string(TableFileSchema::TO_DELETE) << " " << - "WHERE table_id = " << quote << table_id << " AND " << - "date in (" << dateListStr << ");"; + { + ScopedConnection connectionPtr(*mySQLConnectionPool_, safe_grab); - if (!dropPartitionsByDatesQuery.exec()) { - ENGINE_LOG_ERROR << "QUERY ERROR WHEN DROPPING PARTITIONS BY DATES"; - return Status::DBTransactionError("QUERY ERROR WHEN DROPPING PARTITIONS BY DATES", dropPartitionsByDatesQuery.error()); - } + Query dropPartitionsByDatesQuery = connectionPtr->query(); + + dropPartitionsByDatesQuery << "UPDATE TableFiles " << + "SET file_type = " << std::to_string(TableFileSchema::TO_DELETE) << " " << + "WHERE table_id = " << quote << table_id << " AND " << + "date in (" << dateListStr << ");"; + + if (!dropPartitionsByDatesQuery.exec()) { + ENGINE_LOG_ERROR << "QUERY ERROR WHEN DROPPING PARTITIONS BY DATES"; + return Status::DBTransactionError("QUERY ERROR WHEN DROPPING PARTITIONS BY DATES", + dropPartitionsByDatesQuery.error()); + } + } //Scoped Connection } catch (const BadQuery& er) { // Handle any query errors @@ -310,58 +316,61 @@ namespace meta { MetricCollector metric; - ScopedConnection connectionPtr(*mySQLConnectionPool_, safe_grab); + { + ScopedConnection connectionPtr(*mySQLConnectionPool_, safe_grab); - Query createTableQuery = connectionPtr->query(); - - if (table_schema.table_id_.empty()) { - NextTableId(table_schema.table_id_); - } - else { - createTableQuery << "SELECT state FROM meta " << - "WHERE table_id = " << quote << table_schema.table_id_ << ";"; - StoreQueryResult res = createTableQuery.store(); - assert(res && res.num_rows() <= 1); - if (res.num_rows() == 1) { - int state = res[0]["state"]; - std::string msg = (TableSchema::TO_DELETE == state) ? - "Table already exists and it is in delete state, please wait a second" : "Table already exists"; - return Status::Error(msg); + Query createTableQuery = connectionPtr->query(); + ENGINE_LOG_DEBUG << "Create Table in"; + if (table_schema.table_id_.empty()) { + NextTableId(table_schema.table_id_); + } else { + createTableQuery << "SELECT state FROM Tables " << + "WHERE table_id = " << quote << table_schema.table_id_ << ";"; + ENGINE_LOG_DEBUG << "Create Table : " << createTableQuery.str(); + StoreQueryResult res = createTableQuery.store(); + assert(res && res.num_rows() <= 1); + if (res.num_rows() == 1) { + int state = res[0]["state"]; + std::string msg = (TableSchema::TO_DELETE == state) ? + "Table already exists and it is in delete state, please wait a second" + : "Table already exists"; + ENGINE_LOG_WARNING << "MySQLMetaImpl::CreateTable: " << msg; + return Status::Error(msg); + } } - } + ENGINE_LOG_DEBUG << "Create Table start"; - table_schema.files_cnt_ = 0; - table_schema.id_ = -1; - table_schema.created_on_ = utils::GetMicroSecTimeStamp(); + table_schema.files_cnt_ = 0; + table_schema.id_ = -1; + table_schema.created_on_ = utils::GetMicroSecTimeStamp(); // auto start_time = METRICS_NOW_TIME; - std::string id = "NULL"; //auto-increment - std::string table_id = table_schema.table_id_; - std::string state = std::to_string(table_schema.state_); - std::string dimension = std::to_string(table_schema.dimension_); - std::string created_on = std::to_string(table_schema.created_on_); - std::string files_cnt = "0"; - std::string engine_type = std::to_string(table_schema.engine_type_); - std::string store_raw_data = table_schema.store_raw_data_ ? "true" : "false"; + std::string id = "NULL"; //auto-increment + std::string table_id = table_schema.table_id_; + std::string state = std::to_string(table_schema.state_); + std::string dimension = std::to_string(table_schema.dimension_); + std::string created_on = std::to_string(table_schema.created_on_); + std::string files_cnt = "0"; + std::string engine_type = std::to_string(table_schema.engine_type_); + std::string store_raw_data = table_schema.store_raw_data_ ? "true" : "false"; - createTableQuery << "INSERT INTO meta VALUES" << - "(" << id << ", " << quote << table_id << ", " << state << ", " << dimension << ", " << - created_on << ", " << files_cnt << ", " << engine_type << ", " << store_raw_data - << ");"; - - if (SimpleResult res = createTableQuery.execute()) { - table_schema.id_ = res.insert_id(); //Might need to use SELECT LAST_INSERT_ID()? + createTableQuery << "INSERT INTO Tables VALUES" << + "(" << id << ", " << quote << table_id << ", " << state << ", " << dimension << ", " << + created_on << ", " << files_cnt << ", " << engine_type << ", " << store_raw_data << ");"; + ENGINE_LOG_DEBUG << "Create Table : " << createTableQuery.str(); + if (SimpleResult res = createTableQuery.execute()) { + table_schema.id_ = res.insert_id(); //Might need to use SELECT LAST_INSERT_ID()? // std::cout << table_schema.id_ << std::endl; - //Consume all results to avoid "Commands out of sync" error + //Consume all results to avoid "Commands out of sync" error // while (createTableQuery.more_results()) { // createTableQuery.store_next(); // } - } - else { - ENGINE_LOG_ERROR << "Add Table Error"; - return Status::DBTransactionError("Add Table Error", createTableQuery.error()); - } + } else { + ENGINE_LOG_ERROR << "Add Table Error"; + return Status::DBTransactionError("Add Table Error", createTableQuery.error()); + } + } //Scoped Connection // auto end_time = METRICS_NOW_TIME; // auto total_time = METRICS_MICROSECONDS(start_time, end_time); @@ -399,19 +408,22 @@ namespace meta { MetricCollector metric; - ScopedConnection connectionPtr(*mySQLConnectionPool_, safe_grab); + { + ScopedConnection connectionPtr(*mySQLConnectionPool_, safe_grab); - //soft delete table - Query deleteTableQuery = connectionPtr->query(); + //soft delete table + Query deleteTableQuery = connectionPtr->query(); // - deleteTableQuery << "UPDATE meta " << - "SET state = " << std::to_string(TableSchema::TO_DELETE) << " " << - "WHERE table_id = " << quote << table_id << ";"; + deleteTableQuery << "UPDATE Tables " << + "SET state = " << std::to_string(TableSchema::TO_DELETE) << " " << + "WHERE table_id = " << quote << table_id << ";"; - if (!deleteTableQuery.exec()) { - ENGINE_LOG_ERROR << "QUERY ERROR WHEN DELETING TABLE"; - return Status::DBTransactionError("QUERY ERROR WHEN DELETING TABLE", deleteTableQuery.error()); - } + if (!deleteTableQuery.exec()) { + ENGINE_LOG_ERROR << "QUERY ERROR WHEN DELETING TABLE"; + return Status::DBTransactionError("QUERY ERROR WHEN DELETING TABLE", deleteTableQuery.error()); + } + + } //Scoped Connection } catch (const BadQuery& er) { // Handle any query errors ENGINE_LOG_ERROR << "GENERAL ERROR WHEN DELETING TABLE" << ": " << er.what(); @@ -429,21 +441,22 @@ namespace meta { try { MetricCollector metric; - ScopedConnection connectionPtr(*mySQLConnectionPool_, safe_grab); + { + ScopedConnection connectionPtr(*mySQLConnectionPool_, safe_grab); - //soft delete table files - Query deleteTableFilesQuery = connectionPtr->query(); - // - deleteTableFilesQuery << "UPDATE metaFile " << - "SET state = " << std::to_string(TableSchema::TO_DELETE) << ", " << - "updated_time = " << std::to_string(utils::GetMicroSecTimeStamp()) << " " - "WHERE table_id = " << quote << table_id << ";"; - - if (!deleteTableFilesQuery.exec()) { - ENGINE_LOG_ERROR << "QUERY ERROR WHEN DELETING TABLE FILES"; - return Status::DBTransactionError("QUERY ERROR WHEN DELETING TABLE", deleteTableFilesQuery.error()); - } + //soft delete table files + Query deleteTableFilesQuery = connectionPtr->query(); + // + deleteTableFilesQuery << "UPDATE TableFiles " << + "SET state = " << std::to_string(TableSchema::TO_DELETE) << ", " << + "updated_time = " << std::to_string(utils::GetMicroSecTimeStamp()) << " " << + "WHERE table_id = " << quote << table_id << ";"; + if (!deleteTableFilesQuery.exec()) { + ENGINE_LOG_ERROR << "QUERY ERROR WHEN DELETING TABLE FILES"; + return Status::DBTransactionError("QUERY ERROR WHEN DELETING TABLE", deleteTableFilesQuery.error()); + } + } //Scoped Connection } catch (const BadQuery& er) { // Handle any query errors ENGINE_LOG_ERROR << "QUERY ERROR WHEN DELETING TABLE FILES" << ": " << er.what(); @@ -465,14 +478,18 @@ namespace meta { MetricCollector metric; - ScopedConnection connectionPtr(*mySQLConnectionPool_, safe_grab); + StoreQueryResult res; - Query describeTableQuery = connectionPtr->query(); - describeTableQuery << "SELECT id, dimension, files_cnt, engine_type, store_raw_data " << - "FROM meta " << - "WHERE table_id = " << quote << table_schema.table_id_ << " " << - "AND state <> " << std::to_string(TableSchema::TO_DELETE) << ";"; - StoreQueryResult res = describeTableQuery.store(); + { + ScopedConnection connectionPtr(*mySQLConnectionPool_, safe_grab); + + Query describeTableQuery = connectionPtr->query(); + describeTableQuery << "SELECT id, dimension, files_cnt, engine_type, store_raw_data " << + "FROM Tables " << + "WHERE table_id = " << quote << table_schema.table_id_ << " " << + "AND state <> " << std::to_string(TableSchema::TO_DELETE) << ";"; + res = describeTableQuery.store(); + } //Scoped Connection assert(res && res.num_rows() <= 1); if (res.num_rows() == 1) { @@ -516,16 +533,20 @@ namespace meta { MetricCollector metric; - ScopedConnection connectionPtr(*mySQLConnectionPool_, safe_grab); + StoreQueryResult res; - Query hasTableQuery = connectionPtr->query(); - //since table_id is a unique column we just need to check whether it exists or not - hasTableQuery << "SELECT EXISTS " << - "(SELECT 1 FROM meta " << - "WHERE table_id = " << quote << table_id << " " << - "AND state <> " << std::to_string(TableSchema::TO_DELETE) << ") " << - "AS " << quote << "check" << ";"; - StoreQueryResult res = hasTableQuery.store(); + { + ScopedConnection connectionPtr(*mySQLConnectionPool_, safe_grab); + + Query hasTableQuery = connectionPtr->query(); + //since table_id is a unique column we just need to check whether it exists or not + hasTableQuery << "SELECT EXISTS " << + "(SELECT 1 FROM Tables " << + "WHERE table_id = " << quote << table_id << " " << + "AND state <> " << std::to_string(TableSchema::TO_DELETE) << ") " << + "AS " << quote << "check" << ";"; + res = hasTableQuery.store(); + } //Scoped Connection assert(res && res.num_rows() == 1); int check = res[0]["check"]; @@ -552,13 +573,17 @@ namespace meta { MetricCollector metric; - ScopedConnection connectionPtr(*mySQLConnectionPool_, safe_grab); + StoreQueryResult res; - Query allTablesQuery = connectionPtr->query(); - allTablesQuery << "SELECT id, table_id, dimension, files_cnt, engine_type, store_raw_data " << - "FROM meta " << - "WHERE state <> " << std::to_string(TableSchema::TO_DELETE) << ";"; - StoreQueryResult res = allTablesQuery.store(); + { + ScopedConnection connectionPtr(*mySQLConnectionPool_, safe_grab); + + Query allTablesQuery = connectionPtr->query(); + allTablesQuery << "SELECT id, table_id, dimension, files_cnt, engine_type, store_raw_data " << + "FROM Tables " << + "WHERE state <> " << std::to_string(TableSchema::TO_DELETE) << ";"; + res = allTablesQuery.store(); + } //Scoped Connection for (auto& resRow : res) { TableSchema table_schema; @@ -610,8 +635,6 @@ namespace meta { MetricCollector metric; - ScopedConnection connectionPtr(*mySQLConnectionPool_, safe_grab); - NextFileId(file_schema.file_id_); file_schema.file_type_ = TableFileSchema::NEW; file_schema.dimension_ = table_schema.dimension_; @@ -621,7 +644,6 @@ namespace meta { file_schema.engine_type_ = table_schema.engine_type_; GetTableFilePath(file_schema); - Query createTableFileQuery = connectionPtr->query(); std::string id = "NULL"; //auto-increment std::string table_id = file_schema.table_id_; std::string engine_type = std::to_string(file_schema.engine_type_); @@ -632,23 +654,28 @@ namespace meta { std::string created_on = std::to_string(file_schema.created_on_); std::string date = std::to_string(file_schema.date_); - createTableFileQuery << "INSERT INTO metaFile VALUES" << - "(" << id << ", " << quote << table_id << ", " << engine_type << ", " << - quote << file_id << ", " << file_type << ", " << size << ", " << - updated_time << ", " << created_on << ", " << date << ");"; + { + ScopedConnection connectionPtr(*mySQLConnectionPool_, safe_grab); - if (SimpleResult res = createTableFileQuery.execute()) { - file_schema.id_ = res.insert_id(); //Might need to use SELECT LAST_INSERT_ID()? + Query createTableFileQuery = connectionPtr->query(); - //Consume all results to avoid "Commands out of sync" error + createTableFileQuery << "INSERT INTO TableFiles VALUES" << + "(" << id << ", " << quote << table_id << ", " << engine_type << ", " << + quote << file_id << ", " << file_type << ", " << size << ", " << + updated_time << ", " << created_on << ", " << date << ");"; + + if (SimpleResult res = createTableFileQuery.execute()) { + file_schema.id_ = res.insert_id(); //Might need to use SELECT LAST_INSERT_ID()? + + //Consume all results to avoid "Commands out of sync" error // while (createTableFileQuery.more_results()) { // createTableFileQuery.store_next(); // } - } - else { - ENGINE_LOG_ERROR << "QUERY ERROR WHEN ADDING TABLE FILE"; - return Status::DBTransactionError("Add file Error", createTableFileQuery.error()); - } + } else { + ENGINE_LOG_ERROR << "QUERY ERROR WHEN ADDING TABLE FILE"; + return Status::DBTransactionError("Add file Error", createTableFileQuery.error()); + } + } // Scoped Connection auto partition_path = GetTableDatePartitionPath(file_schema.table_id_, file_schema.date_); @@ -685,13 +712,17 @@ namespace meta { MetricCollector metric; - ScopedConnection connectionPtr(*mySQLConnectionPool_, safe_grab); + StoreQueryResult res; - Query filesToIndexQuery = connectionPtr->query(); - filesToIndexQuery << "SELECT id, table_id, engine_type, file_id, file_type, size, date " << - "FROM metaFile " << - "WHERE file_type = " << std::to_string(TableFileSchema::TO_INDEX) << ";"; - StoreQueryResult res = filesToIndexQuery.store(); + { + ScopedConnection connectionPtr(*mySQLConnectionPool_, safe_grab); + + Query filesToIndexQuery = connectionPtr->query(); + filesToIndexQuery << "SELECT id, table_id, engine_type, file_id, file_type, size, date " << + "FROM TableFiles " << + "WHERE file_type = " << std::to_string(TableFileSchema::TO_INDEX) << ";"; + res = filesToIndexQuery.store(); + } //Scoped Connection std::map groups; TableFileSchema table_file; @@ -757,43 +788,44 @@ namespace meta { MetricCollector metric; - ScopedConnection connectionPtr(*mySQLConnectionPool_, safe_grab); - StoreQueryResult res; - if (partition.empty()) { + { + ScopedConnection connectionPtr(*mySQLConnectionPool_, safe_grab); - Query filesToSearchQuery = connectionPtr->query(); - filesToSearchQuery << "SELECT id, table_id, engine_type, file_id, file_type, size, date " << - "FROM metaFile " << - "WHERE table_id = " << quote << table_id << " AND " << - "(file_type = " << std::to_string(TableFileSchema::RAW) << " OR " << - "file_type = " << std::to_string(TableFileSchema::TO_INDEX) << " OR " << - "file_type = " << std::to_string(TableFileSchema::INDEX) << ");"; - res = filesToSearchQuery.store(); + if (partition.empty()) { - } - else { + Query filesToSearchQuery = connectionPtr->query(); + filesToSearchQuery << "SELECT id, table_id, engine_type, file_id, file_type, size, date " << + "FROM TableFiles " << + "WHERE table_id = " << quote << table_id << " AND " << + "(file_type = " << std::to_string(TableFileSchema::RAW) << " OR " << + "file_type = " << std::to_string(TableFileSchema::TO_INDEX) << " OR " << + "file_type = " << std::to_string(TableFileSchema::INDEX) << ");"; + res = filesToSearchQuery.store(); - Query filesToSearchQuery = connectionPtr->query(); + } else { - std::stringstream partitionListSS; - for (auto &date : partition) { - partitionListSS << std::to_string(date) << ", "; - } - std::string partitionListStr = partitionListSS.str(); - partitionListStr = partitionListStr.substr(0, partitionListStr.size() - 2); //remove the last ", " + Query filesToSearchQuery = connectionPtr->query(); - filesToSearchQuery << "SELECT id, table_id, engine_type, file_id, file_type, size, date " << - "FROM metaFile " << + std::stringstream partitionListSS; + for (auto &date : partition) { + partitionListSS << std::to_string(date) << ", "; + } + std::string partitionListStr = partitionListSS.str(); + partitionListStr = partitionListStr.substr(0, partitionListStr.size() - 2); //remove the last ", " + + filesToSearchQuery << "SELECT id, table_id, engine_type, file_id, file_type, size, date " << + "FROM TableFiles " << "WHERE table_id = " << quote << table_id << " AND " << "date IN (" << partitionListStr << ") AND " << "(file_type = " << std::to_string(TableFileSchema::RAW) << " OR " << "file_type = " << std::to_string(TableFileSchema::TO_INDEX) << " OR " << "file_type = " << std::to_string(TableFileSchema::INDEX) << ");"; - res = filesToSearchQuery.store(); + res = filesToSearchQuery.store(); - } + } + } //Scoped Connection TableSchema table_schema; table_schema.table_id_ = table_id; @@ -857,15 +889,19 @@ namespace meta { try { MetricCollector metric; - ScopedConnection connectionPtr(*mySQLConnectionPool_, safe_grab); + StoreQueryResult res; - Query filesToMergeQuery = connectionPtr->query(); - filesToMergeQuery << "SELECT id, table_id, file_id, file_type, size, date " << - "FROM metaFile " << - "WHERE table_id = " << quote << table_id << " AND " << - "file_type = " << std::to_string(TableFileSchema::RAW) << " " << - "ORDER BY size DESC" << ";"; - StoreQueryResult res = filesToMergeQuery.store(); + { + ScopedConnection connectionPtr(*mySQLConnectionPool_, safe_grab); + + Query filesToMergeQuery = connectionPtr->query(); + filesToMergeQuery << "SELECT id, table_id, file_id, file_type, size, date " << + "FROM TableFiles " << + "WHERE table_id = " << quote << table_id << " AND " << + "file_type = " << std::to_string(TableFileSchema::RAW) << " " << + "ORDER BY size DESC" << ";"; + res = filesToMergeQuery.store(); + } //Scoped Connection TableSchema table_schema; table_schema.table_id_ = table_id; @@ -934,14 +970,18 @@ namespace meta { try { - ScopedConnection connectionPtr(*mySQLConnectionPool_, safe_grab); + StoreQueryResult res; - Query getTableFileQuery = connectionPtr->query(); - getTableFileQuery << "SELECT engine_type, file_id, file_type, size, date " << - "FROM metaFile " << - "WHERE table_id = " << quote << table_id << " AND " << - "(" << idStr << ");"; - StoreQueryResult res = getTableFileQuery.store(); + { + ScopedConnection connectionPtr(*mySQLConnectionPool_, safe_grab); + + Query getTableFileQuery = connectionPtr->query(); + getTableFileQuery << "SELECT engine_type, file_id, file_type, size, date " << + "FROM TableFiles " << + "WHERE table_id = " << quote << table_id << " AND " << + "(" << idStr << ");"; + res = getTableFileQuery.store(); + } //Scoped Connection assert(res); @@ -1011,7 +1051,7 @@ namespace meta { ScopedConnection connectionPtr(*mySQLConnectionPool_, safe_grab); Query archiveQuery = connectionPtr->query(); - archiveQuery << "UPDATE metaFile " << + archiveQuery << "UPDATE TableFiles " << "SET file_type = " << std::to_string(TableFileSchema::TO_DELETE) << " " << "WHERE created_on < " << std::to_string(now - usecs) << " AND " << "file_type <> " << std::to_string(TableFileSchema::TO_DELETE) << ";"; @@ -1048,13 +1088,17 @@ namespace meta { result = 0; try { - ScopedConnection connectionPtr(*mySQLConnectionPool_, safe_grab); + StoreQueryResult res; - Query getSizeQuery = connectionPtr->query(); - getSizeQuery << "SELECT SUM(size) AS sum " << - "FROM metaFile " << - "WHERE file_type <> " << std::to_string(TableFileSchema::TO_DELETE) << ";"; - StoreQueryResult res = getSizeQuery.store(); + { + ScopedConnection connectionPtr(*mySQLConnectionPool_, safe_grab); + + Query getSizeQuery = connectionPtr->query(); + getSizeQuery << "SELECT SUM(size) AS sum " << + "FROM TableFiles " << + "WHERE file_type <> " << std::to_string(TableFileSchema::TO_DELETE) << ";"; + res = getSizeQuery.store(); + } //Scoped Connection assert(res && res.num_rows() == 1); // if (!res) { @@ -1097,51 +1141,55 @@ namespace meta { MetricCollector metric; - ScopedConnection connectionPtr(*mySQLConnectionPool_, safe_grab); + bool status; - Query discardFilesQuery = connectionPtr->query(); - discardFilesQuery << "SELECT id, size " << - "FROM metaFile " << - "WHERE file_type <> " << std::to_string(TableFileSchema::TO_DELETE) << " " << - "ORDER BY id ASC " << - "LIMIT 10;"; + { + ScopedConnection connectionPtr(*mySQLConnectionPool_, safe_grab); + + Query discardFilesQuery = connectionPtr->query(); + discardFilesQuery << "SELECT id, size " << + "FROM TableFiles " << + "WHERE file_type <> " << std::to_string(TableFileSchema::TO_DELETE) << " " << + "ORDER BY id ASC " << + "LIMIT 10;"; // std::cout << discardFilesQuery.str() << std::endl; - StoreQueryResult res = discardFilesQuery.store(); + StoreQueryResult res = discardFilesQuery.store(); - assert(res); - if (res.num_rows() == 0) { - return Status::OK(); - } - - TableFileSchema table_file; - std::stringstream idsToDiscardSS; - for (auto& resRow : res) { - if (to_discard_size <= 0) { - break; + assert(res); + if (res.num_rows() == 0) { + return Status::OK(); } - table_file.id_ = resRow["id"]; - table_file.size_ = resRow["size"]; - idsToDiscardSS << "id = " << std::to_string(table_file.id_) << " OR "; - ENGINE_LOG_DEBUG << "Discard table_file.id=" << table_file.file_id_ - << " table_file.size=" << table_file.size_; - to_discard_size -= table_file.size_; - } - std::string idsToDiscardStr = idsToDiscardSS.str(); - idsToDiscardStr = idsToDiscardStr.substr(0, idsToDiscardStr.size() - 4); //remove the last " OR " + TableFileSchema table_file; + std::stringstream idsToDiscardSS; + for (auto &resRow : res) { + if (to_discard_size <= 0) { + break; + } + table_file.id_ = resRow["id"]; + table_file.size_ = resRow["size"]; + idsToDiscardSS << "id = " << std::to_string(table_file.id_) << " OR "; + ENGINE_LOG_DEBUG << "Discard table_file.id=" << table_file.file_id_ + << " table_file.size=" << table_file.size_; + to_discard_size -= table_file.size_; + } - discardFilesQuery << "UPDATE metaFile " << - "SET file_type = " << std::to_string(TableFileSchema::TO_DELETE) << ", " << - "updated_time = " << std::to_string(utils::GetMicroSecTimeStamp()) << " " << - "WHERE " << idsToDiscardStr << ";"; + std::string idsToDiscardStr = idsToDiscardSS.str(); + idsToDiscardStr = idsToDiscardStr.substr(0, idsToDiscardStr.size() - 4); //remove the last " OR " - if (discardFilesQuery.exec()) { - return DiscardFiles(to_discard_size); - } - else { - ENGINE_LOG_ERROR << "QUERY ERROR WHEN DISCARDING FILES"; - return Status::DBTransactionError("QUERY ERROR WHEN DISCARDING FILES", discardFilesQuery.error()); - } + discardFilesQuery << "UPDATE TableFiles " << + "SET file_type = " << std::to_string(TableFileSchema::TO_DELETE) << ", " << + "updated_time = " << std::to_string(utils::GetMicroSecTimeStamp()) << " " << + "WHERE " << idsToDiscardStr << ";"; + + status = discardFilesQuery.exec(); + if (!status) { + ENGINE_LOG_ERROR << "QUERY ERROR WHEN DISCARDING FILES"; + return Status::DBTransactionError("QUERY ERROR WHEN DISCARDING FILES", discardFilesQuery.error()); + } + } //Scoped Connection + + return DiscardFiles(to_discard_size); } catch (const BadQuery& er) { // Handle any query errors @@ -1164,54 +1212,57 @@ namespace meta { MetricCollector metric; - ScopedConnection connectionPtr(*mySQLConnectionPool_, safe_grab); + { + ScopedConnection connectionPtr(*mySQLConnectionPool_, safe_grab); - Query updateTableFileQuery = connectionPtr->query(); + Query updateTableFileQuery = connectionPtr->query(); - //if the table has been deleted, just mark the table file as TO_DELETE - //clean thread will delete the file later - updateTableFileQuery << "SELECT state FROM meta " << - "WHERE table_id = " << quote << file_schema.table_id_ << ";"; - StoreQueryResult res = updateTableFileQuery.store(); - assert(res && res.num_rows() <= 1); - if (res.num_rows() == 1) { - int state = res[0]["state"]; - if (state == TableSchema::TO_DELETE) { + //if the table has been deleted, just mark the table file as TO_DELETE + //clean thread will delete the file later + updateTableFileQuery << "SELECT state FROM Tables " << + "WHERE table_id = " << quote << file_schema.table_id_ << ";"; + StoreQueryResult res = updateTableFileQuery.store(); + + assert(res && res.num_rows() <= 1); + if (res.num_rows() == 1) { + int state = res[0]["state"]; + if (state == TableSchema::TO_DELETE) { + file_schema.file_type_ = TableFileSchema::TO_DELETE; + } + } else { file_schema.file_type_ = TableFileSchema::TO_DELETE; } - } - else { - file_schema.file_type_ = TableFileSchema::TO_DELETE; - } - std::string id = std::to_string(file_schema.id_); - std::string table_id = file_schema.table_id_; - std::string engine_type = std::to_string(file_schema.engine_type_); - std::string file_id = file_schema.file_id_; - std::string file_type = std::to_string(file_schema.file_type_); - std::string size = std::to_string(file_schema.size_); - std::string updated_time = std::to_string(file_schema.updated_time_); - std::string created_on = std::to_string(file_schema.created_on_); - std::string date = std::to_string(file_schema.date_); + std::string id = std::to_string(file_schema.id_); + std::string table_id = file_schema.table_id_; + std::string engine_type = std::to_string(file_schema.engine_type_); + std::string file_id = file_schema.file_id_; + std::string file_type = std::to_string(file_schema.file_type_); + std::string size = std::to_string(file_schema.size_); + std::string updated_time = std::to_string(file_schema.updated_time_); + std::string created_on = std::to_string(file_schema.created_on_); + std::string date = std::to_string(file_schema.date_); - updateTableFileQuery << "UPDATE metaFile " << - "SET table_id = " << quote << table_id << ", " << - "engine_type = " << engine_type << ", " << - "file_id = " << quote << file_id << ", " << - "file_type = " << file_type << ", " << - "size = " << size << ", " << - "updated_time = " << updated_time << ", " << - "created_on = " << created_on << ", " << - "date = " << date << " " << - "WHERE id = " << id << ";"; + updateTableFileQuery << "UPDATE TableFiles " << + "SET table_id = " << quote << table_id << ", " << + "engine_type = " << engine_type << ", " << + "file_id = " << quote << file_id << ", " << + "file_type = " << file_type << ", " << + "size = " << size << ", " << + "updated_time = " << updated_time << ", " << + "created_on = " << created_on << ", " << + "date = " << date << " " << + "WHERE id = " << id << ";"; // std::cout << updateTableFileQuery.str() << std::endl; - if (!updateTableFileQuery.exec()) { - ENGINE_LOG_DEBUG << "table_id= " << file_schema.table_id_ << " file_id=" << file_schema.file_id_; - ENGINE_LOG_ERROR << "QUERY ERROR WHEN UPDATING TABLE FILE"; - return Status::DBTransactionError("QUERY ERROR WHEN UPDATING TABLE FILE", updateTableFileQuery.error()); - } + if (!updateTableFileQuery.exec()) { + ENGINE_LOG_DEBUG << "table_id= " << file_schema.table_id_ << " file_id=" << file_schema.file_id_; + ENGINE_LOG_ERROR << "QUERY ERROR WHEN UPDATING TABLE FILE"; + return Status::DBTransactionError("QUERY ERROR WHEN UPDATING TABLE FILE", + updateTableFileQuery.error()); + } + } //Scoped Connection } catch (const BadQuery& er) { // Handle any query errors @@ -1234,63 +1285,65 @@ namespace meta { try { MetricCollector metric; - ScopedConnection connectionPtr(*mySQLConnectionPool_, safe_grab); + { + ScopedConnection connectionPtr(*mySQLConnectionPool_, safe_grab); - Query updateTableFilesQuery = connectionPtr->query(); + Query updateTableFilesQuery = connectionPtr->query(); - std::map has_tables; - for (auto &file_schema : files) { + std::map has_tables; + for (auto &file_schema : files) { - if(has_tables.find(file_schema.table_id_) != has_tables.end()) { - continue; + if (has_tables.find(file_schema.table_id_) != has_tables.end()) { + continue; + } + + updateTableFilesQuery << "SELECT EXISTS " << + "(SELECT 1 FROM Tables " << + "WHERE table_id = " << quote << file_schema.table_id_ << " " << + "AND state <> " << std::to_string(TableSchema::TO_DELETE) << ") " << + "AS " << quote << "check" << ";"; + StoreQueryResult res = updateTableFilesQuery.store(); + + assert(res && res.num_rows() == 1); + int check = res[0]["check"]; + has_tables[file_schema.table_id_] = (check == 1); } - updateTableFilesQuery << "SELECT EXISTS " << - "(SELECT 1 FROM meta " << - "WHERE table_id = " << quote << file_schema.table_id_ << " " << - "AND state <> " << std::to_string(TableSchema::TO_DELETE) << ") " << - "AS " << quote << "check" << ";"; - StoreQueryResult res = updateTableFilesQuery.store(); + for (auto &file_schema : files) { - assert(res && res.num_rows() == 1); - int check = res[0]["check"]; - has_tables[file_schema.table_id_] = (check == 1); - } + if (!has_tables[file_schema.table_id_]) { + file_schema.file_type_ = TableFileSchema::TO_DELETE; + } + file_schema.updated_time_ = utils::GetMicroSecTimeStamp(); - for (auto& file_schema : files) { + std::string id = std::to_string(file_schema.id_); + std::string table_id = file_schema.table_id_; + std::string engine_type = std::to_string(file_schema.engine_type_); + std::string file_id = file_schema.file_id_; + std::string file_type = std::to_string(file_schema.file_type_); + std::string size = std::to_string(file_schema.size_); + std::string updated_time = std::to_string(file_schema.updated_time_); + std::string created_on = std::to_string(file_schema.created_on_); + std::string date = std::to_string(file_schema.date_); - if(!has_tables[file_schema.table_id_]) { - file_schema.file_type_ = TableFileSchema::TO_DELETE; + updateTableFilesQuery << "UPDATE TableFiles " << + "SET table_id = " << quote << table_id << ", " << + "engine_type = " << engine_type << ", " << + "file_id = " << quote << file_id << ", " << + "file_type = " << file_type << ", " << + "size = " << size << ", " << + "updated_time = " << updated_time << ", " << + "created_on = " << created_on << ", " << + "date = " << date << " " << + "WHERE id = " << id << ";"; + + if (!updateTableFilesQuery.exec()) { + ENGINE_LOG_ERROR << "QUERY ERROR WHEN UPDATING TABLE FILES"; + return Status::DBTransactionError("QUERY ERROR WHEN UPDATING TABLE FILES", + updateTableFilesQuery.error()); + } } - file_schema.updated_time_ = utils::GetMicroSecTimeStamp(); - - std::string id = std::to_string(file_schema.id_); - std::string table_id = file_schema.table_id_; - std::string engine_type = std::to_string(file_schema.engine_type_); - std::string file_id = file_schema.file_id_; - std::string file_type = std::to_string(file_schema.file_type_); - std::string size = std::to_string(file_schema.size_); - std::string updated_time = std::to_string(file_schema.updated_time_); - std::string created_on = std::to_string(file_schema.created_on_); - std::string date = std::to_string(file_schema.date_); - - updateTableFilesQuery << "UPDATE metaFile " << - "SET table_id = " << quote << table_id << ", " << - "engine_type = " << engine_type << ", " << - "file_id = " << quote << file_id << ", " << - "file_type = " << file_type << ", " << - "size = " << size << ", " << - "updated_time = " << updated_time << ", " << - "created_on = " << created_on << ", " << - "date = " << date << " " << - "WHERE id = " << id << ";"; - - } - - if (!updateTableFilesQuery.exec()) { - ENGINE_LOG_ERROR << "QUERY ERROR WHEN UPDATING TABLE FILES"; - return Status::DBTransactionError("QUERY ERROR WHEN UPDATING TABLE FILES", updateTableFilesQuery.error()); - } + } //Scoped Connection } catch (const BadQuery& er) { // Handle any query errors @@ -1314,54 +1367,57 @@ namespace meta { try { MetricCollector metric; - ScopedConnection connectionPtr(*mySQLConnectionPool_, safe_grab); + { + ScopedConnection connectionPtr(*mySQLConnectionPool_, safe_grab); - Query cleanUpFilesWithTTLQuery = connectionPtr->query(); - cleanUpFilesWithTTLQuery << "SELECT id, table_id, file_id, date " << - "FROM metaFile " << - "WHERE file_type = " << std::to_string(TableFileSchema::TO_DELETE) << " AND " << - "updated_time < " << std::to_string(now - seconds * US_PS) << ";"; - StoreQueryResult res = cleanUpFilesWithTTLQuery.store(); + Query cleanUpFilesWithTTLQuery = connectionPtr->query(); + cleanUpFilesWithTTLQuery << "SELECT id, table_id, file_id, date " << + "FROM TableFiles " << + "WHERE file_type = " << std::to_string(TableFileSchema::TO_DELETE) << " AND " << + "updated_time < " << std::to_string(now - seconds * US_PS) << ";"; + StoreQueryResult res = cleanUpFilesWithTTLQuery.store(); - assert(res); + assert(res); - TableFileSchema table_file; - std::vector idsToDelete; + TableFileSchema table_file; + std::vector idsToDelete; - for (auto& resRow : res) { + for (auto &resRow : res) { - table_file.id_ = resRow["id"]; //implicit conversion + table_file.id_ = resRow["id"]; //implicit conversion - std::string table_id; - resRow["table_id"].to_string(table_id); - table_file.table_id_ = table_id; + std::string table_id; + resRow["table_id"].to_string(table_id); + table_file.table_id_ = table_id; - std::string file_id; - resRow["file_id"].to_string(file_id); - table_file.file_id_ = file_id; + std::string file_id; + resRow["file_id"].to_string(file_id); + table_file.file_id_ = file_id; - table_file.date_ = resRow["date"]; + table_file.date_ = resRow["date"]; - GetTableFilePath(table_file); + GetTableFilePath(table_file); - ENGINE_LOG_DEBUG << "Removing deleted id =" << table_file.id_ << " location = " << table_file.location_ << std::endl; - boost::filesystem::remove(table_file.location_); + ENGINE_LOG_DEBUG << "Removing deleted id =" << table_file.id_ << " location = " + << table_file.location_ << std::endl; + boost::filesystem::remove(table_file.location_); - idsToDelete.emplace_back(std::to_string(table_file.id_)); - } + idsToDelete.emplace_back(std::to_string(table_file.id_)); + } - std::stringstream idsToDeleteSS; - for (auto& id : idsToDelete) { - idsToDeleteSS << "id = " << id << " OR "; - } - std::string idsToDeleteStr = idsToDeleteSS.str(); - idsToDeleteStr = idsToDeleteStr.substr(0, idsToDeleteStr.size() - 4); //remove the last " OR " - cleanUpFilesWithTTLQuery << "DELETE FROM metaFile WHERE " << - idsToDeleteStr << ";"; - if (!cleanUpFilesWithTTLQuery.exec()) { - ENGINE_LOG_ERROR << "QUERY ERROR WHEN CLEANING UP FILES WITH TTL"; - return Status::DBTransactionError("CleanUpFilesWithTTL Error", cleanUpFilesWithTTLQuery.error()); - } + std::stringstream idsToDeleteSS; + for (auto &id : idsToDelete) { + idsToDeleteSS << "id = " << id << " OR "; + } + std::string idsToDeleteStr = idsToDeleteSS.str(); + idsToDeleteStr = idsToDeleteStr.substr(0, idsToDeleteStr.size() - 4); //remove the last " OR " + cleanUpFilesWithTTLQuery << "DELETE FROM TableFiles WHERE " << + idsToDeleteStr << ";"; + if (!cleanUpFilesWithTTLQuery.exec()) { + ENGINE_LOG_ERROR << "QUERY ERROR WHEN CLEANING UP FILES WITH TTL"; + return Status::DBTransactionError("CleanUpFilesWithTTL Error", cleanUpFilesWithTTLQuery.error()); + } + } //Scoped Connection } catch (const BadQuery& er) { // Handle any query errors @@ -1376,37 +1432,39 @@ namespace meta { try { MetricCollector metric; - ScopedConnection connectionPtr(*mySQLConnectionPool_, safe_grab); + { + ScopedConnection connectionPtr(*mySQLConnectionPool_, safe_grab); - Query cleanUpFilesWithTTLQuery = connectionPtr->query(); - cleanUpFilesWithTTLQuery << "SELECT id, table_id " << - "FROM meta " << - "WHERE state = " << std::to_string(TableSchema::TO_DELETE) << ";"; - StoreQueryResult res = cleanUpFilesWithTTLQuery.store(); - assert(res); + Query cleanUpFilesWithTTLQuery = connectionPtr->query(); + cleanUpFilesWithTTLQuery << "SELECT id, table_id " << + "FROM Tables " << + "WHERE state = " << std::to_string(TableSchema::TO_DELETE) << ";"; + StoreQueryResult res = cleanUpFilesWithTTLQuery.store(); + assert(res); // std::cout << res.num_rows() << std::endl; - std::stringstream idsToDeleteSS; - for (auto& resRow : res) { - size_t id = resRow["id"]; - std::string table_id; - resRow["table_id"].to_string(table_id); + std::stringstream idsToDeleteSS; + for (auto &resRow : res) { + size_t id = resRow["id"]; + std::string table_id; + resRow["table_id"].to_string(table_id); - auto table_path = GetTablePath(table_id); + auto table_path = GetTablePath(table_id); - ENGINE_LOG_DEBUG << "Remove table folder: " << table_path; - boost::filesystem::remove_all(table_path); - - idsToDeleteSS << "id = " << std::to_string(id) << " OR "; - } - std::string idsToDeleteStr = idsToDeleteSS.str(); - idsToDeleteStr = idsToDeleteStr.substr(0, idsToDeleteStr.size() - 4); //remove the last " OR " - cleanUpFilesWithTTLQuery << "DELETE FROM meta WHERE " << - idsToDeleteStr << ";"; - if (!cleanUpFilesWithTTLQuery.exec()) { - ENGINE_LOG_ERROR << "QUERY ERROR WHEN CLEANING UP FILES WITH TTL"; - return Status::DBTransactionError("QUERY ERROR WHEN CLEANING UP FILES WITH TTL", cleanUpFilesWithTTLQuery.error()); - } + ENGINE_LOG_DEBUG << "Remove table folder: " << table_path; + boost::filesystem::remove_all(table_path); + idsToDeleteSS << "id = " << std::to_string(id) << " OR "; + } + std::string idsToDeleteStr = idsToDeleteSS.str(); + idsToDeleteStr = idsToDeleteStr.substr(0, idsToDeleteStr.size() - 4); //remove the last " OR " + cleanUpFilesWithTTLQuery << "DELETE FROM Tables WHERE " << + idsToDeleteStr << ";"; + if (!cleanUpFilesWithTTLQuery.exec()) { + ENGINE_LOG_ERROR << "QUERY ERROR WHEN CLEANING UP FILES WITH TTL"; + return Status::DBTransactionError("QUERY ERROR WHEN CLEANING UP FILES WITH TTL", + cleanUpFilesWithTTLQuery.error()); + } + } //Scoped Connection } catch (const BadQuery& er) { // Handle any query errors @@ -1430,7 +1488,7 @@ namespace meta { ENGINE_LOG_DEBUG << "Remove table file type as NEW"; Query cleanUpQuery = connectionPtr->query(); - cleanUpQuery << "DELETE FROM metaFile WHERE file_type = " << std::to_string(TableFileSchema::NEW) << ";"; + cleanUpQuery << "DELETE FROM TableFiles WHERE file_type = " << std::to_string(TableFileSchema::NEW) << ";"; if (!cleanUpQuery.exec()) { ENGINE_LOG_ERROR << "QUERY ERROR WHEN CLEANING UP FILES"; @@ -1457,17 +1515,6 @@ namespace meta { try { MetricCollector metric; - ScopedConnection connectionPtr(*mySQLConnectionPool_, safe_grab); - - Query countQuery = connectionPtr->query(); - countQuery << "SELECT size " << - "FROM metaFile " << - "WHERE table_id = " << quote << table_id << " AND " << - "(file_type = " << std::to_string(TableFileSchema::RAW) << " OR " << - "file_type = " << std::to_string(TableFileSchema::TO_INDEX) << " OR " << - "file_type = " << std::to_string(TableFileSchema::INDEX) << ");"; - StoreQueryResult res = countQuery.store(); - TableSchema table_schema; table_schema.table_id_ = table_id; auto status = DescribeTable(table_schema); @@ -1476,6 +1523,21 @@ namespace meta { return status; } + StoreQueryResult res; + + { + ScopedConnection connectionPtr(*mySQLConnectionPool_, safe_grab); + + Query countQuery = connectionPtr->query(); + countQuery << "SELECT size " << + "FROM TableFiles " << + "WHERE table_id = " << quote << table_id << " AND " << + "(file_type = " << std::to_string(TableFileSchema::RAW) << " OR " << + "file_type = " << std::to_string(TableFileSchema::TO_INDEX) << " OR " << + "file_type = " << std::to_string(TableFileSchema::INDEX) << ");"; + res = countQuery.store(); + } //Scoped Connection + result = 0; for (auto &resRow : res) { size_t size = resRow["size"]; @@ -1510,7 +1572,7 @@ namespace meta { ScopedConnection connectionPtr(*mySQLConnectionPool_, safe_grab); Query dropTableQuery = connectionPtr->query(); - dropTableQuery << "DROP TABLE IF EXISTS meta, metaFile;"; + dropTableQuery << "DROP TABLE IF EXISTS Tables, TableFiles;"; if (dropTableQuery.exec()) { return Status::OK(); } diff --git a/cpp/thirdparty/versions.txt b/cpp/thirdparty/versions.txt index 8e63a4b5c9..311760948d 100644 --- a/cpp/thirdparty/versions.txt +++ b/cpp/thirdparty/versions.txt @@ -7,7 +7,7 @@ GTEST_VERSION=1.8.1 JSONCONS_VERSION=0.126.0 LAPACK_VERSION=v3.8.0 LZ4_VERSION=v1.9.1 -MYSQLPP_VERSION=zilliz +MYSQLPP_VERSION=3.2.4 OPENBLAS_VERSION=v0.3.6 PROMETHEUS_VERSION=v0.7.0 ROCKSDB_VERSION=v6.0.2 From 7bd1871f818f6dcad607c35c10ab7f7069320b8d Mon Sep 17 00:00:00 2001 From: zhiru Date: Wed, 26 Jun 2019 13:54:29 +0800 Subject: [PATCH 16/43] clear debug log Former-commit-id: a956a0f8d55bcb51d826f109ef2dc77746b09611 --- cpp/src/db/MySQLConnectionPool.h | 6 +++--- cpp/src/db/MySQLMetaImpl.cpp | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/cpp/src/db/MySQLConnectionPool.h b/cpp/src/db/MySQLConnectionPool.h index 7d3a1f88a3..bd047d56f8 100644 --- a/cpp/src/db/MySQLConnectionPool.h +++ b/cpp/src/db/MySQLConnectionPool.h @@ -45,7 +45,7 @@ public: sleep(1); } - ENGINE_LOG_DEBUG << "conns_in_use_ in grab: " << conns_in_use_ << std::endl; +// ENGINE_LOG_DEBUG << "conns_in_use_ in grab: " << conns_in_use_ << std::endl; ++conns_in_use_; return mysqlpp::ConnectionPool::grab(); } @@ -53,10 +53,10 @@ public: // Other half of in-use conn count limit void release(const mysqlpp::Connection* pc) override { mysqlpp::ConnectionPool::release(pc); - ENGINE_LOG_DEBUG << "conns_in_use_ in release: " << conns_in_use_ << std::endl; +// ENGINE_LOG_DEBUG << "conns_in_use_ in release: " << conns_in_use_ << std::endl; --conns_in_use_; if (conns_in_use_ < 0) { - ENGINE_LOG_DEBUG << "conns_in_use_ in release < 0: " << conns_in_use_ << std::endl; + ENGINE_LOG_ERROR << "conns_in_use_ in release less than zero: " << conns_in_use_ << std::endl; } } diff --git a/cpp/src/db/MySQLMetaImpl.cpp b/cpp/src/db/MySQLMetaImpl.cpp index 6b733ec5ae..c860c667c4 100644 --- a/cpp/src/db/MySQLMetaImpl.cpp +++ b/cpp/src/db/MySQLMetaImpl.cpp @@ -326,7 +326,7 @@ namespace meta { } else { createTableQuery << "SELECT state FROM Tables " << "WHERE table_id = " << quote << table_schema.table_id_ << ";"; - ENGINE_LOG_DEBUG << "Create Table : " << createTableQuery.str(); +// ENGINE_LOG_DEBUG << "Create Table : " << createTableQuery.str(); StoreQueryResult res = createTableQuery.store(); assert(res && res.num_rows() <= 1); if (res.num_rows() == 1) { @@ -338,7 +338,7 @@ namespace meta { return Status::Error(msg); } } - ENGINE_LOG_DEBUG << "Create Table start"; +// ENGINE_LOG_DEBUG << "Create Table start"; table_schema.files_cnt_ = 0; table_schema.id_ = -1; @@ -358,7 +358,7 @@ namespace meta { createTableQuery << "INSERT INTO Tables VALUES" << "(" << id << ", " << quote << table_id << ", " << state << ", " << dimension << ", " << created_on << ", " << files_cnt << ", " << engine_type << ", " << store_raw_data << ");"; - ENGINE_LOG_DEBUG << "Create Table : " << createTableQuery.str(); +// ENGINE_LOG_DEBUG << "Create Table : " << createTableQuery.str(); if (SimpleResult res = createTableQuery.execute()) { table_schema.id_ = res.insert_id(); //Might need to use SELECT LAST_INSERT_ID()? // std::cout << table_schema.id_ << std::endl; From e11cdcfe0fe526fe8f9b2cf61da2d241cc34f8aa Mon Sep 17 00:00:00 2001 From: zhiru Date: Wed, 26 Jun 2019 14:07:04 +0800 Subject: [PATCH 17/43] change conns_in_use_ in connection pool to signed int Former-commit-id: 4c4c2f661deb6f70c2c0b3b0bbe1a67ec182a4aa --- cpp/src/db/MySQLConnectionPool.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/src/db/MySQLConnectionPool.h b/cpp/src/db/MySQLConnectionPool.h index bd047d56f8..63ae65829d 100644 --- a/cpp/src/db/MySQLConnectionPool.h +++ b/cpp/src/db/MySQLConnectionPool.h @@ -94,7 +94,7 @@ protected: private: // Number of connections currently in use - unsigned int conns_in_use_; + int conns_in_use_; // Our connection parameters std::string db_, user_, password_, server_; From af07abbcd1554694a24410582fbbf13cb4889df7 Mon Sep 17 00:00:00 2001 From: zhiru Date: Thu, 27 Jun 2019 11:03:09 +0800 Subject: [PATCH 18/43] change mysql++.so installlation dir to bin/lib Former-commit-id: 2e62c096c62602697b3d8fc8cddcd4ec00e43d59 --- cpp/src/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/src/CMakeLists.txt b/cpp/src/CMakeLists.txt index 3a955e0654..6c1bd12771 100644 --- a/cpp/src/CMakeLists.txt +++ b/cpp/src/CMakeLists.txt @@ -187,6 +187,6 @@ install(FILES ${CMAKE_BINARY_DIR}/mysqlpp_ep-prefix/src/mysqlpp_ep/lib/${CMAKE_SHARED_LIBRARY_PREFIX}mysqlpp${CMAKE_SHARED_LIBRARY_SUFFIX} ${CMAKE_BINARY_DIR}/mysqlpp_ep-prefix/src/mysqlpp_ep/lib/${CMAKE_SHARED_LIBRARY_PREFIX}mysqlpp${CMAKE_SHARED_LIBRARY_SUFFIX}.3 ${CMAKE_BINARY_DIR}/mysqlpp_ep-prefix/src/mysqlpp_ep/lib/${CMAKE_SHARED_LIBRARY_PREFIX}mysqlpp${CMAKE_SHARED_LIBRARY_SUFFIX}.3.2.4 - DESTINATION bin) #need to copy libmysqlpp.so + DESTINATION bin/lib) #need to copy libmysqlpp.so add_subdirectory(sdk) From 1d1da7f9677c4707cfbe2e1861e7670a675c5ded Mon Sep 17 00:00:00 2001 From: zhiru Date: Thu, 27 Jun 2019 14:10:02 +0800 Subject: [PATCH 19/43] fix Former-commit-id: 6665c0dacfefd19b23cf42e340a6ff8d3b751cd1 --- cpp/src/db/MySQLMetaImpl.cpp | 73 +++++++++++++++++++-------------- cpp/src/server/DBWrapper.cpp | 2 +- cpp/src/server/MilvusServer.cpp | 7 ++-- cpp/src/server/RequestTask.cpp | 70 ++++++++++++++++--------------- 4 files changed, 84 insertions(+), 68 deletions(-) diff --git a/cpp/src/db/MySQLMetaImpl.cpp b/cpp/src/db/MySQLMetaImpl.cpp index ad291b6dbc..116636be76 100644 --- a/cpp/src/db/MySQLMetaImpl.cpp +++ b/cpp/src/db/MySQLMetaImpl.cpp @@ -961,6 +961,10 @@ namespace meta { // std::lock_guard lock(mysql_mutex); + if (ids.empty()) { + return Status::OK(); + } + std::stringstream idSS; for (auto& id : ids) { idSS << "id = " << std::to_string(id) << " OR "; @@ -1405,17 +1409,22 @@ namespace meta { idsToDelete.emplace_back(std::to_string(table_file.id_)); } - std::stringstream idsToDeleteSS; - for (auto &id : idsToDelete) { - idsToDeleteSS << "id = " << id << " OR "; - } - std::string idsToDeleteStr = idsToDeleteSS.str(); - idsToDeleteStr = idsToDeleteStr.substr(0, idsToDeleteStr.size() - 4); //remove the last " OR " - cleanUpFilesWithTTLQuery << "DELETE FROM TableFiles WHERE " << - idsToDeleteStr << ";"; - if (!cleanUpFilesWithTTLQuery.exec()) { - ENGINE_LOG_ERROR << "QUERY ERROR WHEN CLEANING UP FILES WITH TTL"; - return Status::DBTransactionError("CleanUpFilesWithTTL Error", cleanUpFilesWithTTLQuery.error()); + if (!idsToDelete.empty()) { + + std::stringstream idsToDeleteSS; + for (auto &id : idsToDelete) { + idsToDeleteSS << "id = " << id << " OR "; + } + + std::string idsToDeleteStr = idsToDeleteSS.str(); + idsToDeleteStr = idsToDeleteStr.substr(0, idsToDeleteStr.size() - 4); //remove the last " OR " + cleanUpFilesWithTTLQuery << "DELETE FROM TableFiles WHERE " << + idsToDeleteStr << ";"; + if (!cleanUpFilesWithTTLQuery.exec()) { + ENGINE_LOG_ERROR << "QUERY ERROR WHEN CLEANING UP FILES WITH TTL"; + return Status::DBTransactionError("CleanUpFilesWithTTL Error", + cleanUpFilesWithTTLQuery.error()); + } } } //Scoped Connection @@ -1442,29 +1451,33 @@ namespace meta { StoreQueryResult res = cleanUpFilesWithTTLQuery.store(); assert(res); // std::cout << res.num_rows() << std::endl; - std::stringstream idsToDeleteSS; - for (auto &resRow : res) { - size_t id = resRow["id"]; - std::string table_id; - resRow["table_id"].to_string(table_id); - auto table_path = GetTablePath(table_id); + if (!res.empty()) { - ENGINE_LOG_DEBUG << "Remove table folder: " << table_path; - boost::filesystem::remove_all(table_path); + std::stringstream idsToDeleteSS; + for (auto &resRow : res) { + size_t id = resRow["id"]; + std::string table_id; + resRow["table_id"].to_string(table_id); - idsToDeleteSS << "id = " << std::to_string(id) << " OR "; + auto table_path = GetTablePath(table_id); + + ENGINE_LOG_DEBUG << "Remove table folder: " << table_path; + boost::filesystem::remove_all(table_path); + + idsToDeleteSS << "id = " << std::to_string(id) << " OR "; + } + std::string idsToDeleteStr = idsToDeleteSS.str(); + idsToDeleteStr = idsToDeleteStr.substr(0, idsToDeleteStr.size() - 4); //remove the last " OR " + cleanUpFilesWithTTLQuery << "DELETE FROM Tables WHERE " << + idsToDeleteStr << ";"; + if (!cleanUpFilesWithTTLQuery.exec()) { + ENGINE_LOG_ERROR << "QUERY ERROR WHEN CLEANING UP FILES WITH TTL"; + return Status::DBTransactionError("QUERY ERROR WHEN CLEANING UP FILES WITH TTL", + cleanUpFilesWithTTLQuery.error()); + } } - std::string idsToDeleteStr = idsToDeleteSS.str(); - idsToDeleteStr = idsToDeleteStr.substr(0, idsToDeleteStr.size() - 4); //remove the last " OR " - cleanUpFilesWithTTLQuery << "DELETE FROM Tables WHERE " << - idsToDeleteStr << ";"; - if (!cleanUpFilesWithTTLQuery.exec()) { - ENGINE_LOG_ERROR << "QUERY ERROR WHEN CLEANING UP FILES WITH TTL"; - return Status::DBTransactionError("QUERY ERROR WHEN CLEANING UP FILES WITH TTL", - cleanUpFilesWithTTLQuery.error()); - } - } //Scoped Connection + } //Scoped Connection } catch (const BadQuery& er) { // Handle any query errors diff --git a/cpp/src/server/DBWrapper.cpp b/cpp/src/server/DBWrapper.cpp index 4cae31ea6b..0721de1325 100644 --- a/cpp/src/server/DBWrapper.cpp +++ b/cpp/src/server/DBWrapper.cpp @@ -28,7 +28,7 @@ DBWrapper::DBWrapper() { zilliz::milvus::engine::DB::Open(opt, &db_); if(db_ == nullptr) { - SERVER_LOG_ERROR << "Failed to open db"; + SERVER_LOG_ERROR << "Failed to open db. Provided database uri = " << opt.meta.backend_uri; throw ServerException(SERVER_NULL_POINTER, "Failed to open db"); } } diff --git a/cpp/src/server/MilvusServer.cpp b/cpp/src/server/MilvusServer.cpp index 5e42eb3d7a..1b1b85e883 100644 --- a/cpp/src/server/MilvusServer.cpp +++ b/cpp/src/server/MilvusServer.cpp @@ -8,6 +8,7 @@ #include "ServerConfig.h" #include "ThreadPoolServer.h" #include "DBWrapper.h" +#include "utils/Log.h" #include "milvus_types.h" #include "milvus_constants.h" @@ -67,7 +68,7 @@ MilvusServer::StartService() { } else if (protocol == "compact") { protocol_factory.reset(new TCompactProtocolFactory()); } else { - //SERVER_LOG_INFO << "Service protocol: " << protocol << " is not supported currently"; + SERVER_LOG_ERROR << "Service protocol: " << protocol << " is not supported currently"; return; } @@ -88,11 +89,11 @@ MilvusServer::StartService() { threadManager)); s_server->serve(); } else { - //SERVER_LOG_INFO << "Service mode: " << mode << " is not supported currently"; + SERVER_LOG_ERROR << "Service mode: " << mode << " is not supported currently"; return; } } catch (apache::thrift::TException& ex) { - //SERVER_LOG_ERROR << "Server encounter exception: " << ex.what(); + SERVER_LOG_ERROR << "Server encounter exception: " << ex.what(); } } diff --git a/cpp/src/server/RequestTask.cpp b/cpp/src/server/RequestTask.cpp index dbe4e6a740..77396d0046 100644 --- a/cpp/src/server/RequestTask.cpp +++ b/cpp/src/server/RequestTask.cpp @@ -140,7 +140,9 @@ ServerError CreateTableTask::OnExecute() { //step 1: check arguments if(schema_.table_name.empty() || schema_.dimension <= 0) { error_code_ = SERVER_INVALID_ARGUMENT; - error_msg_ = "Invalid table name or dimension"; +// error_msg_ = schema_.table_name.empty() ? + error_msg_ = "CreateTableTask: Invalid table name or dimension. table name = " + schema_.table_name + + "dimension = " + std::to_string(schema_.dimension); SERVER_LOG_ERROR << error_msg_; return error_code_; } @@ -148,7 +150,7 @@ ServerError CreateTableTask::OnExecute() { engine::EngineType engine_type = EngineType(schema_.index_type); if(engine_type == engine::EngineType::INVALID) { error_code_ = SERVER_INVALID_ARGUMENT; - error_msg_ = "Invalid index type"; + error_msg_ = "CreateTableTask: Invalid index type. type = " + std::to_string(schema_.index_type); SERVER_LOG_ERROR << error_msg_; return error_code_; } @@ -164,7 +166,7 @@ ServerError CreateTableTask::OnExecute() { engine::Status stat = DBWrapper::DB()->CreateTable(table_info); if(!stat.ok()) {//table could exist error_code_ = SERVER_UNEXPECTED_ERROR; - error_msg_ = "Engine failed: " + stat.ToString(); + error_msg_ = "CreateTableTask: Engine failed: " + stat.ToString(); SERVER_LOG_ERROR << error_msg_; return error_code_; } @@ -172,7 +174,7 @@ ServerError CreateTableTask::OnExecute() { } catch (std::exception& ex) { error_code_ = SERVER_UNEXPECTED_ERROR; error_msg_ = ex.what(); - SERVER_LOG_ERROR << error_msg_; + SERVER_LOG_ERROR << "CreateTableTask: " << error_msg_; return error_code_; } @@ -200,7 +202,7 @@ ServerError DescribeTableTask::OnExecute() { //step 1: check arguments if(table_name_.empty()) { error_code_ = SERVER_INVALID_ARGUMENT; - error_msg_ = "Table name cannot be empty"; + error_msg_ = "DescribeTableTask: Table name cannot be empty"; SERVER_LOG_ERROR << error_msg_; return error_code_; } @@ -211,7 +213,7 @@ ServerError DescribeTableTask::OnExecute() { engine::Status stat = DBWrapper::DB()->DescribeTable(table_info); if(!stat.ok()) { error_code_ = SERVER_TABLE_NOT_EXIST; - error_msg_ = "Engine failed: " + stat.ToString(); + error_msg_ = "DescribeTableTask: Engine failed: " + stat.ToString(); SERVER_LOG_ERROR << error_msg_; return error_code_; } @@ -224,7 +226,7 @@ ServerError DescribeTableTask::OnExecute() { } catch (std::exception& ex) { error_code_ = SERVER_UNEXPECTED_ERROR; error_msg_ = ex.what(); - SERVER_LOG_ERROR << error_msg_; + SERVER_LOG_ERROR << "DescribeTableTask: " << error_msg_; return SERVER_UNEXPECTED_ERROR; } @@ -251,7 +253,7 @@ ServerError DeleteTableTask::OnExecute() { //step 1: check arguments if (table_name_.empty()) { error_code_ = SERVER_INVALID_ARGUMENT; - error_msg_ = "Table name cannot be empty"; + error_msg_ = "DeleteTableTask: Table name cannot be empty"; SERVER_LOG_ERROR << error_msg_; return error_code_; } @@ -262,7 +264,7 @@ ServerError DeleteTableTask::OnExecute() { engine::Status stat = DBWrapper::DB()->DescribeTable(table_info); if(!stat.ok()) { error_code_ = SERVER_TABLE_NOT_EXIST; - error_msg_ = "Engine failed: " + stat.ToString(); + error_msg_ = "DeleteTableTask: Engine failed: " + stat.ToString(); SERVER_LOG_ERROR << error_msg_; return error_code_; } @@ -273,16 +275,16 @@ ServerError DeleteTableTask::OnExecute() { std::vector dates; stat = DBWrapper::DB()->DeleteTable(table_name_, dates); if(!stat.ok()) { - SERVER_LOG_ERROR << "Engine failed: " << stat.ToString(); + SERVER_LOG_ERROR << "DeleteTableTask: Engine failed: " << stat.ToString(); return SERVER_UNEXPECTED_ERROR; } rc.Record("deleta table"); - rc.Elapse("totally cost"); + rc.Elapse("total cost"); } catch (std::exception& ex) { error_code_ = SERVER_UNEXPECTED_ERROR; error_msg_ = ex.what(); - SERVER_LOG_ERROR << error_msg_; + SERVER_LOG_ERROR << "DeleteTableTask: " << error_msg_; return error_code_; } @@ -305,7 +307,7 @@ ServerError ShowTablesTask::OnExecute() { engine::Status stat = DBWrapper::DB()->AllTables(schema_array); if(!stat.ok()) { error_code_ = SERVER_UNEXPECTED_ERROR; - error_msg_ = "Engine failed: " + stat.ToString(); + error_msg_ = "ShowTablesTask: Engine failed: " + stat.ToString(); SERVER_LOG_ERROR << error_msg_; return error_code_; } @@ -342,14 +344,14 @@ ServerError AddVectorTask::OnExecute() { //step 1: check arguments if (table_name_.empty()) { error_code_ = SERVER_INVALID_ARGUMENT; - error_msg_ = "Table name cannot be empty"; + error_msg_ = "AddVectorTask: Table name cannot be empty"; SERVER_LOG_ERROR << error_msg_; return error_code_; } if(record_array_.empty()) { error_code_ = SERVER_INVALID_ARGUMENT; - error_msg_ = "Row record array is empty"; + error_msg_ = "AddVectorTask: Row record array is empty"; SERVER_LOG_ERROR << error_msg_; return error_code_; } @@ -360,7 +362,7 @@ ServerError AddVectorTask::OnExecute() { engine::Status stat = DBWrapper::DB()->DescribeTable(table_info); if(!stat.ok()) { error_code_ = SERVER_TABLE_NOT_EXIST; - error_msg_ = "Engine failed: " + stat.ToString(); + error_msg_ = "AddVectorTask: Engine failed when DescribeTable: " + stat.ToString(); SERVER_LOG_ERROR << error_msg_; return error_code_; } @@ -371,7 +373,7 @@ ServerError AddVectorTask::OnExecute() { std::vector vec_f; error_code_ = ConvertRowRecordToFloatArray(record_array_, table_info.dimension_, vec_f); if(error_code_ != SERVER_SUCCESS) { - error_msg_ = "Invalid row record data"; + error_msg_ = "AddVectorTask when ConvertRowRecordToFloatArray: Invalid row record data"; return error_code_; } @@ -383,23 +385,23 @@ ServerError AddVectorTask::OnExecute() { rc.Record("add vectors to engine"); if(!stat.ok()) { error_code_ = SERVER_UNEXPECTED_ERROR; - error_msg_ = "Engine failed: " + stat.ToString(); + error_msg_ = "AddVectorTask: Engine failed when InsertVectors: " + stat.ToString(); SERVER_LOG_ERROR << error_msg_; return error_code_; } if(record_ids_.size() != vec_count) { - SERVER_LOG_ERROR << "Vector ID not returned"; + SERVER_LOG_ERROR << "AddVectorTask: Vector ID not returned"; return SERVER_UNEXPECTED_ERROR; } rc.Record("do insert"); - rc.Elapse("totally cost"); + rc.Elapse("total cost"); } catch (std::exception& ex) { error_code_ = SERVER_UNEXPECTED_ERROR; error_msg_ = ex.what(); - SERVER_LOG_ERROR << error_msg_; + SERVER_LOG_ERROR << "AddVectorTask: " << error_msg_; return error_code_; } @@ -440,14 +442,14 @@ ServerError SearchVectorTask::OnExecute() { //step 1: check arguments if (table_name_.empty()) { error_code_ = SERVER_INVALID_ARGUMENT; - error_msg_ = "Table name cannot be empty"; + error_msg_ = "SearchVectorTask: Table name cannot be empty"; SERVER_LOG_ERROR << error_msg_; return error_code_; } if(top_k_ <= 0 || record_array_.empty()) { error_code_ = SERVER_INVALID_ARGUMENT; - error_msg_ = "Invalid topk value, or query record array is empty"; + error_msg_ = "SearchVectorTask: Invalid topk value, or query record array is empty"; SERVER_LOG_ERROR << error_msg_; return error_code_; } @@ -458,7 +460,7 @@ ServerError SearchVectorTask::OnExecute() { engine::Status stat = DBWrapper::DB()->DescribeTable(table_info); if(!stat.ok()) { error_code_ = SERVER_TABLE_NOT_EXIST; - error_msg_ = "Engine failed: " + stat.ToString(); + error_msg_ = "SearchVectorTask: Engine failed when DescribeTable: " + stat.ToString(); SERVER_LOG_ERROR << error_msg_; return error_code_; } @@ -467,7 +469,7 @@ ServerError SearchVectorTask::OnExecute() { std::vector dates; error_code_ = ConvertTimeRangeToDBDates(range_array_, dates); if(error_code_ != SERVER_SUCCESS) { - error_msg_ = "Invalid query range"; + error_msg_ = "SearchVectorTask: Invalid query range when ConvertTimeRangeToDBDates"; return error_code_; } @@ -477,7 +479,7 @@ ServerError SearchVectorTask::OnExecute() { std::vector vec_f; error_code_ = ConvertRowRecordToFloatArray(record_array_, table_info.dimension_, vec_f); if(error_code_ != SERVER_SUCCESS) { - error_msg_ = "Invalid row record data"; + error_msg_ = "Invalid row record data when ConvertRowRecordToFloatArray"; return error_code_; } @@ -495,12 +497,12 @@ ServerError SearchVectorTask::OnExecute() { rc.Record("search vectors from engine"); if(!stat.ok()) { - SERVER_LOG_ERROR << "Engine failed: " << stat.ToString(); + SERVER_LOG_ERROR << "SearchVectorTask: Engine failed: " << stat.ToString(); return SERVER_UNEXPECTED_ERROR; } if(results.size() != record_count) { - SERVER_LOG_ERROR << "Search result not returned"; + SERVER_LOG_ERROR << "SearchVectorTask: Search result not returned"; return SERVER_UNEXPECTED_ERROR; } @@ -523,12 +525,12 @@ ServerError SearchVectorTask::OnExecute() { result_array_.emplace_back(thrift_topk_result); } rc.Record("construct result"); - rc.Elapse("totally cost"); + rc.Elapse("total cost"); } catch (std::exception& ex) { error_code_ = SERVER_UNEXPECTED_ERROR; error_msg_ = ex.what(); - SERVER_LOG_ERROR << error_msg_; + SERVER_LOG_ERROR << "SearchVectorTask: " << error_msg_; return error_code_; } @@ -554,7 +556,7 @@ ServerError GetTableRowCountTask::OnExecute() { //step 1: check arguments if (table_name_.empty()) { error_code_ = SERVER_INVALID_ARGUMENT; - error_msg_ = "Table name cannot be empty"; + error_msg_ = "GetTableRowCountTask: Table name cannot be empty"; SERVER_LOG_ERROR << error_msg_; return error_code_; } @@ -564,19 +566,19 @@ ServerError GetTableRowCountTask::OnExecute() { engine::Status stat = DBWrapper::DB()->GetTableRowCount(table_name_, row_count); if (!stat.ok()) { error_code_ = SERVER_UNEXPECTED_ERROR; - error_msg_ = "Engine failed: " + stat.ToString(); + error_msg_ = "GetTableRowCountTask: Engine failed: " + stat.ToString(); SERVER_LOG_ERROR << error_msg_; return error_code_; } row_count_ = (int64_t) row_count; - rc.Elapse("totally cost"); + rc.Elapse("total cost"); } catch (std::exception& ex) { error_code_ = SERVER_UNEXPECTED_ERROR; error_msg_ = ex.what(); - SERVER_LOG_ERROR << error_msg_; + SERVER_LOG_ERROR << "GetTableRowCountTask: " << error_msg_; return error_code_; } From bd4977b64f5dbcfc647c6e7c6fbab6cf87118c8a Mon Sep 17 00:00:00 2001 From: zhiru Date: Thu, 27 Jun 2019 14:52:21 +0800 Subject: [PATCH 20/43] update Former-commit-id: 23e7315c4445705a07cac462f4fa6ffae337d133 --- cpp/src/db/DBImpl.cpp | 8 +++++++- cpp/src/db/Factories.cpp | 2 ++ cpp/src/db/Options.h | 1 + cpp/src/server/DBWrapper.cpp | 1 + cpp/src/server/ServerConfig.h | 1 + 5 files changed, 12 insertions(+), 1 deletion(-) diff --git a/cpp/src/db/DBImpl.cpp b/cpp/src/db/DBImpl.cpp index 17280b37d3..e39942d35d 100644 --- a/cpp/src/db/DBImpl.cpp +++ b/cpp/src/db/DBImpl.cpp @@ -13,6 +13,7 @@ #include "scheduler/context/SearchContext.h" #include "scheduler/context/DeleteContext.h" #include "utils/TimeRecorder.h" +#include "MetaConsts.h" #include #include @@ -595,7 +596,12 @@ void DBImpl::BackgroundCompaction(std::set table_ids) { } meta_ptr_->Archive(); - meta_ptr_->CleanUpFilesWithTTL(1); + + int ttl = 1; + if (options_.mode == "cluster") { + ttl = meta::D_SEC; + } + meta_ptr_->CleanUpFilesWithTTL(ttl); } void DBImpl::StartBuildIndexTask() { diff --git a/cpp/src/db/Factories.cpp b/cpp/src/db/Factories.cpp index fd005d1ff5..a6998d10dc 100644 --- a/cpp/src/db/Factories.cpp +++ b/cpp/src/db/Factories.cpp @@ -81,9 +81,11 @@ std::shared_ptr DBMetaImplFactory::Build(const DBMetaOptions& metaOp std::string dialect = pieces_match[1].str(); std::transform(dialect.begin(), dialect.end(), dialect.begin(), ::tolower); if (dialect.find("mysql") != std::string::npos) { + ENGINE_LOG_DEBUG << "Using MySQL"; return std::make_shared(meta::MySQLMetaImpl(metaOptions)); } else if (dialect.find("sqlite") != std::string::npos) { + ENGINE_LOG_DEBUG << "Using SQLite"; return std::make_shared(meta::DBMetaImpl(metaOptions)); } else { diff --git a/cpp/src/db/Options.h b/cpp/src/db/Options.h index ddb7d1bff5..039134ebaa 100644 --- a/cpp/src/db/Options.h +++ b/cpp/src/db/Options.h @@ -47,6 +47,7 @@ struct Options { uint16_t merge_trigger_number = 2; size_t index_trigger_size = ONE_GB; //unit: byte DBMetaOptions meta; + std::string mode; }; // Options diff --git a/cpp/src/server/DBWrapper.cpp b/cpp/src/server/DBWrapper.cpp index 0721de1325..c38c5d8843 100644 --- a/cpp/src/server/DBWrapper.cpp +++ b/cpp/src/server/DBWrapper.cpp @@ -23,6 +23,7 @@ DBWrapper::DBWrapper() { if(index_size > 0) {//ensure larger than zero, unit is MB opt.index_trigger_size = (size_t)index_size * engine::ONE_MB; } + opt.mode = config.GetValue(CONFIG_CLUSTER_MODE, "single"); CommonUtil::CreateDirectory(opt.meta.path); diff --git a/cpp/src/server/ServerConfig.h b/cpp/src/server/ServerConfig.h index dd7c9d2966..aa77b081d7 100644 --- a/cpp/src/server/ServerConfig.h +++ b/cpp/src/server/ServerConfig.h @@ -19,6 +19,7 @@ static const std::string CONFIG_SERVER_ADDRESS = "address"; static const std::string CONFIG_SERVER_PORT = "port"; static const std::string CONFIG_SERVER_PROTOCOL = "transfer_protocol"; static const std::string CONFIG_SERVER_MODE = "server_mode"; +static const std::string CONFIG_CLUSTER_MODE = "mode"; static const std::string CONFIG_DB = "db_config"; static const std::string CONFIG_DB_URL = "db_backend_url"; From 9b74b7ae4183f8dffcd8318da4438ff9a1263a37 Mon Sep 17 00:00:00 2001 From: zhiru Date: Thu, 27 Jun 2019 15:52:22 +0800 Subject: [PATCH 21/43] update Former-commit-id: c7a11335bbf73b5100243b31ca34c36257604909 --- cpp/src/db/DBImpl.cpp | 1 + cpp/src/db/MySQLConnectionPool.h | 2 +- cpp/src/db/MySQLMetaImpl.cpp | 5 +++-- cpp/src/server/DBWrapper.cpp | 4 +++- 4 files changed, 8 insertions(+), 4 deletions(-) diff --git a/cpp/src/db/DBImpl.cpp b/cpp/src/db/DBImpl.cpp index e39942d35d..c7ee986c96 100644 --- a/cpp/src/db/DBImpl.cpp +++ b/cpp/src/db/DBImpl.cpp @@ -600,6 +600,7 @@ void DBImpl::BackgroundCompaction(std::set table_ids) { int ttl = 1; if (options_.mode == "cluster") { ttl = meta::D_SEC; + ENGINE_LOG_DEBUG << "Server mode is cluster. Clean up files with ttl = " << std::to_string(ttl) << "seconds."; } meta_ptr_->CleanUpFilesWithTTL(ttl); } diff --git a/cpp/src/db/MySQLConnectionPool.h b/cpp/src/db/MySQLConnectionPool.h index 98b7f67895..ebb2c5eb54 100644 --- a/cpp/src/db/MySQLConnectionPool.h +++ b/cpp/src/db/MySQLConnectionPool.h @@ -56,7 +56,7 @@ public: // ENGINE_LOG_DEBUG << "conns_in_use_ in release: " << conns_in_use_ << std::endl; --conns_in_use_; if (conns_in_use_ < 0) { - ENGINE_LOG_ERROR << "MySQLConnetionPool::release: conns_in_use_ is less than zero. conns_in_use_ = " << conns_in_use_ << std::endl; + ENGINE_LOG_WARNING << "MySQLConnetionPool::release: conns_in_use_ is less than zero. conns_in_use_ = " << conns_in_use_ << std::endl; } } diff --git a/cpp/src/db/MySQLMetaImpl.cpp b/cpp/src/db/MySQLMetaImpl.cpp index 116636be76..40ff30b88a 100644 --- a/cpp/src/db/MySQLMetaImpl.cpp +++ b/cpp/src/db/MySQLMetaImpl.cpp @@ -448,9 +448,10 @@ namespace meta { Query deleteTableFilesQuery = connectionPtr->query(); // deleteTableFilesQuery << "UPDATE TableFiles " << - "SET state = " << std::to_string(TableSchema::TO_DELETE) << ", " << + "SET file_type = " << std::to_string(TableSchema::TO_DELETE) << ", " << "updated_time = " << std::to_string(utils::GetMicroSecTimeStamp()) << " " << - "WHERE table_id = " << quote << table_id << ";"; + "WHERE table_id = " << quote << table_id << " AND " << + "file_type <> " << std::to_string(TableSchema::TO_DELETE) << ";"; if (!deleteTableFilesQuery.exec()) { ENGINE_LOG_ERROR << "QUERY ERROR WHEN DELETING TABLE FILES"; diff --git a/cpp/src/server/DBWrapper.cpp b/cpp/src/server/DBWrapper.cpp index c38c5d8843..6ad308a8d6 100644 --- a/cpp/src/server/DBWrapper.cpp +++ b/cpp/src/server/DBWrapper.cpp @@ -23,7 +23,9 @@ DBWrapper::DBWrapper() { if(index_size > 0) {//ensure larger than zero, unit is MB opt.index_trigger_size = (size_t)index_size * engine::ONE_MB; } - opt.mode = config.GetValue(CONFIG_CLUSTER_MODE, "single"); + ConfigNode& serverConfig = ServerConfig::GetInstance().GetConfig(CONFIG_SERVER); + opt.mode = serverConfig.GetValue(CONFIG_CLUSTER_MODE, "single"); +// std::cout << "mode = " << opt.mode << std::endl; CommonUtil::CreateDirectory(opt.meta.path); From 669e134fa148fca7c95b945f74af0a4063abda21 Mon Sep 17 00:00:00 2001 From: zhiru Date: Thu, 27 Jun 2019 16:54:16 +0800 Subject: [PATCH 22/43] temporarily disable decrementing conns_in_use_ when it's already <= 0 Former-commit-id: 4025643a777ae3566bee289d6d5db5c80409a6e5 --- cpp/conf/log_config.conf | 4 ++-- cpp/conf/server_config.yaml | 6 +++--- cpp/src/db/MySQLConnectionPool.h | 6 ++++-- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/cpp/conf/log_config.conf b/cpp/conf/log_config.conf index 80710b570e..79a3965719 100644 --- a/cpp/conf/log_config.conf +++ b/cpp/conf/log_config.conf @@ -20,8 +20,8 @@ TO_STANDARD_OUTPUT = false ## Error logs * ERROR: - ENABLED = false + ENABLED = true FILENAME = "/tmp/milvus/logs/milvus-%datetime{%H:%m}-error.log" * FATAL: - ENABLED = false + ENABLED = true FILENAME = "/tmp/milvus/logs/milvus-%datetime{%H:%m}-fatal.log" \ No newline at end of file diff --git a/cpp/conf/server_config.yaml b/cpp/conf/server_config.yaml index bcf9d2116d..33858c9455 100644 --- a/cpp/conf/server_config.yaml +++ b/cpp/conf/server_config.yaml @@ -1,15 +1,15 @@ server_config: address: 0.0.0.0 - port: 19530 # the port milvus listen to, default: 19530, range: 1025 ~ 65534 + port: 19531 # the port milvus listen to, default: 19530, range: 1025 ~ 65534 gpu_index: 0 # the gpu milvus use, default: 0, range: 0 ~ gpu number - 1 - mode: single # milvus deployment type: single, cluster + mode: cluster # milvus deployment type: single, cluster db_config: db_path: /tmp/milvus # milvus data storage path #URI format: dialect://username:password@host:port/database #All parts except dialect are optional, but you MUST include the delimiters #Currently supports mysql or sqlite - db_backend_url: dialect://username:password@host:port/database # meta database uri + db_backend_url: mysql://root:1234@:/test # meta database uri index_building_threshold: 1024 # index building trigger threshold, default: 1024, unit: MB metric_config: diff --git a/cpp/src/db/MySQLConnectionPool.h b/cpp/src/db/MySQLConnectionPool.h index ebb2c5eb54..8a240102dc 100644 --- a/cpp/src/db/MySQLConnectionPool.h +++ b/cpp/src/db/MySQLConnectionPool.h @@ -54,10 +54,12 @@ public: void release(const mysqlpp::Connection* pc) override { mysqlpp::ConnectionPool::release(pc); // ENGINE_LOG_DEBUG << "conns_in_use_ in release: " << conns_in_use_ << std::endl; - --conns_in_use_; - if (conns_in_use_ < 0) { + if (conns_in_use_ <= 0) { ENGINE_LOG_WARNING << "MySQLConnetionPool::release: conns_in_use_ is less than zero. conns_in_use_ = " << conns_in_use_ << std::endl; } + else { + --conns_in_use_; + } } void set_max_idle_time(int max_idle) { From d2c515eeeb41cc5432232cc3876bc46f06803682 Mon Sep 17 00:00:00 2001 From: zhiru Date: Thu, 27 Jun 2019 21:17:07 +0800 Subject: [PATCH 23/43] update Former-commit-id: d702986a5a998695905d3c0faad9591e244ea2a8 --- cpp/src/server/DBWrapper.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cpp/src/server/DBWrapper.cpp b/cpp/src/server/DBWrapper.cpp index 7892a57f2b..84a0f4797d 100644 --- a/cpp/src/server/DBWrapper.cpp +++ b/cpp/src/server/DBWrapper.cpp @@ -23,6 +23,9 @@ DBWrapper::DBWrapper() { if(index_size > 0) {//ensure larger than zero, unit is MB opt.index_trigger_size = (size_t)index_size * engine::ONE_MB; } + ConfigNode& serverConfig = ServerConfig::GetInstance().GetConfig(CONFIG_SERVER); + opt.mode = serverConfig.GetValue(CONFIG_CLUSTER_MODE, "single"); + //set archive config engine::ArchiveConf::CriteriaT criterial; From 13232ec79275b92016099b9c15159e1cabe4b7f3 Mon Sep 17 00:00:00 2001 From: zhiru Date: Fri, 28 Jun 2019 11:08:10 +0800 Subject: [PATCH 24/43] update Former-commit-id: a8d0615a42384353a3d99f4765e5e6c320e2d245 --- cpp/src/db/DBImpl.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/src/db/DBImpl.cpp b/cpp/src/db/DBImpl.cpp index c7ee986c96..b028dd3ea3 100644 --- a/cpp/src/db/DBImpl.cpp +++ b/cpp/src/db/DBImpl.cpp @@ -600,7 +600,7 @@ void DBImpl::BackgroundCompaction(std::set table_ids) { int ttl = 1; if (options_.mode == "cluster") { ttl = meta::D_SEC; - ENGINE_LOG_DEBUG << "Server mode is cluster. Clean up files with ttl = " << std::to_string(ttl) << "seconds."; +// ENGINE_LOG_DEBUG << "Server mode is cluster. Clean up files with ttl = " << std::to_string(ttl) << "seconds."; } meta_ptr_->CleanUpFilesWithTTL(ttl); } From 1ad8a04d8edd56b5a2499371a03f43ba4d585819 Mon Sep 17 00:00:00 2001 From: zhiru Date: Fri, 28 Jun 2019 15:28:31 +0800 Subject: [PATCH 25/43] update Former-commit-id: a393a1e7552cf59edf5d1f12633f686aee286414 --- cpp/src/db/DBImpl.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cpp/src/db/DBImpl.cpp b/cpp/src/db/DBImpl.cpp index b028dd3ea3..fec7035c1f 100644 --- a/cpp/src/db/DBImpl.cpp +++ b/cpp/src/db/DBImpl.cpp @@ -139,7 +139,9 @@ DBImpl::DBImpl(const Options& options) meta_ptr_ = DBMetaImplFactory::Build(options.meta); mem_mgr_ = std::make_shared(meta_ptr_, options_); // mem_mgr_ = (MemManagerPtr)(new MemManager(meta_ptr_, options_)); - StartTimerTasks(); + if (options.mode != "read_only") { + StartTimerTasks(); + } } Status DBImpl::CreateTable(meta::TableSchema& table_schema) { From 20abc4d2dc6f3e7fc7dd9fb9c3f3c42c53369b92 Mon Sep 17 00:00:00 2001 From: zhiru Date: Sun, 30 Jun 2019 10:43:06 +0800 Subject: [PATCH 26/43] update Former-commit-id: 798f3825230e84b917ed0ad4fe2fde556427e4e5 --- cpp/conf/server_config.yaml | 4 ++-- cpp/src/db/DBImpl.cpp | 2 +- cpp/src/db/Factories.cpp | 11 ++++++----- cpp/src/db/Factories.h | 2 +- cpp/src/db/MySQLMetaImpl.cpp | 16 +++++++++++----- cpp/src/db/MySQLMetaImpl.h | 3 ++- cpp/src/server/DBWrapper.cpp | 5 ++++- 7 files changed, 27 insertions(+), 16 deletions(-) diff --git a/cpp/conf/server_config.yaml b/cpp/conf/server_config.yaml index 5fe8096435..c937c57833 100644 --- a/cpp/conf/server_config.yaml +++ b/cpp/conf/server_config.yaml @@ -1,8 +1,8 @@ server_config: address: 0.0.0.0 - port: 19531 # the port milvus listen to, default: 19530, range: 1025 ~ 65534 + port: 19530 # the port milvus listen to, default: 19530, range: 1025 ~ 65534 gpu_index: 0 # the gpu milvus use, default: 0, range: 0 ~ gpu number - 1 - mode: cluster # milvus deployment type: single, cluster + mode: single # milvus deployment type: single, cluster, read_only db_config: db_path: /tmp/milvus # milvus data storage path diff --git a/cpp/src/db/DBImpl.cpp b/cpp/src/db/DBImpl.cpp index fec7035c1f..def216358b 100644 --- a/cpp/src/db/DBImpl.cpp +++ b/cpp/src/db/DBImpl.cpp @@ -136,7 +136,7 @@ DBImpl::DBImpl(const Options& options) shutting_down_(false), compact_thread_pool_(1, 1), index_thread_pool_(1, 1) { - meta_ptr_ = DBMetaImplFactory::Build(options.meta); + meta_ptr_ = DBMetaImplFactory::Build(options.meta, options.mode); mem_mgr_ = std::make_shared(meta_ptr_, options_); // mem_mgr_ = (MemManagerPtr)(new MemManager(meta_ptr_, options_)); if (options.mode != "read_only") { diff --git a/cpp/src/db/Factories.cpp b/cpp/src/db/Factories.cpp index 15e188d3db..a4abb6e4f1 100644 --- a/cpp/src/db/Factories.cpp +++ b/cpp/src/db/Factories.cpp @@ -53,7 +53,8 @@ std::shared_ptr DBMetaImplFactory::Build() { return std::shared_ptr(new meta::DBMetaImpl(options)); } -std::shared_ptr DBMetaImplFactory::Build(const DBMetaOptions& metaOptions) { +std::shared_ptr DBMetaImplFactory::Build(const DBMetaOptions& metaOptions, + const std::string& mode) { std::string uri = metaOptions.backend_uri; // if (uri.empty()) { @@ -82,20 +83,20 @@ std::shared_ptr DBMetaImplFactory::Build(const DBMetaOptions& metaOp std::transform(dialect.begin(), dialect.end(), dialect.begin(), ::tolower); if (dialect.find("mysql") != std::string::npos) { ENGINE_LOG_INFO << "Using MySQL"; - return std::make_shared(meta::MySQLMetaImpl(metaOptions)); + return std::make_shared(meta::MySQLMetaImpl(metaOptions, mode)); } else if (dialect.find("sqlite") != std::string::npos) { ENGINE_LOG_INFO << "Using SQLite"; return std::make_shared(meta::DBMetaImpl(metaOptions)); } else { - LOG(ERROR) << "Invalid dialect in URI: dialect = " << dialect; + ENGINE_LOG_ERROR << "Invalid dialect in URI: dialect = " << dialect; throw InvalidArgumentException("URI dialect is not mysql / sqlite"); } } else { - LOG(ERROR) << "Wrong URI format: URI = " << uri; - throw InvalidArgumentException("Wrong URI format"); + ENGINE_LOG_ERROR << "Wrong URI format: URI = " << uri; + throw InvalidArgumentException("Wrong URI format "); } } diff --git a/cpp/src/db/Factories.h b/cpp/src/db/Factories.h index 31afd2b5ba..48bea9b291 100644 --- a/cpp/src/db/Factories.h +++ b/cpp/src/db/Factories.h @@ -28,7 +28,7 @@ struct OptionsFactory { struct DBMetaImplFactory { static std::shared_ptr Build(); - static std::shared_ptr Build(const DBMetaOptions& metaOptions); + static std::shared_ptr Build(const DBMetaOptions& metaOptions, const std::string& mode); }; struct DBFactory { diff --git a/cpp/src/db/MySQLMetaImpl.cpp b/cpp/src/db/MySQLMetaImpl.cpp index b0e505aac3..86283de85e 100644 --- a/cpp/src/db/MySQLMetaImpl.cpp +++ b/cpp/src/db/MySQLMetaImpl.cpp @@ -103,8 +103,9 @@ namespace meta { return Status::OK(); } - MySQLMetaImpl::MySQLMetaImpl(const DBMetaOptions &options_) - : options_(options_) { + MySQLMetaImpl::MySQLMetaImpl(const DBMetaOptions &options_, const std::string& mode) + : options_(options_), + mode_(mode) { Initialize(); } @@ -439,7 +440,12 @@ namespace meta { } //Scoped Connection - DeleteTableFiles(table_id); + +// ConfigNode& serverConfig = ServerConfig::GetInstance().GetConfig(CONFIG_SERVER); +// opt.mode = serverConfig.GetValue(CONFIG_CLUSTER_MODE, "single"); + if (mode_ != "single") { + DeleteTableFiles(table_id); + } } catch (const BadQuery& er) { // Handle any query errors @@ -469,10 +475,10 @@ namespace meta { Query deleteTableFilesQuery = connectionPtr->query(); // deleteTableFilesQuery << "UPDATE TableFiles " << - "SET file_type = " << std::to_string(TableSchema::TO_DELETE) << ", " << + "SET file_type = " << std::to_string(TableFileSchema::TO_DELETE) << ", " << "updated_time = " << std::to_string(utils::GetMicroSecTimeStamp()) << " " << "WHERE table_id = " << quote << table_id << " AND " << - "file_type <> " << std::to_string(TableSchema::TO_DELETE) << ";"; + "file_type <> " << std::to_string(TableFileSchema::TO_DELETE) << ";"; if (!deleteTableFilesQuery.exec()) { ENGINE_LOG_ERROR << "QUERY ERROR WHEN DELETING TABLE FILES"; diff --git a/cpp/src/db/MySQLMetaImpl.h b/cpp/src/db/MySQLMetaImpl.h index 033fa99453..4805076d45 100644 --- a/cpp/src/db/MySQLMetaImpl.h +++ b/cpp/src/db/MySQLMetaImpl.h @@ -22,7 +22,7 @@ namespace meta { class MySQLMetaImpl : public Meta { public: - MySQLMetaImpl(const DBMetaOptions& options_); + MySQLMetaImpl(const DBMetaOptions& options_, const std::string& mode); virtual Status CreateTable(TableSchema& table_schema) override; virtual Status DescribeTable(TableSchema& group_info_) override; @@ -77,6 +77,7 @@ namespace meta { Status Initialize(); const DBMetaOptions options_; + const std::string mode_; std::shared_ptr mySQLConnectionPool_; bool safe_grab = false; diff --git a/cpp/src/server/DBWrapper.cpp b/cpp/src/server/DBWrapper.cpp index 84a0f4797d..ceaa6e8130 100644 --- a/cpp/src/server/DBWrapper.cpp +++ b/cpp/src/server/DBWrapper.cpp @@ -25,7 +25,10 @@ DBWrapper::DBWrapper() { } ConfigNode& serverConfig = ServerConfig::GetInstance().GetConfig(CONFIG_SERVER); opt.mode = serverConfig.GetValue(CONFIG_CLUSTER_MODE, "single"); - + if (opt.mode != "single" && opt.mode != "cluster" && opt.mode != "read_only") { + std::cout << "ERROR: mode specified in server_config is not one of ['single', 'cluster', 'read_only']" << std::endl; + kill(0, SIGUSR1); + } //set archive config engine::ArchiveConf::CriteriaT criterial; From 60cd1729d325259d07717e9db324bbb9195b58fb Mon Sep 17 00:00:00 2001 From: groot Date: Sun, 30 Jun 2019 11:54:57 +0800 Subject: [PATCH 27/43] add more unitest Former-commit-id: a97158e5a0c0a719fa4d688f2cc2ef010a15838b --- cpp/src/db/scheduler/TaskDispatchQueue.cpp | 14 -- cpp/src/db/scheduler/TaskDispatchStrategy.cpp | 18 ++- cpp/unittest/db/CMakeLists.txt | 5 +- cpp/unittest/db/scheduler_test.cpp | 124 ++++++++++++++++++ 4 files changed, 138 insertions(+), 23 deletions(-) create mode 100644 cpp/unittest/db/scheduler_test.cpp diff --git a/cpp/src/db/scheduler/TaskDispatchQueue.cpp b/cpp/src/db/scheduler/TaskDispatchQueue.cpp index 2ce0e933b4..b728e925a9 100644 --- a/cpp/src/db/scheduler/TaskDispatchQueue.cpp +++ b/cpp/src/db/scheduler/TaskDispatchQueue.cpp @@ -24,14 +24,6 @@ TaskDispatchQueue::Put(const ScheduleContextPtr &context) { return; } - if (queue_.size() >= capacity_) { - std::string error_msg = - "blocking queue is full, capacity: " + std::to_string(capacity_) + " queue_size: " + - std::to_string(queue_.size()); - SERVER_LOG_ERROR << error_msg; - throw server::ServerException(server::SERVER_BLOCKING_QUEUE_EMPTY, error_msg); - } - TaskDispatchStrategy::Schedule(context, queue_); empty_.notify_all(); @@ -42,12 +34,6 @@ TaskDispatchQueue::Take() { std::unique_lock lock(mtx); empty_.wait(lock, [this] { return !queue_.empty(); }); - if (queue_.empty()) { - std::string error_msg = "blocking queue empty"; - SERVER_LOG_ERROR << error_msg; - throw server::ServerException(server::SERVER_BLOCKING_QUEUE_EMPTY, error_msg); - } - ScheduleTaskPtr front(queue_.front()); queue_.pop_front(); full_.notify_all(); diff --git a/cpp/src/db/scheduler/TaskDispatchStrategy.cpp b/cpp/src/db/scheduler/TaskDispatchStrategy.cpp index 7200f2584f..985f86cb09 100644 --- a/cpp/src/db/scheduler/TaskDispatchStrategy.cpp +++ b/cpp/src/db/scheduler/TaskDispatchStrategy.cpp @@ -74,20 +74,26 @@ public: } std::string table_id = context->table_id(); - for(auto iter = task_list.begin(); iter != task_list.end(); ++iter) { + + //put delete task to proper position + //for example: task_list has 10 IndexLoadTask, only the No.5 IndexLoadTask is for table1 + //if user want to delete table1, the DeleteTask will be insert into No.6 position + for(std::list::reverse_iterator iter = task_list.rbegin(); iter != task_list.rend(); ++iter) { if((*iter)->type() != ScheduleTaskType::kIndexLoad) { continue; } - //put delete task to proper position IndexLoadTaskPtr loader = std::static_pointer_cast(*iter); - if(loader->file_->table_id_ == table_id) { - - task_list.insert(++iter, delete_task); - break; + if(loader->file_->table_id_ != table_id) { + continue; } + + task_list.insert(iter.base(), delete_task); + return true; } + //no task is searching this table, put DeleteTask to front of list so that the table will be delete asap + task_list.push_front(delete_task); return true; } }; diff --git a/cpp/unittest/db/CMakeLists.txt b/cpp/unittest/db/CMakeLists.txt index c9b7ea7dcc..8427639ab9 100644 --- a/cpp/unittest/db/CMakeLists.txt +++ b/cpp/unittest/db/CMakeLists.txt @@ -7,6 +7,7 @@ aux_source_directory(${MILVUS_ENGINE_SRC}/db db_srcs) aux_source_directory(${MILVUS_ENGINE_SRC}/config config_files) aux_source_directory(${MILVUS_ENGINE_SRC}/cache cache_srcs) aux_source_directory(${MILVUS_ENGINE_SRC}/wrapper wrapper_src) +aux_source_directory(./ test_srcs) aux_source_directory(${MILVUS_ENGINE_SRC}/db/scheduler scheduler_files) aux_source_directory(${MILVUS_ENGINE_SRC}/db/scheduler/context scheduler_context_files) @@ -30,9 +31,7 @@ set(db_test_src ${db_scheduler_srcs} ${wrapper_src} ${require_files} - utils.cpp - db_tests.cpp - meta_tests.cpp) + ${test_srcs}) cuda_add_executable(db_test ${db_test_src}) diff --git a/cpp/unittest/db/scheduler_test.cpp b/cpp/unittest/db/scheduler_test.cpp new file mode 100644 index 0000000000..01a7057e00 --- /dev/null +++ b/cpp/unittest/db/scheduler_test.cpp @@ -0,0 +1,124 @@ +//////////////////////////////////////////////////////////////////////////////// +// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved +// Unauthorized copying of this file, via any medium is strictly prohibited. +// Proprietary and confidential. +//////////////////////////////////////////////////////////////////////////////// +#include +#include +#include +#include + +#include "db/scheduler/TaskScheduler.h" +#include "db/scheduler/TaskDispatchStrategy.h" +#include "db/scheduler/TaskDispatchQueue.h" +#include "db/scheduler/task/SearchTask.h" +#include "db/scheduler/task/DeleteTask.h" +#include "db/scheduler/task/IndexLoadTask.h" + +using namespace zilliz::milvus; + +namespace { + +engine::TableFileSchemaPtr CreateTabileFileStruct(size_t id, const std::string& table_id) { + auto file = std::make_shared(); + file->id_ = id; + file->table_id_ = table_id; + return file; +} + +} + +TEST(DBSchedulerTest, TASK_QUEUE_TEST) { + engine::TaskDispatchQueue queue; + queue.SetCapacity(1000); + queue.Put(nullptr); + ASSERT_EQ(queue.Size(), 1UL); + + auto ptr = queue.Take(); + ASSERT_EQ(ptr, nullptr); + ASSERT_TRUE(queue.Empty()); + + engine::SearchContextPtr context_ptr = std::make_shared(1, 1, nullptr); + for(size_t i = 0; i < 10; i++) { + auto file = CreateTabileFileStruct(i, "tbl"); + context_ptr->AddIndexFile(file); + } + + queue.Put(context_ptr); + ASSERT_EQ(queue.Size(), 10); + + auto index_files = context_ptr->GetIndexMap(); + + ptr = queue.Front(); + ASSERT_EQ(ptr->type(), engine::ScheduleTaskType::kIndexLoad); + engine::IndexLoadTaskPtr load_task = std::static_pointer_cast(ptr); + ASSERT_EQ(load_task->file_->id_, index_files.begin()->first); + + ptr = queue.Back(); + ASSERT_EQ(ptr->type(), engine::ScheduleTaskType::kIndexLoad); +} + +TEST(DBSchedulerTest, SEARCH_SCHEDULER_TEST) { + std::list task_list; + bool ret = engine::TaskDispatchStrategy::Schedule(nullptr, task_list); + ASSERT_FALSE(ret); + + for(size_t i = 10; i < 30; i++) { + engine::IndexLoadTaskPtr task_ptr = std::make_shared(); + task_ptr->file_ = CreateTabileFileStruct(i, "tbl"); + task_list.push_back(task_ptr); + } + + engine::SearchContextPtr context_ptr = std::make_shared(1, 1, nullptr); + for(size_t i = 0; i < 20; i++) { + auto file = CreateTabileFileStruct(i, "tbl"); + context_ptr->AddIndexFile(file); + } + + ret = engine::TaskDispatchStrategy::Schedule(context_ptr, task_list); + ASSERT_TRUE(ret); + ASSERT_EQ(task_list.size(), 30); +} + +TEST(DBSchedulerTest, DELETE_SCHEDULER_TEST) { + std::list task_list; + bool ret = engine::TaskDispatchStrategy::Schedule(nullptr, task_list); + ASSERT_FALSE(ret); + + const std::string table_id = "to_delete_table"; + for(size_t i = 0; i < 10; i++) { + engine::IndexLoadTaskPtr task_ptr = std::make_shared(); + task_ptr->file_ = CreateTabileFileStruct(i, table_id); + task_list.push_back(task_ptr); + } + + for(size_t i = 0; i < 10; i++) { + engine::IndexLoadTaskPtr task_ptr = std::make_shared(); + task_ptr->file_ = CreateTabileFileStruct(i, "other_table"); + task_list.push_back(task_ptr); + } + + engine::meta::Meta::Ptr meta_ptr; + engine::DeleteContextPtr context_ptr = std::make_shared(table_id, meta_ptr); + ret = engine::TaskDispatchStrategy::Schedule(context_ptr, task_list); + ASSERT_TRUE(ret); + ASSERT_EQ(task_list.size(), 21); + + auto temp_list = task_list; + for(size_t i = 0; ; i++) { + engine::ScheduleTaskPtr task_ptr = temp_list.front(); + temp_list.pop_front(); + if(task_ptr->type() == engine::ScheduleTaskType::kDelete) { + ASSERT_EQ(i, 10); + break; + } + } + + context_ptr = std::make_shared("no_task_table", meta_ptr); + ret = engine::TaskDispatchStrategy::Schedule(context_ptr, task_list); + ASSERT_TRUE(ret); + ASSERT_EQ(task_list.size(), 22); + + engine::ScheduleTaskPtr task_ptr = task_list.front(); + ASSERT_EQ(task_ptr->type(), engine::ScheduleTaskType::kDelete); +} From f08fb195c7ace3ca623f222b59b87a32b8c5cd46 Mon Sep 17 00:00:00 2001 From: zhiru Date: Sun, 30 Jun 2019 13:28:38 +0800 Subject: [PATCH 28/43] update Former-commit-id: 2825bffeb532d4022ea0a36dfa46bba387295598 --- cpp/CMakeLists.txt | 8 - cpp/mysqlNotes | 8 - cpp/src/db/DBImpl.cpp | 4 +- cpp/src/db/Factories.cpp | 14 +- cpp/src/db/Factories.h | 2 +- cpp/src/db/MySQLConnectionPool.h | 18 +- cpp/src/db/MySQLMetaImpl.cpp | 303 +++++++++++++++++++++++-------- cpp/src/db/MySQLMetaImpl.h | 6 +- cpp/src/db/Options.h | 10 +- cpp/src/server/DBWrapper.cpp | 25 ++- cpp/src/server/ServerConfig.h | 1 + 11 files changed, 271 insertions(+), 128 deletions(-) delete mode 100644 cpp/mysqlNotes diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 58124c5296..116f30026d 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -113,21 +113,13 @@ link_directories(${MILVUS_BINARY_DIR}) set(MILVUS_ENGINE_INCLUDE ${PROJECT_SOURCE_DIR}/include) set(MILVUS_ENGINE_SRC ${PROJECT_SOURCE_DIR}/src) -#set(MILVUS_THIRD_PARTY ${CMAKE_CURRENT_SOURCE_DIR}/third_party) -#set(MILVUS_THIRD_PARTY_BUILD ${CMAKE_CURRENT_SOURCE_DIR}/third_party/build) add_compile_definitions(PROFILER=${PROFILER}) include_directories(${MILVUS_ENGINE_INCLUDE}) include_directories(${MILVUS_ENGINE_SRC}) -include_directories(/usr/local/cuda/include) -#include_directories(${MILVUS_THIRD_PARTY_BUILD}/include) link_directories(${CMAKE_CURRRENT_BINARY_DIR}) -#link_directories(${MILVUS_THIRD_PARTY_BUILD}/lib) -#link_directories(${MILVUS_THIRD_PARTY_BUILD}/lib64) -#execute_process(COMMAND bash build.sh -# WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/third_party) add_subdirectory(src) diff --git a/cpp/mysqlNotes b/cpp/mysqlNotes deleted file mode 100644 index 8aa151efff..0000000000 --- a/cpp/mysqlNotes +++ /dev/null @@ -1,8 +0,0 @@ -sudo apt-get install mysql-server -sudo apt-get install libmysqlclient-dev -sudo ln -s libmysqlclient.so libmysqlclient_r.so - -Install MySQL++ -./configure --enable-thread-check LDFLAGS='-pthread' -make -sudo make install diff --git a/cpp/src/db/DBImpl.cpp b/cpp/src/db/DBImpl.cpp index def216358b..92bbff2b84 100644 --- a/cpp/src/db/DBImpl.cpp +++ b/cpp/src/db/DBImpl.cpp @@ -139,7 +139,7 @@ DBImpl::DBImpl(const Options& options) meta_ptr_ = DBMetaImplFactory::Build(options.meta, options.mode); mem_mgr_ = std::make_shared(meta_ptr_, options_); // mem_mgr_ = (MemManagerPtr)(new MemManager(meta_ptr_, options_)); - if (options.mode != "read_only") { + if (options.mode != Options::MODE::READ_ONLY) { StartTimerTasks(); } } @@ -600,7 +600,7 @@ void DBImpl::BackgroundCompaction(std::set table_ids) { meta_ptr_->Archive(); int ttl = 1; - if (options_.mode == "cluster") { + if (options_.mode == Options::MODE::CLUSTER) { ttl = meta::D_SEC; // ENGINE_LOG_DEBUG << "Server mode is cluster. Clean up files with ttl = " << std::to_string(ttl) << "seconds."; } diff --git a/cpp/src/db/Factories.cpp b/cpp/src/db/Factories.cpp index a4abb6e4f1..4b24bd3a1c 100644 --- a/cpp/src/db/Factories.cpp +++ b/cpp/src/db/Factories.cpp @@ -29,15 +29,8 @@ DBMetaOptions DBMetaOptionsFactory::Build(const std::string& path) { p = ss.str(); } -// std::string uri; -// const char* uri_p = getenv("MILVUS_DB_META_URI"); -// if (uri_p) { -// uri = uri_p; -// } - DBMetaOptions meta; meta.path = p; -// meta.backend_uri = uri; return meta; } @@ -54,14 +47,9 @@ std::shared_ptr DBMetaImplFactory::Build() { } std::shared_ptr DBMetaImplFactory::Build(const DBMetaOptions& metaOptions, - const std::string& mode) { + const int& mode) { std::string uri = metaOptions.backend_uri; -// if (uri.empty()) { -// //Default to sqlite if uri is empty -//// return std::make_shared(new meta::DBMetaImpl(metaOptions)); -// return std::shared_ptr(new meta::DBMetaImpl(metaOptions)); -// } std::string dialectRegex = "(.*)"; std::string usernameRegex = "(.*)"; diff --git a/cpp/src/db/Factories.h b/cpp/src/db/Factories.h index 48bea9b291..889922b17a 100644 --- a/cpp/src/db/Factories.h +++ b/cpp/src/db/Factories.h @@ -28,7 +28,7 @@ struct OptionsFactory { struct DBMetaImplFactory { static std::shared_ptr Build(); - static std::shared_ptr Build(const DBMetaOptions& metaOptions, const std::string& mode); + static std::shared_ptr Build(const DBMetaOptions& metaOptions, const int& mode); }; struct DBFactory { diff --git a/cpp/src/db/MySQLConnectionPool.h b/cpp/src/db/MySQLConnectionPool.h index b79089c1da..8992ba274c 100644 --- a/cpp/src/db/MySQLConnectionPool.h +++ b/cpp/src/db/MySQLConnectionPool.h @@ -21,12 +21,12 @@ public: password_(passWord), server_(serverIp), port_(port), - maxPoolSize_(maxPoolSize) + max_pool_size_(maxPoolSize) { conns_in_use_ = 0; - maxIdleTime_ = 10; //10 seconds + max_idle_time_ = 10; //10 seconds } // The destructor. We _must_ call ConnectionPool::clear() here, @@ -41,12 +41,10 @@ public: // connections actually in use, not those created. Also note that // we keep our own count; ConnectionPool::size() isn't the same! mysqlpp::Connection* grab() override { - while (conns_in_use_ > maxPoolSize_) { -// cout.put('R'); cout.flush(); // indicate waiting for release + while (conns_in_use_ > max_pool_size_) { sleep(1); } -// ENGINE_LOG_DEBUG << "conns_in_use_ in grab: " << conns_in_use_ << std::endl; ++conns_in_use_; return mysqlpp::ConnectionPool::grab(); } @@ -68,7 +66,7 @@ public: } void set_max_idle_time(int max_idle) { - maxIdleTime_ = max_idle; + max_idle_time_ = max_idle; } protected: @@ -77,7 +75,6 @@ protected: mysqlpp::Connection* create() override { // Create connection using the parameters we were passed upon // creation. -// cout.put('C'); cout.flush(); // indicate connection creation mysqlpp::Connection* conn = new mysqlpp::Connection(); conn->set_option(new mysqlpp::ReconnectOption(true)); conn->connect(db_.empty() ? 0 : db_.c_str(), @@ -91,12 +88,11 @@ protected: void destroy(mysqlpp::Connection* cp) override { // Our superclass can't know how we created the Connection, so // it delegates destruction to us, to be safe. -// cout.put('D'); cout.flush(); // indicate connection destruction delete cp; } unsigned int max_idle_time() override { - return maxIdleTime_; + return max_idle_time_; } private: @@ -107,7 +103,7 @@ private: std::string db_, user_, password_, server_; int port_; - int maxPoolSize_; + int max_pool_size_; - unsigned int maxIdleTime_; + unsigned int max_idle_time_; }; \ No newline at end of file diff --git a/cpp/src/db/MySQLMetaImpl.cpp b/cpp/src/db/MySQLMetaImpl.cpp index 86283de85e..f57bed7420 100644 --- a/cpp/src/db/MySQLMetaImpl.cpp +++ b/cpp/src/db/MySQLMetaImpl.cpp @@ -103,7 +103,7 @@ namespace meta { return Status::OK(); } - MySQLMetaImpl::MySQLMetaImpl(const DBMetaOptions &options_, const std::string& mode) + MySQLMetaImpl::MySQLMetaImpl(const DBMetaOptions &options_, const int& mode) : options_(options_), mode_(mode) { Initialize(); @@ -157,7 +157,7 @@ namespace meta { // connectionPtr->set_option(new mysqlpp::ReconnectOption(true)); int threadHint = std::thread::hardware_concurrency(); int maxPoolSize = threadHint == 0 ? 8 : threadHint; - mySQLConnectionPool_ = std::make_shared(dbName, username, password, serverAddress, port, maxPoolSize); + mysql_connection_pool_ = std::make_shared(dbName, username, password, serverAddress, port, maxPoolSize); // std::cout << "MySQL++ thread aware:" << std::to_string(connectionPtr->thread_aware()) << std::endl; ENGINE_LOG_DEBUG << "MySQL connection pool: maximum pool size = " << std::to_string(maxPoolSize); try { @@ -165,9 +165,9 @@ namespace meta { CleanUp(); { - ScopedConnection connectionPtr(*mySQLConnectionPool_, safe_grab); + ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); -// ENGINE_LOG_DEBUG << "MySQLMetaImpl::Initialize: connections in use = " << mySQLConnectionPool_->getConnectionsInUse(); +// ENGINE_LOG_DEBUG << "MySQLMetaImpl::Initialize: connections in use = " << mysql_connection_pool_->getConnectionsInUse(); // if (!connectionPtr->connect(dbName, serverAddress, username, password, port)) { // return Status::Error("DB connection failed: ", connectionPtr->error()); // } @@ -192,6 +192,11 @@ namespace meta { "files_cnt BIGINT DEFAULT 0 NOT NULL, " << "engine_type INT DEFAULT 1 NOT NULL, " << "store_raw_data BOOL DEFAULT false NOT NULL);"; + + if (options_.sql_echo) { + ENGINE_LOG_DEBUG << "MySQLMetaImpl::Initialize: " << InitializeQuery.str(); + } + if (!InitializeQuery.exec()) { return Status::DBTransactionError("Initialization Error", InitializeQuery.error()); } @@ -206,6 +211,11 @@ namespace meta { "updated_time BIGINT NOT NULL, " << "created_on BIGINT NOT NULL, " << "date INT DEFAULT -1 NOT NULL);"; + + if (options_.sql_echo) { + ENGINE_LOG_DEBUG << "MySQLMetaImpl::Initialize: " << InitializeQuery.str(); + } + if (!InitializeQuery.exec()) { return Status::DBTransactionError("Initialization Error", InitializeQuery.error()); } @@ -282,10 +292,10 @@ namespace meta { dateListStr = dateListStr.substr(0, dateListStr.size() - 2); //remove the last ", " { - ScopedConnection connectionPtr(*mySQLConnectionPool_, safe_grab); + ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); -// if (mySQLConnectionPool_->getConnectionsInUse() <= 0) { -// ENGINE_LOG_WARNING << "MySQLMetaImpl::DropPartitionsByDates connection in use = " << mySQLConnectionPool_->getConnectionsInUse(); +// if (mysql_connection_pool_->getConnectionsInUse() <= 0) { +// ENGINE_LOG_WARNING << "MySQLMetaImpl::DropPartitionsByDates connection in use = " << mysql_connection_pool_->getConnectionsInUse(); // } Query dropPartitionsByDatesQuery = connectionPtr->query(); @@ -295,6 +305,10 @@ namespace meta { "WHERE table_id = " << quote << table_id << " AND " << "date in (" << dateListStr << ");"; + if (options_.sql_echo) { + ENGINE_LOG_DEBUG << "MySQLMetaImpl::DropPartitionsByDates: " << dropPartitionsByDatesQuery.str(); + } + if (!dropPartitionsByDatesQuery.exec()) { ENGINE_LOG_ERROR << "QUERY ERROR WHEN DROPPING PARTITIONS BY DATES"; return Status::DBTransactionError("QUERY ERROR WHEN DROPPING PARTITIONS BY DATES", @@ -323,10 +337,10 @@ namespace meta { MetricCollector metric; { - ScopedConnection connectionPtr(*mySQLConnectionPool_, safe_grab); + ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); -// if (mySQLConnectionPool_->getConnectionsInUse() <= 0) { -// ENGINE_LOG_WARNING << "MySQLMetaImpl::CreateTable connection in use = " << mySQLConnectionPool_->getConnectionsInUse(); +// if (mysql_connection_pool_->getConnectionsInUse() <= 0) { +// ENGINE_LOG_WARNING << "MySQLMetaImpl::CreateTable connection in use = " << mysql_connection_pool_->getConnectionsInUse(); // } Query createTableQuery = connectionPtr->query(); @@ -337,6 +351,11 @@ namespace meta { createTableQuery << "SELECT state FROM Tables " << "WHERE table_id = " << quote << table_schema.table_id_ << ";"; // ENGINE_LOG_DEBUG << "Create Table : " << createTableQuery.str(); + + if (options_.sql_echo) { + ENGINE_LOG_DEBUG << "MySQLMetaImpl::CreateTable: " << createTableQuery.str(); + } + StoreQueryResult res = createTableQuery.store(); assert(res && res.num_rows() <= 1); if (res.num_rows() == 1) { @@ -370,6 +389,11 @@ namespace meta { "(" << id << ", " << quote << table_id << ", " << state << ", " << dimension << ", " << created_on << ", " << files_cnt << ", " << engine_type << ", " << store_raw_data << ");"; // ENGINE_LOG_DEBUG << "Create Table : " << createTableQuery.str(); + + if (options_.sql_echo) { + ENGINE_LOG_DEBUG << "MySQLMetaImpl::CreateTable: " << createTableQuery.str(); + } + if (SimpleResult res = createTableQuery.execute()) { table_schema.id_ = res.insert_id(); //Might need to use SELECT LAST_INSERT_ID()? // std::cout << table_schema.id_ << std::endl; @@ -420,10 +444,10 @@ namespace meta { MetricCollector metric; { - ScopedConnection connectionPtr(*mySQLConnectionPool_, safe_grab); + ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); -// if (mySQLConnectionPool_->getConnectionsInUse() <= 0) { -// ENGINE_LOG_WARNING << "MySQLMetaImpl::DeleteTable connection in use = " << mySQLConnectionPool_->getConnectionsInUse(); +// if (mysql_connection_pool_->getConnectionsInUse() <= 0) { +// ENGINE_LOG_WARNING << "MySQLMetaImpl::DeleteTable connection in use = " << mysql_connection_pool_->getConnectionsInUse(); // } //soft delete table @@ -433,6 +457,10 @@ namespace meta { "SET state = " << std::to_string(TableSchema::TO_DELETE) << " " << "WHERE table_id = " << quote << table_id << ";"; + if (options_.sql_echo) { + ENGINE_LOG_DEBUG << "MySQLMetaImpl::DeleteTable: " << deleteTableQuery.str(); + } + if (!deleteTableQuery.exec()) { ENGINE_LOG_ERROR << "QUERY ERROR WHEN DELETING TABLE"; return Status::DBTransactionError("QUERY ERROR WHEN DELETING TABLE", deleteTableQuery.error()); @@ -441,9 +469,7 @@ namespace meta { } //Scoped Connection -// ConfigNode& serverConfig = ServerConfig::GetInstance().GetConfig(CONFIG_SERVER); -// opt.mode = serverConfig.GetValue(CONFIG_CLUSTER_MODE, "single"); - if (mode_ != "single") { + if (mode_ != Options::MODE::SINGLE) { DeleteTableFiles(table_id); } @@ -465,10 +491,10 @@ namespace meta { MetricCollector metric; { - ScopedConnection connectionPtr(*mySQLConnectionPool_, safe_grab); + ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); -// if (mySQLConnectionPool_->getConnectionsInUse() <= 0) { -// ENGINE_LOG_WARNING << "MySQLMetaImpl::DeleteTableFiles connection in use = " << mySQLConnectionPool_->getConnectionsInUse(); +// if (mysql_connection_pool_->getConnectionsInUse() <= 0) { +// ENGINE_LOG_WARNING << "MySQLMetaImpl::DeleteTableFiles connection in use = " << mysql_connection_pool_->getConnectionsInUse(); // } //soft delete table files @@ -480,6 +506,10 @@ namespace meta { "WHERE table_id = " << quote << table_id << " AND " << "file_type <> " << std::to_string(TableFileSchema::TO_DELETE) << ";"; + if (options_.sql_echo) { + ENGINE_LOG_DEBUG << "MySQLMetaImpl::DeleteTableFiles: " << deleteTableFilesQuery.str(); + } + if (!deleteTableFilesQuery.exec()) { ENGINE_LOG_ERROR << "QUERY ERROR WHEN DELETING TABLE FILES"; return Status::DBTransactionError("QUERY ERROR WHEN DELETING TABLE", deleteTableFilesQuery.error()); @@ -509,10 +539,10 @@ namespace meta { StoreQueryResult res; { - ScopedConnection connectionPtr(*mySQLConnectionPool_, safe_grab); + ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); -// if (mySQLConnectionPool_->getConnectionsInUse() <= 0) { -// ENGINE_LOG_WARNING << "MySQLMetaImpl::DescribeTable connection in use = " << mySQLConnectionPool_->getConnectionsInUse(); +// if (mysql_connection_pool_->getConnectionsInUse() <= 0) { +// ENGINE_LOG_WARNING << "MySQLMetaImpl::DescribeTable connection in use = " << mysql_connection_pool_->getConnectionsInUse(); // } Query describeTableQuery = connectionPtr->query(); @@ -520,6 +550,11 @@ namespace meta { "FROM Tables " << "WHERE table_id = " << quote << table_schema.table_id_ << " " << "AND state <> " << std::to_string(TableSchema::TO_DELETE) << ";"; + + if (options_.sql_echo) { + ENGINE_LOG_DEBUG << "MySQLMetaImpl::DescribeTable: " << describeTableQuery.str(); + } + res = describeTableQuery.store(); } //Scoped Connection @@ -535,7 +570,7 @@ namespace meta { table_schema.engine_type_ = resRow["engine_type"]; - table_schema.store_raw_data_ = (resRow["store_raw_data"].compare("true") == 0); + table_schema.store_raw_data_ = (resRow["store_raw_data"] == 1); } else { return Status::NotFound("Table " + table_schema.table_id_ + " not found"); @@ -568,10 +603,10 @@ namespace meta { StoreQueryResult res; { - ScopedConnection connectionPtr(*mySQLConnectionPool_, safe_grab); + ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); -// if (mySQLConnectionPool_->getConnectionsInUse() <= 0) { -// ENGINE_LOG_WARNING << "MySQLMetaImpl::HasTable connection in use = " << mySQLConnectionPool_->getConnectionsInUse(); +// if (mysql_connection_pool_->getConnectionsInUse() <= 0) { +// ENGINE_LOG_WARNING << "MySQLMetaImpl::HasTable connection in use = " << mysql_connection_pool_->getConnectionsInUse(); // } Query hasTableQuery = connectionPtr->query(); @@ -581,6 +616,11 @@ namespace meta { "WHERE table_id = " << quote << table_id << " " << "AND state <> " << std::to_string(TableSchema::TO_DELETE) << ") " << "AS " << quote << "check" << ";"; + + if (options_.sql_echo) { + ENGINE_LOG_DEBUG << "MySQLMetaImpl::HasTable: " << hasTableQuery.str(); + } + res = hasTableQuery.store(); } //Scoped Connection @@ -612,16 +652,21 @@ namespace meta { StoreQueryResult res; { - ScopedConnection connectionPtr(*mySQLConnectionPool_, safe_grab); + ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); -// if (mySQLConnectionPool_->getConnectionsInUse() <= 0) { -// ENGINE_LOG_WARNING << "MySQLMetaImpl::AllTables connection in use = " << mySQLConnectionPool_->getConnectionsInUse(); +// if (mysql_connection_pool_->getConnectionsInUse() <= 0) { +// ENGINE_LOG_WARNING << "MySQLMetaImpl::AllTables connection in use = " << mysql_connection_pool_->getConnectionsInUse(); // } Query allTablesQuery = connectionPtr->query(); allTablesQuery << "SELECT id, table_id, dimension, files_cnt, engine_type, store_raw_data " << "FROM Tables " << "WHERE state <> " << std::to_string(TableSchema::TO_DELETE) << ";"; + + if (options_.sql_echo) { + ENGINE_LOG_DEBUG << "MySQLMetaImpl::AllTables: " << allTablesQuery.str(); + } + res = allTablesQuery.store(); } //Scoped Connection @@ -640,7 +685,7 @@ namespace meta { table_schema.engine_type_ = resRow["engine_type"]; - table_schema.store_raw_data_ = (resRow["store_raw_data"].compare("true") == 0); + table_schema.store_raw_data_ = (resRow["store_raw_data"] == 1); table_schema_array.emplace_back(table_schema); } @@ -695,10 +740,10 @@ namespace meta { std::string date = std::to_string(file_schema.date_); { - ScopedConnection connectionPtr(*mySQLConnectionPool_, safe_grab); + ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); -// if (mySQLConnectionPool_->getConnectionsInUse() <= 0) { -// ENGINE_LOG_WARNING << "MySQLMetaImpl::CreateTableFile connection in use = " << mySQLConnectionPool_->getConnectionsInUse(); +// if (mysql_connection_pool_->getConnectionsInUse() <= 0) { +// ENGINE_LOG_WARNING << "MySQLMetaImpl::CreateTableFile connection in use = " << mysql_connection_pool_->getConnectionsInUse(); // } Query createTableFileQuery = connectionPtr->query(); @@ -708,6 +753,10 @@ namespace meta { quote << file_id << ", " << file_type << ", " << size << ", " << updated_time << ", " << created_on << ", " << date << ");"; + if (options_.sql_echo) { + ENGINE_LOG_DEBUG << "MySQLMetaImpl::CreateTableFile: " << createTableFileQuery.str(); + } + if (SimpleResult res = createTableFileQuery.execute()) { file_schema.id_ = res.insert_id(); //Might need to use SELECT LAST_INSERT_ID()? @@ -759,16 +808,21 @@ namespace meta { StoreQueryResult res; { - ScopedConnection connectionPtr(*mySQLConnectionPool_, safe_grab); + ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); -// if (mySQLConnectionPool_->getConnectionsInUse() <= 0) { -// ENGINE_LOG_WARNING << "MySQLMetaImpl::FilesToIndex connection in use = " << mySQLConnectionPool_->getConnectionsInUse(); +// if (mysql_connection_pool_->getConnectionsInUse() <= 0) { +// ENGINE_LOG_WARNING << "MySQLMetaImpl::FilesToIndex connection in use = " << mysql_connection_pool_->getConnectionsInUse(); // } Query filesToIndexQuery = connectionPtr->query(); filesToIndexQuery << "SELECT id, table_id, engine_type, file_id, file_type, size, date " << "FROM TableFiles " << "WHERE file_type = " << std::to_string(TableFileSchema::TO_INDEX) << ";"; + + if (options_.sql_echo) { + ENGINE_LOG_DEBUG << "MySQLMetaImpl::FilesToIndex: " << filesToIndexQuery.str(); + } + res = filesToIndexQuery.store(); } //Scoped Connection @@ -839,10 +893,10 @@ namespace meta { StoreQueryResult res; { - ScopedConnection connectionPtr(*mySQLConnectionPool_, safe_grab); + ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); -// if (mySQLConnectionPool_->getConnectionsInUse() <= 0) { -// ENGINE_LOG_WARNING << "MySQLMetaImpl::FilesToSearch connection in use = " << mySQLConnectionPool_->getConnectionsInUse(); +// if (mysql_connection_pool_->getConnectionsInUse() <= 0) { +// ENGINE_LOG_WARNING << "MySQLMetaImpl::FilesToSearch connection in use = " << mysql_connection_pool_->getConnectionsInUse(); // } if (partition.empty()) { @@ -854,6 +908,11 @@ namespace meta { "(file_type = " << std::to_string(TableFileSchema::RAW) << " OR " << "file_type = " << std::to_string(TableFileSchema::TO_INDEX) << " OR " << "file_type = " << std::to_string(TableFileSchema::INDEX) << ");"; + + if (options_.sql_echo) { + ENGINE_LOG_DEBUG << "MySQLMetaImpl::FilesToSearch: " << filesToSearchQuery.str(); + } + res = filesToSearchQuery.store(); } else { @@ -874,6 +933,11 @@ namespace meta { "(file_type = " << std::to_string(TableFileSchema::RAW) << " OR " << "file_type = " << std::to_string(TableFileSchema::TO_INDEX) << " OR " << "file_type = " << std::to_string(TableFileSchema::INDEX) << ");"; + + if (options_.sql_echo) { + ENGINE_LOG_DEBUG << "MySQLMetaImpl::FilesToSearch: " << filesToSearchQuery.str(); + } + res = filesToSearchQuery.store(); } @@ -944,10 +1008,10 @@ namespace meta { StoreQueryResult res; { - ScopedConnection connectionPtr(*mySQLConnectionPool_, safe_grab); + ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); -// if (mySQLConnectionPool_->getConnectionsInUse() <= 0) { -// ENGINE_LOG_WARNING << "MySQLMetaImpl::FilesToMerge connection in use = " << mySQLConnectionPool_->getConnectionsInUse(); +// if (mysql_connection_pool_->getConnectionsInUse() <= 0) { +// ENGINE_LOG_WARNING << "MySQLMetaImpl::FilesToMerge connection in use = " << mysql_connection_pool_->getConnectionsInUse(); // } Query filesToMergeQuery = connectionPtr->query(); @@ -956,6 +1020,11 @@ namespace meta { "WHERE table_id = " << quote << table_id << " AND " << "file_type = " << std::to_string(TableFileSchema::RAW) << " " << "ORDER BY size DESC" << ";"; + + if (options_.sql_echo) { + ENGINE_LOG_DEBUG << "MySQLMetaImpl::FilesToMerge: " << filesToMergeQuery.str(); + } + res = filesToMergeQuery.store(); } //Scoped Connection @@ -1033,10 +1102,10 @@ namespace meta { StoreQueryResult res; { - ScopedConnection connectionPtr(*mySQLConnectionPool_, safe_grab); + ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); -// if (mySQLConnectionPool_->getConnectionsInUse() <= 0) { -// ENGINE_LOG_WARNING << "MySQLMetaImpl::GetTableFiles connection in use = " << mySQLConnectionPool_->getConnectionsInUse(); +// if (mysql_connection_pool_->getConnectionsInUse() <= 0) { +// ENGINE_LOG_WARNING << "MySQLMetaImpl::GetTableFiles connection in use = " << mysql_connection_pool_->getConnectionsInUse(); // } Query getTableFileQuery = connectionPtr->query(); @@ -1044,6 +1113,11 @@ namespace meta { "FROM TableFiles " << "WHERE table_id = " << quote << table_id << " AND " << "(" << idStr << ");"; + + if (options_.sql_echo) { + ENGINE_LOG_DEBUG << "MySQLMetaImpl::GetTableFiles: " << getTableFileQuery.str(); + } + res = getTableFileQuery.store(); } //Scoped Connection @@ -1112,10 +1186,10 @@ namespace meta { try { - ScopedConnection connectionPtr(*mySQLConnectionPool_, safe_grab); + ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); -// if (mySQLConnectionPool_->getConnectionsInUse() <= 0) { -// ENGINE_LOG_WARNING << "MySQLMetaImpl::Archive connection in use = " << mySQLConnectionPool_->getConnectionsInUse(); +// if (mysql_connection_pool_->getConnectionsInUse() <= 0) { +// ENGINE_LOG_WARNING << "MySQLMetaImpl::Archive connection in use = " << mysql_connection_pool_->getConnectionsInUse(); // } Query archiveQuery = connectionPtr->query(); @@ -1123,6 +1197,11 @@ namespace meta { "SET file_type = " << std::to_string(TableFileSchema::TO_DELETE) << " " << "WHERE created_on < " << std::to_string(now - usecs) << " AND " << "file_type <> " << std::to_string(TableFileSchema::TO_DELETE) << ";"; + + if (options_.sql_echo) { + ENGINE_LOG_DEBUG << "MySQLMetaImpl::Archive: " << archiveQuery.str(); + } + if (!archiveQuery.exec()) { return Status::DBTransactionError("QUERY ERROR DURING ARCHIVE", archiveQuery.error()); } @@ -1159,16 +1238,21 @@ namespace meta { StoreQueryResult res; { - ScopedConnection connectionPtr(*mySQLConnectionPool_, safe_grab); + ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); -// if (mySQLConnectionPool_->getConnectionsInUse() <= 0) { -// ENGINE_LOG_WARNING << "MySQLMetaImpl::Size connection in use = " << mySQLConnectionPool_->getConnectionsInUse(); +// if (mysql_connection_pool_->getConnectionsInUse() <= 0) { +// ENGINE_LOG_WARNING << "MySQLMetaImpl::Size connection in use = " << mysql_connection_pool_->getConnectionsInUse(); // } Query getSizeQuery = connectionPtr->query(); getSizeQuery << "SELECT SUM(size) AS sum " << "FROM TableFiles " << "WHERE file_type <> " << std::to_string(TableFileSchema::TO_DELETE) << ";"; + + if (options_.sql_echo) { + ENGINE_LOG_DEBUG << "MySQLMetaImpl::Size: " << getSizeQuery.str(); + } + res = getSizeQuery.store(); } //Scoped Connection @@ -1216,10 +1300,10 @@ namespace meta { bool status; { - ScopedConnection connectionPtr(*mySQLConnectionPool_, safe_grab); + ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); -// if (mySQLConnectionPool_->getConnectionsInUse() <= 0) { -// ENGINE_LOG_WARNING << "MySQLMetaImpl::DiscardFiles connection in use = " << mySQLConnectionPool_->getConnectionsInUse(); +// if (mysql_connection_pool_->getConnectionsInUse() <= 0) { +// ENGINE_LOG_WARNING << "MySQLMetaImpl::DiscardFiles connection in use = " << mysql_connection_pool_->getConnectionsInUse(); // } Query discardFilesQuery = connectionPtr->query(); @@ -1228,7 +1312,12 @@ namespace meta { "WHERE file_type <> " << std::to_string(TableFileSchema::TO_DELETE) << " " << "ORDER BY id ASC " << "LIMIT 10;"; -// std::cout << discardFilesQuery.str() << std::endl; + + if (options_.sql_echo) { + ENGINE_LOG_DEBUG << "MySQLMetaImpl::DiscardFiles: " << discardFilesQuery.str(); + } + + // std::cout << discardFilesQuery.str() << std::endl; StoreQueryResult res = discardFilesQuery.store(); assert(res); @@ -1258,6 +1347,10 @@ namespace meta { "updated_time = " << std::to_string(utils::GetMicroSecTimeStamp()) << " " << "WHERE " << idsToDiscardStr << ";"; + if (options_.sql_echo) { + ENGINE_LOG_DEBUG << "MySQLMetaImpl::DiscardFiles: " << discardFilesQuery.str(); + } + status = discardFilesQuery.exec(); if (!status) { ENGINE_LOG_ERROR << "QUERY ERROR WHEN DISCARDING FILES"; @@ -1289,10 +1382,10 @@ namespace meta { MetricCollector metric; { - ScopedConnection connectionPtr(*mySQLConnectionPool_, safe_grab); + ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); -// if (mySQLConnectionPool_->getConnectionsInUse() <= 0) { -// ENGINE_LOG_WARNING << "MySQLMetaImpl::UpdateTableFile connection in use = " << mySQLConnectionPool_->getConnectionsInUse(); +// if (mysql_connection_pool_->getConnectionsInUse() <= 0) { +// ENGINE_LOG_WARNING << "MySQLMetaImpl::UpdateTableFile connection in use = " << mysql_connection_pool_->getConnectionsInUse(); // } Query updateTableFileQuery = connectionPtr->query(); @@ -1301,6 +1394,11 @@ namespace meta { //clean thread will delete the file later updateTableFileQuery << "SELECT state FROM Tables " << "WHERE table_id = " << quote << file_schema.table_id_ << ";"; + + if (options_.sql_echo) { + ENGINE_LOG_DEBUG << "MySQLMetaImpl::UpdateTableFile: " << updateTableFileQuery.str(); + } + StoreQueryResult res = updateTableFileQuery.store(); assert(res && res.num_rows() <= 1); @@ -1334,7 +1432,11 @@ namespace meta { "date = " << date << " " << "WHERE id = " << id << ";"; -// std::cout << updateTableFileQuery.str() << std::endl; + if (options_.sql_echo) { + ENGINE_LOG_DEBUG << "MySQLMetaImpl::UpdateTableFile: " << updateTableFileQuery.str(); + } + + // std::cout << updateTableFileQuery.str() << std::endl; if (!updateTableFileQuery.exec()) { ENGINE_LOG_DEBUG << "table_id= " << file_schema.table_id_ << " file_id=" << file_schema.file_id_; @@ -1366,10 +1468,10 @@ namespace meta { MetricCollector metric; { - ScopedConnection connectionPtr(*mySQLConnectionPool_, safe_grab); + ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); -// if (mySQLConnectionPool_->getConnectionsInUse() <= 0) { -// ENGINE_LOG_WARNING << "MySQLMetaImpl::UpdateTableFiles connection in use = " << mySQLConnectionPool_->getConnectionsInUse(); +// if (mysql_connection_pool_->getConnectionsInUse() <= 0) { +// ENGINE_LOG_WARNING << "MySQLMetaImpl::UpdateTableFiles connection in use = " << mysql_connection_pool_->getConnectionsInUse(); // } Query updateTableFilesQuery = connectionPtr->query(); @@ -1386,6 +1488,11 @@ namespace meta { "WHERE table_id = " << quote << file_schema.table_id_ << " " << "AND state <> " << std::to_string(TableSchema::TO_DELETE) << ") " << "AS " << quote << "check" << ";"; + + if (options_.sql_echo) { + ENGINE_LOG_DEBUG << "MySQLMetaImpl::UpdateTableFiles: " << updateTableFilesQuery.str(); + } + StoreQueryResult res = updateTableFilesQuery.store(); assert(res && res.num_rows() == 1); @@ -1421,6 +1528,10 @@ namespace meta { "date = " << date << " " << "WHERE id = " << id << ";"; + if (options_.sql_echo) { + ENGINE_LOG_DEBUG << "MySQLMetaImpl::UpdateTableFiles: " << updateTableFilesQuery.str(); + } + if (!updateTableFilesQuery.exec()) { ENGINE_LOG_ERROR << "QUERY ERROR WHEN UPDATING TABLE FILES"; return Status::DBTransactionError("QUERY ERROR WHEN UPDATING TABLE FILES", @@ -1454,13 +1565,13 @@ namespace meta { { // ENGINE_LOG_WARNING << "MySQLMetaImpl::CleanUpFilesWithTTL: clean table files: connection in use before creating ScopedConnection = " -// << mySQLConnectionPool_->getConnectionsInUse(); +// << mysql_connection_pool_->getConnectionsInUse(); - ScopedConnection connectionPtr(*mySQLConnectionPool_, safe_grab); + ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); -// if (mySQLConnectionPool_->getConnectionsInUse() <= 0) { +// if (mysql_connection_pool_->getConnectionsInUse() <= 0) { // ENGINE_LOG_WARNING << "MySQLMetaImpl::CleanUpFilesWithTTL: clean table files: connection in use after creating ScopedConnection = " -// << mySQLConnectionPool_->getConnectionsInUse(); +// << mysql_connection_pool_->getConnectionsInUse(); // } Query cleanUpFilesWithTTLQuery = connectionPtr->query(); @@ -1468,6 +1579,11 @@ namespace meta { "FROM TableFiles " << "WHERE file_type = " << std::to_string(TableFileSchema::TO_DELETE) << " AND " << "updated_time < " << std::to_string(now - seconds * US_PS) << ";"; + + if (options_.sql_echo) { + ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUpFilesWithTTL: " << cleanUpFilesWithTTLQuery.str(); + } + StoreQueryResult res = cleanUpFilesWithTTLQuery.store(); assert(res); @@ -1509,6 +1625,11 @@ namespace meta { idsToDeleteStr = idsToDeleteStr.substr(0, idsToDeleteStr.size() - 4); //remove the last " OR " cleanUpFilesWithTTLQuery << "DELETE FROM TableFiles WHERE " << idsToDeleteStr << ";"; + + if (options_.sql_echo) { + ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUpFilesWithTTL: " << cleanUpFilesWithTTLQuery.str(); + } + if (!cleanUpFilesWithTTLQuery.exec()) { ENGINE_LOG_ERROR << "QUERY ERROR WHEN CLEANING UP FILES WITH TTL"; return Status::DBTransactionError("CleanUpFilesWithTTL Error", @@ -1532,19 +1653,24 @@ namespace meta { { // ENGINE_LOG_WARNING << "MySQLMetaImpl::CleanUpFilesWithTTL: clean tables: connection in use before creating ScopedConnection = " -// << mySQLConnectionPool_->getConnectionsInUse(); +// << mysql_connection_pool_->getConnectionsInUse(); - ScopedConnection connectionPtr(*mySQLConnectionPool_, safe_grab); + ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); -// if (mySQLConnectionPool_->getConnectionsInUse() <= 0) { +// if (mysql_connection_pool_->getConnectionsInUse() <= 0) { // ENGINE_LOG_WARNING << "MySQLMetaImpl::CleanUpFilesWithTTL: clean tables: connection in use after creating ScopedConnection = " -// << mySQLConnectionPool_->getConnectionsInUse(); +// << mysql_connection_pool_->getConnectionsInUse(); // } Query cleanUpFilesWithTTLQuery = connectionPtr->query(); cleanUpFilesWithTTLQuery << "SELECT id, table_id " << "FROM Tables " << "WHERE state = " << std::to_string(TableSchema::TO_DELETE) << ";"; + + if (options_.sql_echo) { + ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUpFilesWithTTL: " << cleanUpFilesWithTTLQuery.str(); + } + StoreQueryResult res = cleanUpFilesWithTTLQuery.store(); assert(res); // std::cout << res.num_rows() << std::endl; @@ -1568,6 +1694,11 @@ namespace meta { idsToDeleteStr = idsToDeleteStr.substr(0, idsToDeleteStr.size() - 4); //remove the last " OR " cleanUpFilesWithTTLQuery << "DELETE FROM Tables WHERE " << idsToDeleteStr << ";"; + + if (options_.sql_echo) { + ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUpFilesWithTTL: " << cleanUpFilesWithTTLQuery.str(); + } + if (!cleanUpFilesWithTTLQuery.exec()) { ENGINE_LOG_ERROR << "QUERY ERROR WHEN CLEANING UP FILES WITH TTL"; return Status::DBTransactionError("QUERY ERROR WHEN CLEANING UP FILES WITH TTL", @@ -1594,16 +1725,20 @@ namespace meta { // std::lock_guard lock(mysql_mutex); try { - ScopedConnection connectionPtr(*mySQLConnectionPool_, safe_grab); + ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); -// if (mySQLConnectionPool_->getConnectionsInUse() <= 0) { -// ENGINE_LOG_WARNING << "MySQLMetaImpl::CleanUp: connection in use = " << mySQLConnectionPool_->getConnectionsInUse(); +// if (mysql_connection_pool_->getConnectionsInUse() <= 0) { +// ENGINE_LOG_WARNING << "MySQLMetaImpl::CleanUp: connection in use = " << mysql_connection_pool_->getConnectionsInUse(); // } ENGINE_LOG_DEBUG << "Remove table file type as NEW"; Query cleanUpQuery = connectionPtr->query(); cleanUpQuery << "DELETE FROM TableFiles WHERE file_type = " << std::to_string(TableFileSchema::NEW) << ";"; + if (options_.sql_echo) { + ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUp: " << cleanUpQuery.str(); + } + if (!cleanUpQuery.exec()) { ENGINE_LOG_ERROR << "QUERY ERROR WHEN CLEANING UP FILES"; return Status::DBTransactionError("Clean up Error", cleanUpQuery.error()); @@ -1640,10 +1775,10 @@ namespace meta { StoreQueryResult res; { - ScopedConnection connectionPtr(*mySQLConnectionPool_, safe_grab); + ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); -// if (mySQLConnectionPool_->getConnectionsInUse() <= 0) { -// ENGINE_LOG_WARNING << "MySQLMetaImpl::Count: connection in use = " << mySQLConnectionPool_->getConnectionsInUse(); +// if (mysql_connection_pool_->getConnectionsInUse() <= 0) { +// ENGINE_LOG_WARNING << "MySQLMetaImpl::Count: connection in use = " << mysql_connection_pool_->getConnectionsInUse(); // } Query countQuery = connectionPtr->query(); @@ -1653,6 +1788,11 @@ namespace meta { "(file_type = " << std::to_string(TableFileSchema::RAW) << " OR " << "file_type = " << std::to_string(TableFileSchema::TO_INDEX) << " OR " << "file_type = " << std::to_string(TableFileSchema::INDEX) << ");"; + + if (options_.sql_echo) { + ENGINE_LOG_DEBUG << "MySQLMetaImpl::Count: " << countQuery.str(); + } + res = countQuery.store(); } //Scoped Connection @@ -1687,14 +1827,19 @@ namespace meta { } try { - ScopedConnection connectionPtr(*mySQLConnectionPool_, safe_grab); + ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); -// if (mySQLConnectionPool_->getConnectionsInUse() <= 0) { -// ENGINE_LOG_WARNING << "MySQLMetaImpl::DropAll: connection in use = " << mySQLConnectionPool_->getConnectionsInUse(); +// if (mysql_connection_pool_->getConnectionsInUse() <= 0) { +// ENGINE_LOG_WARNING << "MySQLMetaImpl::DropAll: connection in use = " << mysql_connection_pool_->getConnectionsInUse(); // } Query dropTableQuery = connectionPtr->query(); dropTableQuery << "DROP TABLE IF EXISTS Tables, TableFiles;"; + + if (options_.sql_echo) { + ENGINE_LOG_DEBUG << "MySQLMetaImpl::DropAll: " << dropTableQuery.str(); + } + if (dropTableQuery.exec()) { return Status::OK(); } diff --git a/cpp/src/db/MySQLMetaImpl.h b/cpp/src/db/MySQLMetaImpl.h index 4805076d45..9ff8254b60 100644 --- a/cpp/src/db/MySQLMetaImpl.h +++ b/cpp/src/db/MySQLMetaImpl.h @@ -22,7 +22,7 @@ namespace meta { class MySQLMetaImpl : public Meta { public: - MySQLMetaImpl(const DBMetaOptions& options_, const std::string& mode); + MySQLMetaImpl(const DBMetaOptions& options_, const int& mode); virtual Status CreateTable(TableSchema& table_schema) override; virtual Status DescribeTable(TableSchema& group_info_) override; @@ -77,9 +77,9 @@ namespace meta { Status Initialize(); const DBMetaOptions options_; - const std::string mode_; + const int mode_; - std::shared_ptr mySQLConnectionPool_; + std::shared_ptr mysql_connection_pool_; bool safe_grab = false; // std::mutex connectionMutex_; diff --git a/cpp/src/db/Options.h b/cpp/src/db/Options.h index dbe80f8d5f..609e3ca245 100644 --- a/cpp/src/db/Options.h +++ b/cpp/src/db/Options.h @@ -45,15 +45,23 @@ struct DBMetaOptions { std::string path; std::string backend_uri; ArchiveConf archive_conf = ArchiveConf("delete"); + bool sql_echo = false; }; // DBMetaOptions struct Options { + + typedef enum { + SINGLE, + CLUSTER, + READ_ONLY + } MODE; + Options(); uint16_t memory_sync_interval = 1; //unit: second uint16_t merge_trigger_number = 2; size_t index_trigger_size = ONE_GB; //unit: byte DBMetaOptions meta; - std::string mode; + int mode = MODE::SINGLE; }; // Options diff --git a/cpp/src/server/DBWrapper.cpp b/cpp/src/server/DBWrapper.cpp index ceaa6e8130..a3db0bf110 100644 --- a/cpp/src/server/DBWrapper.cpp +++ b/cpp/src/server/DBWrapper.cpp @@ -23,9 +23,30 @@ DBWrapper::DBWrapper() { if(index_size > 0) {//ensure larger than zero, unit is MB opt.index_trigger_size = (size_t)index_size * engine::ONE_MB; } + std::string sql_echo = config.GetValue(CONFIG_DB_SQL_ECHO, "off"); + if (sql_echo == "on") { + opt.meta.sql_echo = true; + } + else if (sql_echo == "off") { + opt.meta.sql_echo = false; + } + else { + std::cout << "ERROR: sql_echo specified in db_config is not one of ['on', 'off']" << std::endl; + kill(0, SIGUSR1); + } + ConfigNode& serverConfig = ServerConfig::GetInstance().GetConfig(CONFIG_SERVER); - opt.mode = serverConfig.GetValue(CONFIG_CLUSTER_MODE, "single"); - if (opt.mode != "single" && opt.mode != "cluster" && opt.mode != "read_only") { + std::string mode = serverConfig.GetValue(CONFIG_CLUSTER_MODE, "single"); + if (mode == "single") { + opt.mode = zilliz::milvus::engine::Options::MODE::SINGLE; + } + else if (mode == "cluster") { + opt.mode = zilliz::milvus::engine::Options::MODE::CLUSTER; + } + else if (mode == "read_only") { + opt.mode = zilliz::milvus::engine::Options::MODE::READ_ONLY; + } + else { std::cout << "ERROR: mode specified in server_config is not one of ['single', 'cluster', 'read_only']" << std::endl; kill(0, SIGUSR1); } diff --git a/cpp/src/server/ServerConfig.h b/cpp/src/server/ServerConfig.h index f337275a46..768430f023 100644 --- a/cpp/src/server/ServerConfig.h +++ b/cpp/src/server/ServerConfig.h @@ -27,6 +27,7 @@ static const std::string CONFIG_DB_PATH = "db_path"; static const std::string CONFIG_DB_INDEX_TRIGGER_SIZE = "index_building_threshold"; static const std::string CONFIG_DB_ARCHIVE_DISK = "archive_disk_threshold"; static const std::string CONFIG_DB_ARCHIVE_DAYS = "archive_days_threshold"; +static const std::string CONFIG_DB_SQL_ECHO = "sql_echo"; static const std::string CONFIG_LOG = "log_config"; From 1982887eb2d51b972418e99a1cf61b06c195d529 Mon Sep 17 00:00:00 2001 From: zhiru Date: Sun, 30 Jun 2019 13:32:54 +0800 Subject: [PATCH 29/43] update Former-commit-id: 8d5eaf28efc86d4550a9a0bfdc18d15e29c2607e --- cpp/src/db/MySQLMetaImpl.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/cpp/src/db/MySQLMetaImpl.cpp b/cpp/src/db/MySQLMetaImpl.cpp index f57bed7420..7acff7ab05 100644 --- a/cpp/src/db/MySQLMetaImpl.cpp +++ b/cpp/src/db/MySQLMetaImpl.cpp @@ -570,7 +570,8 @@ namespace meta { table_schema.engine_type_ = resRow["engine_type"]; - table_schema.store_raw_data_ = (resRow["store_raw_data"] == 1); + int store_raw_data = resRow["store_raw_data"]; + table_schema.store_raw_data_ = (store_raw_data == 1); } else { return Status::NotFound("Table " + table_schema.table_id_ + " not found"); @@ -685,7 +686,8 @@ namespace meta { table_schema.engine_type_ = resRow["engine_type"]; - table_schema.store_raw_data_ = (resRow["store_raw_data"] == 1); + int store_raw_data = resRow["store_raw_data"]; + table_schema.store_raw_data_ = (store_raw_data == 1); table_schema_array.emplace_back(table_schema); } From 210c801839d081f1169a2c1eaf55b12b0213fbdf Mon Sep 17 00:00:00 2001 From: zhiru Date: Sun, 30 Jun 2019 14:05:45 +0800 Subject: [PATCH 30/43] update Former-commit-id: 0eb9b5f027c49b5734411b5ae4ad9e384129a134 --- cpp/src/db/MySQLMetaImpl.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/src/db/MySQLMetaImpl.cpp b/cpp/src/db/MySQLMetaImpl.cpp index 7acff7ab05..0b0cc01e5d 100644 --- a/cpp/src/db/MySQLMetaImpl.cpp +++ b/cpp/src/db/MySQLMetaImpl.cpp @@ -1247,7 +1247,7 @@ namespace meta { // } Query getSizeQuery = connectionPtr->query(); - getSizeQuery << "SELECT SUM(size) AS sum " << + getSizeQuery << "SELECT IFNULL(SUM(size),0) AS sum " << "FROM TableFiles " << "WHERE file_type <> " << std::to_string(TableFileSchema::TO_DELETE) << ";"; From bf41a0a14e3faf63bcea1c6f996644c74d259abc Mon Sep 17 00:00:00 2001 From: groot Date: Sun, 30 Jun 2019 14:20:56 +0800 Subject: [PATCH 31/43] add more unitest Former-commit-id: 569a97813365258f191a5035a52cd6f8671da77c --- cpp/src/db/DBImpl.cpp | 276 ++++++++++++++++----------------- cpp/src/db/DBImpl.h | 4 +- cpp/src/db/ExecutionEngine.cpp | 11 +- cpp/src/db/ExecutionEngine.h | 3 +- cpp/unittest/db/db_tests.cpp | 30 ++-- cpp/unittest/db/meta_tests.cpp | 10 ++ cpp/unittest/db/misc_test.cpp | 103 ++++++++++++ 7 files changed, 275 insertions(+), 162 deletions(-) create mode 100644 cpp/unittest/db/misc_test.cpp diff --git a/cpp/src/db/DBImpl.cpp b/cpp/src/db/DBImpl.cpp index a0abb5e1ad..ebc836a978 100644 --- a/cpp/src/db/DBImpl.cpp +++ b/cpp/src/db/DBImpl.cpp @@ -248,144 +248,144 @@ Status DBImpl::Query(const std::string& table_id, const std::vector return QueryAsync(table_id, files_array, k, nq, vectors, dates, results); } -Status DBImpl::QuerySync(const std::string& table_id, uint64_t k, uint64_t nq, - const float* vectors, const meta::DatesT& dates, QueryResults& results) { - meta::DatePartionedTableFilesSchema files; - auto status = meta_ptr_->FilesToSearch(table_id, dates, files); - if (!status.ok()) { return status; } - - ENGINE_LOG_DEBUG << "Search DateT Size = " << files.size(); - - meta::TableFilesSchema index_files; - meta::TableFilesSchema raw_files; - for (auto &day_files : files) { - for (auto &file : day_files.second) { - file.file_type_ == meta::TableFileSchema::INDEX ? - index_files.push_back(file) : raw_files.push_back(file); - } - } - - int dim = 0; - if (!index_files.empty()) { - dim = index_files[0].dimension_; - } else if (!raw_files.empty()) { - dim = raw_files[0].dimension_; - } else { - ENGINE_LOG_DEBUG << "no files to search"; - return Status::OK(); - } - - { - // [{ids, distence}, ...] - using SearchResult = std::pair, std::vector>; - std::vector batchresult(nq); // allocate nq cells. - - auto cluster = [&](long *nns, float *dis, const int& k) -> void { - for (int i = 0; i < nq; ++i) { - auto f_begin = batchresult[i].first.cbegin(); - auto s_begin = batchresult[i].second.cbegin(); - batchresult[i].first.insert(f_begin, nns + i * k, nns + i * k + k); - batchresult[i].second.insert(s_begin, dis + i * k, dis + i * k + k); - } - }; - - // Allocate Memory - float *output_distence; - long *output_ids; - output_distence = (float *) malloc(k * nq * sizeof(float)); - output_ids = (long *) malloc(k * nq * sizeof(long)); - memset(output_distence, 0, k * nq * sizeof(float)); - memset(output_ids, 0, k * nq * sizeof(long)); - - long search_set_size = 0; - - auto search_in_index = [&](meta::TableFilesSchema& file_vec) -> void { - for (auto &file : file_vec) { - - ExecutionEnginePtr index = EngineFactory::Build(file.dimension_, file.location_, (EngineType)file.engine_type_); - index->Load(); - auto file_size = index->PhysicalSize(); - search_set_size += file_size; - - ENGINE_LOG_DEBUG << "Search file_type " << file.file_type_ << " Of Size: " - << file_size/(1024*1024) << " M"; - - int inner_k = index->Count() < k ? index->Count() : k; - auto start_time = METRICS_NOW_TIME; - index->Search(nq, vectors, inner_k, output_distence, output_ids); - auto end_time = METRICS_NOW_TIME; - auto total_time = METRICS_MICROSECONDS(start_time, end_time); - CollectFileMetrics(file.file_type_, file_size, total_time); - cluster(output_ids, output_distence, inner_k); // cluster to each query - memset(output_distence, 0, k * nq * sizeof(float)); - memset(output_ids, 0, k * nq * sizeof(long)); - } - }; - - auto topk_cpu = [](const std::vector &input_data, - const int &k, - float *output_distence, - long *output_ids) -> void { - std::map> inverted_table; - for (int i = 0; i < input_data.size(); ++i) { - if (inverted_table.count(input_data[i]) == 1) { - auto& ori_vec = inverted_table[input_data[i]]; - ori_vec.push_back(i); - } - else { - inverted_table[input_data[i]] = std::vector{i}; - } - } - - int count = 0; - for (auto &item : inverted_table){ - if (count == k) break; - for (auto &id : item.second){ - output_distence[count] = item.first; - output_ids[count] = id; - if (++count == k) break; - } - } - }; - auto cluster_topk = [&]() -> void { - QueryResult res; - for (auto &result_pair : batchresult) { - auto &dis = result_pair.second; - auto &nns = result_pair.first; - - topk_cpu(dis, k, output_distence, output_ids); - - int inner_k = dis.size() < k ? dis.size() : k; - for (int i = 0; i < inner_k; ++i) { - res.emplace_back(std::make_pair(nns[output_ids[i]], output_distence[i])); // mapping - } - results.push_back(res); // append to result list - res.clear(); - memset(output_distence, 0, k * nq * sizeof(float)); - memset(output_ids, 0, k * nq * sizeof(long)); - } - }; - - search_in_index(raw_files); - search_in_index(index_files); - - ENGINE_LOG_DEBUG << "Search Overall Set Size = " << search_set_size << " M"; - cluster_topk(); - - free(output_distence); - free(output_ids); - } - - if (results.empty()) { - return Status::NotFound("Group " + table_id + ", search result not found!"); - } - - QueryResults temp_results; - CalcScore(nq, vectors, dim, results, temp_results); - results.swap(temp_results); - - return Status::OK(); -} +//Status DBImpl::QuerySync(const std::string& table_id, uint64_t k, uint64_t nq, +// const float* vectors, const meta::DatesT& dates, QueryResults& results) { +// meta::DatePartionedTableFilesSchema files; +// auto status = meta_ptr_->FilesToSearch(table_id, dates, files); +// if (!status.ok()) { return status; } +// +// ENGINE_LOG_DEBUG << "Search DateT Size = " << files.size(); +// +// meta::TableFilesSchema index_files; +// meta::TableFilesSchema raw_files; +// for (auto &day_files : files) { +// for (auto &file : day_files.second) { +// file.file_type_ == meta::TableFileSchema::INDEX ? +// index_files.push_back(file) : raw_files.push_back(file); +// } +// } +// +// int dim = 0; +// if (!index_files.empty()) { +// dim = index_files[0].dimension_; +// } else if (!raw_files.empty()) { +// dim = raw_files[0].dimension_; +// } else { +// ENGINE_LOG_DEBUG << "no files to search"; +// return Status::OK(); +// } +// +// { +// // [{ids, distence}, ...] +// using SearchResult = std::pair, std::vector>; +// std::vector batchresult(nq); // allocate nq cells. +// +// auto cluster = [&](long *nns, float *dis, const int& k) -> void { +// for (int i = 0; i < nq; ++i) { +// auto f_begin = batchresult[i].first.cbegin(); +// auto s_begin = batchresult[i].second.cbegin(); +// batchresult[i].first.insert(f_begin, nns + i * k, nns + i * k + k); +// batchresult[i].second.insert(s_begin, dis + i * k, dis + i * k + k); +// } +// }; +// +// // Allocate Memory +// float *output_distence; +// long *output_ids; +// output_distence = (float *) malloc(k * nq * sizeof(float)); +// output_ids = (long *) malloc(k * nq * sizeof(long)); +// memset(output_distence, 0, k * nq * sizeof(float)); +// memset(output_ids, 0, k * nq * sizeof(long)); +// +// long search_set_size = 0; +// +// auto search_in_index = [&](meta::TableFilesSchema& file_vec) -> void { +// for (auto &file : file_vec) { +// +// ExecutionEnginePtr index = EngineFactory::Build(file.dimension_, file.location_, (EngineType)file.engine_type_); +// index->Load(); +// auto file_size = index->PhysicalSize(); +// search_set_size += file_size; +// +// ENGINE_LOG_DEBUG << "Search file_type " << file.file_type_ << " Of Size: " +// << file_size/(1024*1024) << " M"; +// +// int inner_k = index->Count() < k ? index->Count() : k; +// auto start_time = METRICS_NOW_TIME; +// index->Search(nq, vectors, inner_k, output_distence, output_ids); +// auto end_time = METRICS_NOW_TIME; +// auto total_time = METRICS_MICROSECONDS(start_time, end_time); +// CollectFileMetrics(file.file_type_, file_size, total_time); +// cluster(output_ids, output_distence, inner_k); // cluster to each query +// memset(output_distence, 0, k * nq * sizeof(float)); +// memset(output_ids, 0, k * nq * sizeof(long)); +// } +// }; +// +// auto topk_cpu = [](const std::vector &input_data, +// const int &k, +// float *output_distence, +// long *output_ids) -> void { +// std::map> inverted_table; +// for (int i = 0; i < input_data.size(); ++i) { +// if (inverted_table.count(input_data[i]) == 1) { +// auto& ori_vec = inverted_table[input_data[i]]; +// ori_vec.push_back(i); +// } +// else { +// inverted_table[input_data[i]] = std::vector{i}; +// } +// } +// +// int count = 0; +// for (auto &item : inverted_table){ +// if (count == k) break; +// for (auto &id : item.second){ +// output_distence[count] = item.first; +// output_ids[count] = id; +// if (++count == k) break; +// } +// } +// }; +// auto cluster_topk = [&]() -> void { +// QueryResult res; +// for (auto &result_pair : batchresult) { +// auto &dis = result_pair.second; +// auto &nns = result_pair.first; +// +// topk_cpu(dis, k, output_distence, output_ids); +// +// int inner_k = dis.size() < k ? dis.size() : k; +// for (int i = 0; i < inner_k; ++i) { +// res.emplace_back(std::make_pair(nns[output_ids[i]], output_distence[i])); // mapping +// } +// results.push_back(res); // append to result list +// res.clear(); +// memset(output_distence, 0, k * nq * sizeof(float)); +// memset(output_ids, 0, k * nq * sizeof(long)); +// } +// }; +// +// search_in_index(raw_files); +// search_in_index(index_files); +// +// ENGINE_LOG_DEBUG << "Search Overall Set Size = " << search_set_size << " M"; +// cluster_topk(); +// +// free(output_distence); +// free(output_ids); +// } +// +// if (results.empty()) { +// return Status::NotFound("Group " + table_id + ", search result not found!"); +// } +// +// QueryResults temp_results; +// CalcScore(nq, vectors, dim, results, temp_results); +// results.swap(temp_results); +// +// return Status::OK(); +//} Status DBImpl::QueryAsync(const std::string& table_id, const meta::TableFilesSchema& files, uint64_t k, uint64_t nq, const float* vectors, diff --git a/cpp/src/db/DBImpl.h b/cpp/src/db/DBImpl.h index b4d60a27a9..43627cbace 100644 --- a/cpp/src/db/DBImpl.h +++ b/cpp/src/db/DBImpl.h @@ -62,8 +62,8 @@ public: virtual ~DBImpl(); private: - Status QuerySync(const std::string& table_id, uint64_t k, uint64_t nq, - const float* vectors, const meta::DatesT& dates, QueryResults& results); +// Status QuerySync(const std::string& table_id, uint64_t k, uint64_t nq, +// const float* vectors, const meta::DatesT& dates, QueryResults& results); Status QueryAsync(const std::string& table_id, const meta::TableFilesSchema& files, uint64_t k, uint64_t nq, const float* vectors, diff --git a/cpp/src/db/ExecutionEngine.cpp b/cpp/src/db/ExecutionEngine.cpp index f27d04dfa0..3412eb34bd 100644 --- a/cpp/src/db/ExecutionEngine.cpp +++ b/cpp/src/db/ExecutionEngine.cpp @@ -11,14 +11,9 @@ namespace zilliz { namespace milvus { namespace engine { -Status ExecutionEngine::AddWithIds(const std::vector& vectors, const std::vector& vector_ids) { - long n1 = (long)vectors.size(); - long n2 = (long)vector_ids.size(); - if (n1 != n2) { - LOG(ERROR) << "vectors size is not equal to the size of vector_ids: " << n1 << "!=" << n2; - return Status::Error("Error: AddWithIds"); - } - return AddWithIds(n1, vectors.data(), vector_ids.data()); +Status ExecutionEngine::AddWithIdArray(const std::vector& vectors, const std::vector& vector_ids) { + long n = (long)vector_ids.size(); + return AddWithIds(n, vectors.data(), vector_ids.data()); } diff --git a/cpp/src/db/ExecutionEngine.h b/cpp/src/db/ExecutionEngine.h index f26dce6371..d2b4d01e67 100644 --- a/cpp/src/db/ExecutionEngine.h +++ b/cpp/src/db/ExecutionEngine.h @@ -23,8 +23,7 @@ enum class EngineType { class ExecutionEngine { public: - virtual Status AddWithIds(const std::vector& vectors, - const std::vector& vector_ids); + virtual Status AddWithIdArray(const std::vector& vectors, const std::vector& vector_ids); virtual Status AddWithIds(long n, const float *xdata, const long *xids) = 0; diff --git a/cpp/unittest/db/db_tests.cpp b/cpp/unittest/db/db_tests.cpp index aa311550ee..7b33545d3a 100644 --- a/cpp/unittest/db/db_tests.cpp +++ b/cpp/unittest/db/db_tests.cpp @@ -88,20 +88,14 @@ TEST_F(DBTest, CONFIG_TEST) { TEST_F(DBTest, DB_TEST) { - static const std::string table_name = "test_group"; - static const int table_dim = 256; - - engine::meta::TableSchema table_info; - table_info.dimension_ = table_dim; - table_info.table_id_ = table_name; - table_info.engine_type_ = (int)engine::EngineType::FAISS_IDMAP; + engine::meta::TableSchema table_info = BuildTableSchema(); engine::Status stat = db_->CreateTable(table_info); engine::meta::TableSchema table_info_get; - table_info_get.table_id_ = table_name; + table_info_get.table_id_ = TABLE_NAME; stat = db_->DescribeTable(table_info_get); ASSERT_STATS(stat); - ASSERT_EQ(table_info_get.dimension_, table_dim); + ASSERT_EQ(table_info_get.dimension_, TABLE_DIM); engine::IDNumbers vector_ids; engine::IDNumbers target_ids; @@ -130,7 +124,7 @@ TEST_F(DBTest, DB_TEST) { prev_count = count; START_TIMER; - stat = db_->Query(table_name, k, qb, qxb.data(), results); + stat = db_->Query(TABLE_NAME, k, qb, qxb.data(), results); ss << "Search " << j << " With Size " << count/engine::meta::M << " M"; STOP_TIMER(ss.str()); @@ -153,10 +147,10 @@ TEST_F(DBTest, DB_TEST) { for (auto i=0; iInsertVectors(table_name, qb, qxb.data(), target_ids); + db_->InsertVectors(TABLE_NAME, qb, qxb.data(), target_ids); ASSERT_EQ(target_ids.size(), qb); } else { - db_->InsertVectors(table_name, nb, xb.data(), vector_ids); + db_->InsertVectors(TABLE_NAME, nb, xb.data(), vector_ids); } std::this_thread::sleep_for(std::chrono::microseconds(1)); } @@ -223,6 +217,18 @@ TEST_F(DBTest2, ARHIVE_DISK_CHECK) { engine::meta::TableSchema table_info = BuildTableSchema(); engine::Status stat = db_->CreateTable(table_info); + std::vector table_schema_array; + stat = db_->AllTables(table_schema_array); + ASSERT_STATS(stat); + bool bfound = false; + for(auto& schema : table_schema_array) { + if(schema.table_id_ == TABLE_NAME) { + bfound = true; + break; + } + } + ASSERT_TRUE(bfound); + engine::meta::TableSchema table_info_get; table_info_get.table_id_ = TABLE_NAME; stat = db_->DescribeTable(table_info_get); diff --git a/cpp/unittest/db/meta_tests.cpp b/cpp/unittest/db/meta_tests.cpp index 9baef712ab..49cc47b4e0 100644 --- a/cpp/unittest/db/meta_tests.cpp +++ b/cpp/unittest/db/meta_tests.cpp @@ -39,6 +39,10 @@ TEST_F(MetaTest, TABLE_TEST) { table.table_id_ = table_id; status = impl_->CreateTable(table); ASSERT_TRUE(status.ok()); + + table.table_id_ = ""; + status = impl_->CreateTable(table); + ASSERT_TRUE(status.ok()); } TEST_F(MetaTest, TABLE_FILE_TEST) { @@ -46,6 +50,7 @@ TEST_F(MetaTest, TABLE_FILE_TEST) { meta::TableSchema table; table.table_id_ = table_id; + table.dimension_ = 256; auto status = impl_->CreateTable(table); meta::TableFileSchema table_file; @@ -54,6 +59,11 @@ TEST_F(MetaTest, TABLE_FILE_TEST) { ASSERT_TRUE(status.ok()); ASSERT_EQ(table_file.file_type_, meta::TableFileSchema::NEW); + uint64_t cnt = 0; + status = impl_->Count(table_id, cnt); + ASSERT_TRUE(status.ok()); + ASSERT_EQ(cnt, 0UL); + auto file_id = table_file.file_id_; auto new_file_type = meta::TableFileSchema::INDEX; diff --git a/cpp/unittest/db/misc_test.cpp b/cpp/unittest/db/misc_test.cpp new file mode 100644 index 0000000000..6f1f87c874 --- /dev/null +++ b/cpp/unittest/db/misc_test.cpp @@ -0,0 +1,103 @@ +//////////////////////////////////////////////////////////////////////////////// +// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved +// Unauthorized copying of this file, via any medium is strictly prohibited. +// Proprietary and confidential. +//////////////////////////////////////////////////////////////////////////////// +#include +#include +#include +#include + +#include "db/FaissExecutionEngine.h" +#include "db/Exception.h" +#include "db/Status.h" +#include "db/Options.h" +#include "db/DBMetaImpl.h" + +#include + +using namespace zilliz::milvus; + +TEST(DBMiscTest, ENGINE_API_TEST) { + //engine api AddWithIdArray + const uint16_t dim = 512; + const long n = 10; + engine::FaissExecutionEngine engine(512, "/tmp/1", "IDMap", "IDMap,Flat"); + std::vector vectors; + std::vector ids; + for (long i = 0; i < n; i++) { + for (uint16_t k = 0; k < dim; k++) { + vectors.push_back((float) k); + } + ids.push_back(i); + } + + auto status = engine.AddWithIdArray(vectors, ids); + ASSERT_TRUE(status.ok()); +} + +TEST(DBMiscTest, EXCEPTION_TEST) { + engine::Exception ex1(""); + std::string what = ex1.what(); + ASSERT_FALSE(what.empty()); + + engine::OutOfRangeException ex2; + what = ex2.what(); + ASSERT_FALSE(what.empty()); +} + +TEST(DBMiscTest, STATUS_TEST) { + engine::Status status = engine::Status::OK(); + std::string str = status.ToString(); + ASSERT_FALSE(str.empty()); + + status = engine::Status::Error("wrong", "mistake"); + ASSERT_TRUE(status.IsError()); + str = status.ToString(); + ASSERT_FALSE(str.empty()); + + status = engine::Status::NotFound("wrong", "mistake"); + ASSERT_TRUE(status.IsNotFound()); + str = status.ToString(); + ASSERT_FALSE(str.empty()); + + status = engine::Status::DBTransactionError("wrong", "mistake"); + ASSERT_TRUE(status.IsDBTransactionError()); + str = status.ToString(); + ASSERT_FALSE(str.empty()); +} + +TEST(DBMiscTest, OPTIONS_TEST) { + try { + engine::ArchiveConf archive("$$##"); + } catch (std::exception& ex) { + ASSERT_TRUE(true); + } + + { + engine::ArchiveConf archive("delete", "no"); + ASSERT_TRUE(archive.GetCriterias().empty()); + } + + { + engine::ArchiveConf archive("delete", "1:2"); + ASSERT_TRUE(archive.GetCriterias().empty()); + } + + { + engine::ArchiveConf archive("delete", "1:2:3"); + ASSERT_TRUE(archive.GetCriterias().empty()); + } +} + +TEST(DBMiscTest, META_TEST) { + engine::DBMetaOptions options; + options.path = "/tmp/milvus_test"; + engine::meta::DBMetaImpl impl(options); + + time_t tt; + time( &tt ); + int delta = 10; + engine::meta::DateT dt = impl.GetDate(tt, delta); + ASSERT_GT(dt, 0); +} \ No newline at end of file From f1ccbef06eb6d4f192b30fb7efeb8280c9a6eac4 Mon Sep 17 00:00:00 2001 From: zhiru Date: Sun, 30 Jun 2019 15:20:08 +0800 Subject: [PATCH 32/43] update Former-commit-id: ebcc47ee9f40f0616f1b7035f407a787afd39ea0 --- cpp/coverage.sh | 19 +++++++----- cpp/unittest/db/MySQLMetaImpl_test.cpp | 40 +++++++++++++++++--------- cpp/unittest/db/db_tests.cpp | 30 +++++++++++-------- cpp/unittest/db/utils.cpp | 2 ++ 4 files changed, 59 insertions(+), 32 deletions(-) diff --git a/cpp/coverage.sh b/cpp/coverage.sh index 3415e752bd..54cf480703 100755 --- a/cpp/coverage.sh +++ b/cpp/coverage.sh @@ -20,15 +20,20 @@ if [ $? -ne 0 ]; then fi for test in `ls ${DIR_UNITTEST}`; do - echo $test - case ${test} in + case ${test} in + db_test) + # set run args for db_test + args="mysql://root:Fantast1c@192.168.1.194:3306/test" + ;; *_test) - # run unittest - ./${DIR_UNITTEST}/${test} - if [ $? -ne 0 ]; then - echo ${DIR_UNITTEST}/${test} "run failed" - fi + args="" + ;; esac + # run unittest + ./${DIR_UNITTEST}/${test} "${args}" + if [ $? -ne 0 ]; then + echo ${DIR_UNITTEST}/${test} "run failed" + fi done # gen test converage diff --git a/cpp/unittest/db/MySQLMetaImpl_test.cpp b/cpp/unittest/db/MySQLMetaImpl_test.cpp index 57fad28468..93e0ce9b28 100644 --- a/cpp/unittest/db/MySQLMetaImpl_test.cpp +++ b/cpp/unittest/db/MySQLMetaImpl_test.cpp @@ -36,7 +36,8 @@ TEST_F(MySQLTest, core) { // //dialect+driver://username:password@host:port/database // options.backend_uri = "mysql://root:1234@:/test"; // options.path = "/tmp/vecwise_test"; - meta::MySQLMetaImpl impl(getDBMetaOptions()); + int mode = Options::MODE::SINGLE; + meta::MySQLMetaImpl impl(getDBMetaOptions(), mode); // auto status = impl.Initialize(); // ASSERT_TRUE(status.ok()); @@ -58,7 +59,7 @@ TEST_F(MySQLTest, core) { status = impl.CreateTable(schema2); // std::cout << status.ToString() << std::endl; // ASSERT_THROW(impl.CreateTable(schema), mysqlpp::BadQuery); - ASSERT_FALSE(status.ok()); + ASSERT_TRUE(status.ok()); status = impl.DeleteTable(schema2.table_id_); // std::cout << status.ToString() << std::endl; @@ -191,8 +192,9 @@ TEST_F(MySQLTest, core) { } TEST_F(MySQLTest, GROUP_TEST) { - - meta::MySQLMetaImpl impl(getDBMetaOptions()); + + int mode = Options::MODE::SINGLE; + meta::MySQLMetaImpl impl(getDBMetaOptions(), mode); auto table_id = "meta_test_group"; @@ -214,7 +216,12 @@ TEST_F(MySQLTest, GROUP_TEST) { group.table_id_ = table_id; status = impl.CreateTable(group); - ASSERT_TRUE(!status.ok()); + ASSERT_TRUE(status.ok()); + + group.table_id_ = ""; + status = impl.CreateTable(group); + ASSERT_TRUE(status.ok()); + status = impl.DropAll(); ASSERT_TRUE(status.ok()); @@ -222,12 +229,14 @@ TEST_F(MySQLTest, GROUP_TEST) { TEST_F(MySQLTest, table_file_TEST) { - meta::MySQLMetaImpl impl(getDBMetaOptions()); + int mode = Options::MODE::SINGLE; + meta::MySQLMetaImpl impl(getDBMetaOptions(), mode); auto table_id = "meta_test_group"; meta::TableSchema group; group.table_id_ = table_id; + group.dimension_ = 256; auto status = impl.CreateTable(group); meta::TableFileSchema table_file; @@ -237,6 +246,11 @@ TEST_F(MySQLTest, table_file_TEST) { ASSERT_TRUE(status.ok()); ASSERT_EQ(table_file.file_type_, meta::TableFileSchema::NEW); + uint64_t cnt = 0; + status = impl.Count(table_id, cnt); + ASSERT_TRUE(status.ok()); + ASSERT_EQ(cnt, 0UL); + auto file_id = table_file.file_id_; auto new_file_type = meta::TableFileSchema::INDEX; @@ -287,8 +301,8 @@ TEST_F(MySQLTest, ARCHIVE_TEST_DAYS) { std::stringstream ss; ss << "days:" << days_num; options.archive_conf = ArchiveConf("delete", ss.str()); - - meta::MySQLMetaImpl impl(options); + int mode = Options::MODE::SINGLE; + meta::MySQLMetaImpl impl(options, mode); auto table_id = "meta_test_group"; @@ -336,11 +350,10 @@ TEST_F(MySQLTest, ARCHIVE_TEST_DAYS) { } TEST_F(MySQLTest, ARCHIVE_TEST_DISK) { - DBMetaOptions options; - options.path = "/tmp/milvus_test"; + DBMetaOptions options = getDBMetaOptions(); options.archive_conf = ArchiveConf("delete", "disk:11"); - - auto impl = meta::DBMetaImpl(options); + int mode = Options::MODE::SINGLE; + auto impl = meta::MySQLMetaImpl(options, mode); auto table_id = "meta_test_group"; meta::TableSchema group; @@ -385,7 +398,8 @@ TEST_F(MySQLTest, ARCHIVE_TEST_DISK) { TEST_F(MySQLTest, TABLE_FILES_TEST) { - auto impl = meta::DBMetaImpl(getDBMetaOptions()); + int mode = Options::MODE::SINGLE; + auto impl = meta::MySQLMetaImpl(getDBMetaOptions(), mode); auto table_id = "meta_test_group"; diff --git a/cpp/unittest/db/db_tests.cpp b/cpp/unittest/db/db_tests.cpp index b1738abcdf..b5eb8e642b 100644 --- a/cpp/unittest/db/db_tests.cpp +++ b/cpp/unittest/db/db_tests.cpp @@ -293,20 +293,14 @@ TEST_F(MySQLDBTest, DB_TEST) { auto options = GetOptions(); auto db_ = engine::DBFactory::Build(options); - static const std::string table_name = "test_group"; - static const int table_dim = 256; - - engine::meta::TableSchema table_info; - table_info.dimension_ = table_dim; - table_info.table_id_ = table_name; - table_info.engine_type_ = (int)engine::EngineType::FAISS_IDMAP; + engine::meta::TableSchema table_info = BuildTableSchema(); engine::Status stat = db_->CreateTable(table_info); engine::meta::TableSchema table_info_get; - table_info_get.table_id_ = table_name; + table_info_get.table_id_ = TABLE_NAME; stat = db_->DescribeTable(table_info_get); ASSERT_STATS(stat); - ASSERT_EQ(table_info_get.dimension_, table_dim); + ASSERT_EQ(table_info_get.dimension_, TABLE_DIM); engine::IDNumbers vector_ids; engine::IDNumbers target_ids; @@ -335,7 +329,7 @@ TEST_F(MySQLDBTest, DB_TEST) { prev_count = count; START_TIMER; - stat = db_->Query(table_name, k, qb, qxb.data(), results); + stat = db_->Query(TABLE_NAME, k, qb, qxb.data(), results); ss << "Search " << j << " With Size " << count/engine::meta::M << " M"; STOP_TIMER(ss.str()); @@ -358,10 +352,10 @@ TEST_F(MySQLDBTest, DB_TEST) { for (auto i=0; iInsertVectors(table_name, qb, qxb.data(), target_ids); + db_->InsertVectors(TABLE_NAME, qb, qxb.data(), target_ids); ASSERT_EQ(target_ids.size(), qb); } else { - db_->InsertVectors(table_name, nb, xb.data(), vector_ids); + db_->InsertVectors(TABLE_NAME, nb, xb.data(), vector_ids); } std::this_thread::sleep_for(std::chrono::microseconds(1)); } @@ -447,6 +441,18 @@ TEST_F(MySQLDBTest, ARHIVE_DISK_CHECK) { engine::meta::TableSchema table_info = BuildTableSchema(); engine::Status stat = db_->CreateTable(table_info); + std::vector table_schema_array; + stat = db_->AllTables(table_schema_array); + ASSERT_STATS(stat); + bool bfound = false; + for(auto& schema : table_schema_array) { + if(schema.table_id_ == TABLE_NAME) { + bfound = true; + break; + } + } + ASSERT_TRUE(bfound); + engine::meta::TableSchema table_info_get; table_info_get.table_id_ = TABLE_NAME; stat = db_->DescribeTable(table_info_get); diff --git a/cpp/unittest/db/utils.cpp b/cpp/unittest/db/utils.cpp index 1f2ec5399c..6b1fc1e407 100644 --- a/cpp/unittest/db/utils.cpp +++ b/cpp/unittest/db/utils.cpp @@ -53,6 +53,7 @@ void DBTest::InitLog() { engine::Options DBTest::GetOptions() { auto options = engine::OptionsFactory::Build(); options.meta.path = "/tmp/milvus_test"; + options.meta.backend_uri = "sqlite://:@:/"; return options; } @@ -71,6 +72,7 @@ engine::Options DBTest2::GetOptions() { auto options = engine::OptionsFactory::Build(); options.meta.path = "/tmp/milvus_test"; options.meta.archive_conf = engine::ArchiveConf("delete", "disk:1"); + options.meta.backend_uri = "sqlite://:@:/"; return options; } From 01d9689de044b362d96fabc51c562eaf20d914b2 Mon Sep 17 00:00:00 2001 From: groot Date: Sun, 30 Jun 2019 15:20:36 +0800 Subject: [PATCH 33/43] add more unittest Former-commit-id: 70b143a2a9464ed5d50692e9c093039010bbb060 --- cpp/unittest/db/meta_tests.cpp | 5 +++++ cpp/unittest/db/misc_test.cpp | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+) diff --git a/cpp/unittest/db/meta_tests.cpp b/cpp/unittest/db/meta_tests.cpp index 49cc47b4e0..0f3a92af09 100644 --- a/cpp/unittest/db/meta_tests.cpp +++ b/cpp/unittest/db/meta_tests.cpp @@ -264,4 +264,9 @@ TEST_F(MetaTest, TABLE_FILES_TEST) { ASSERT_TRUE(status.ok()); ASSERT_EQ(dated_files[table_file.date_].size(), to_index_files_cnt+raw_files_cnt+index_files_cnt); + + status = impl_->FilesToSearch(table_id, meta::DatesT(), dated_files); + ASSERT_TRUE(status.ok()); + ASSERT_EQ(dated_files[table_file.date_].size(), + to_index_files_cnt+raw_files_cnt+index_files_cnt); } diff --git a/cpp/unittest/db/misc_test.cpp b/cpp/unittest/db/misc_test.cpp index 6f1f87c874..4356746fc2 100644 --- a/cpp/unittest/db/misc_test.cpp +++ b/cpp/unittest/db/misc_test.cpp @@ -13,11 +13,19 @@ #include "db/Status.h" #include "db/Options.h" #include "db/DBMetaImpl.h" +#include "db/EngineFactory.h" #include using namespace zilliz::milvus; +namespace { + void CopyStatus(engine::Status& st1, engine::Status& st2) { + st1 = st2; + } + +} + TEST(DBMiscTest, ENGINE_API_TEST) { //engine api AddWithIdArray const uint16_t dim = 512; @@ -34,6 +42,15 @@ TEST(DBMiscTest, ENGINE_API_TEST) { auto status = engine.AddWithIdArray(vectors, ids); ASSERT_TRUE(status.ok()); + + auto engine_ptr = engine::EngineFactory::Build(128, "/tmp", engine::EngineType::INVALID); + ASSERT_EQ(engine_ptr, nullptr); + + engine_ptr = engine::EngineFactory::Build(128, "/tmp", engine::EngineType::FAISS_IVFFLAT); + ASSERT_NE(engine_ptr, nullptr); + + engine_ptr = engine::EngineFactory::Build(128, "/tmp", engine::EngineType::FAISS_IDMAP); + ASSERT_NE(engine_ptr, nullptr); } TEST(DBMiscTest, EXCEPTION_TEST) { @@ -65,6 +82,10 @@ TEST(DBMiscTest, STATUS_TEST) { ASSERT_TRUE(status.IsDBTransactionError()); str = status.ToString(); ASSERT_FALSE(str.empty()); + + engine::Status status_copy = engine::Status::OK(); + CopyStatus(status_copy, status); + ASSERT_TRUE(status.IsDBTransactionError()); } TEST(DBMiscTest, OPTIONS_TEST) { @@ -88,6 +109,19 @@ TEST(DBMiscTest, OPTIONS_TEST) { engine::ArchiveConf archive("delete", "1:2:3"); ASSERT_TRUE(archive.GetCriterias().empty()); } + + { + engine::ArchiveConf archive("delete"); + engine::ArchiveConf::CriteriaT criterial = { + {"disk", 1024}, + {"days", 100} + }; + archive.SetCriterias(criterial); + + auto crit = archive.GetCriterias(); + ASSERT_EQ(criterial["disk"], 1024); + ASSERT_EQ(criterial["days"], 100); + } } TEST(DBMiscTest, META_TEST) { From b487aaf789f151ab81b4f6de0b50275c95a26d1d Mon Sep 17 00:00:00 2001 From: zhiru Date: Sun, 30 Jun 2019 15:58:22 +0800 Subject: [PATCH 34/43] edit README.md Former-commit-id: dbfa99efc83ec995a40f24f17d2c4427f612dc71 --- cpp/README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/cpp/README.md b/cpp/README.md index c1cd381442..1b2f507db2 100644 --- a/cpp/README.md +++ b/cpp/README.md @@ -53,6 +53,12 @@ If you encounter the following error when building: ### Launch server Set config in cpp/conf/server_config.yaml +Add milvus/bin/lib to LD_LIBRARY_PATH + +``` +export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/path/to/milvus/bin/lib +``` + Then launch server with config: cd [build output path] start_server.sh From 9e87c4b147aabd53783a532003e71750925c633a Mon Sep 17 00:00:00 2001 From: zhiru Date: Sun, 30 Jun 2019 16:08:56 +0800 Subject: [PATCH 35/43] edit server_config.template Former-commit-id: e88640834e9999cda6eec885cdb91bcaefe1b566 --- cpp/conf/server_config.template | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/cpp/conf/server_config.template b/cpp/conf/server_config.template index 1a1c8303f2..6b432056c5 100644 --- a/cpp/conf/server_config.template +++ b/cpp/conf/server_config.template @@ -6,7 +6,12 @@ server_config: db_config: db_path: @MILVUS_DB_PATH@ # milvus data storage path - db_backend_url: http://127.0.0.1 # meta database uri + + # URI format: dialect://username:password@host:port/database + # All parts except dialect are optional, but you MUST include the delimiters + # Currently dialect supports mysql or sqlite + db_backend_url: dialect://username:password@host:port/database # meta database uri + index_building_threshold: 1024 # index building trigger threshold, default: 1024, unit: MB archive_disk_threshold: 512 # triger archive action if storage size exceed this value, unit: GB archive_days_threshold: 30 # files older than x days will be archived, unit: day From 2f12194cc068f9b4041cd900ca1597685a686dea Mon Sep 17 00:00:00 2001 From: groot Date: Sun, 30 Jun 2019 16:19:04 +0800 Subject: [PATCH 36/43] fix build error Former-commit-id: 403ec4922b0d18bec2039da8ca9bc0d944429b84 --- cpp/conf/server_config.template | 2 +- cpp/src/server/DBWrapper.cpp | 10 ++- cpp/unittest/metrics/CMakeLists.txt | 10 +-- cpp/unittest/metrics/metricbase_test.cpp | 2 +- cpp/unittest/metrics/metrics_test.cpp | 4 +- cpp/unittest/metrics/prometheus_test.cpp | 2 +- cpp/unittest/metrics/utils.cpp | 79 ++++++++++++++++++++++++ cpp/unittest/metrics/utils.h | 64 +++++++++++++++++++ 8 files changed, 161 insertions(+), 12 deletions(-) create mode 100644 cpp/unittest/metrics/utils.cpp create mode 100644 cpp/unittest/metrics/utils.h diff --git a/cpp/conf/server_config.template b/cpp/conf/server_config.template index 1a1c8303f2..6082969fae 100644 --- a/cpp/conf/server_config.template +++ b/cpp/conf/server_config.template @@ -6,7 +6,7 @@ server_config: db_config: db_path: @MILVUS_DB_PATH@ # milvus data storage path - db_backend_url: http://127.0.0.1 # meta database uri + db_backend_url: sqlite://:@:/ # meta database uri index_building_threshold: 1024 # index building trigger threshold, default: 1024, unit: MB archive_disk_threshold: 512 # triger archive action if storage size exceed this value, unit: GB archive_days_threshold: 30 # files older than x days will be archived, unit: day diff --git a/cpp/src/server/DBWrapper.cpp b/cpp/src/server/DBWrapper.cpp index a3db0bf110..bf859b3b4f 100644 --- a/cpp/src/server/DBWrapper.cpp +++ b/cpp/src/server/DBWrapper.cpp @@ -70,9 +70,15 @@ DBWrapper::DBWrapper() { kill(0, SIGUSR1); } - zilliz::milvus::engine::DB::Open(opt, &db_); + std::string msg = opt.meta.path; + try { + zilliz::milvus::engine::DB::Open(opt, &db_); + } catch(std::exception& ex) { + msg = ex.what(); + } + if(db_ == nullptr) { - std::cout << "ERROR! Failed to open database" << std::endl; + std::cout << "ERROR! Failed to open database: " << msg << std::endl; kill(0, SIGUSR1); } } diff --git a/cpp/unittest/metrics/CMakeLists.txt b/cpp/unittest/metrics/CMakeLists.txt index 80210772a8..d31e44c056 100644 --- a/cpp/unittest/metrics/CMakeLists.txt +++ b/cpp/unittest/metrics/CMakeLists.txt @@ -17,6 +17,7 @@ aux_source_directory(../../src/config config_files) aux_source_directory(../../src/cache cache_srcs) aux_source_directory(../../src/wrapper wrapper_src) aux_source_directory(../../src/metrics metrics_src) +aux_source_directory(./ test_srcs) aux_source_directory(${MILVUS_ENGINE_SRC}/db/scheduler scheduler_files) aux_source_directory(${MILVUS_ENGINE_SRC}/db/scheduler/context scheduler_context_files) @@ -35,6 +36,7 @@ link_directories("/usr/local/cuda/lib64") #include_directories(../db/utils.h) include_directories(../../src/metrics) +include_directories(/usr/include/mysql) #set(metrics_src_files # ../../src/metrics/Metrics.cpp @@ -47,17 +49,14 @@ include_directories(../../src/metrics) # ) set(count_test_src - ${unittest_srcs} ${config_files} ${cache_srcs} ${db_srcs} ${db_scheduler_srcs} ${wrapper_src} ${metrics_src} - metrics_test.cpp - prometheus_test.cpp - ../db/utils.cpp - metricbase_test.cpp) + ${test_srcs} + ) add_executable(metrics_test ${count_test_src} ${require_files} ) @@ -75,6 +74,7 @@ target_link_libraries(metrics_test gtest pthread z + mysqlpp ${unittest_libs} ) diff --git a/cpp/unittest/metrics/metricbase_test.cpp b/cpp/unittest/metrics/metricbase_test.cpp index ac850c7b48..1997748fdd 100644 --- a/cpp/unittest/metrics/metricbase_test.cpp +++ b/cpp/unittest/metrics/metricbase_test.cpp @@ -11,7 +11,7 @@ using namespace zilliz::milvus; -TEST(MetricbaseTest, Metricbase_Test){ +TEST(MetricbaseTest, METRICBASE_TEST){ server::MetricsBase instance = server::MetricsBase::GetInstance(); instance.Init(); server::SystemInfo::GetInstance().Init(); diff --git a/cpp/unittest/metrics/metrics_test.cpp b/cpp/unittest/metrics/metrics_test.cpp index 923c7b717b..883e63ed03 100644 --- a/cpp/unittest/metrics/metrics_test.cpp +++ b/cpp/unittest/metrics/metrics_test.cpp @@ -15,7 +15,7 @@ #include #include "metrics/Metrics.h" -#include "../db/utils.h" +#include "utils.h" #include "db/DB.h" #include "db/DBMetaImpl.h" #include "db/Factories.h" @@ -24,7 +24,7 @@ using namespace zilliz::milvus; -TEST_F(DBTest, Metric_Tes) { +TEST_F(MetricTest, Metric_Tes) { server::SystemInfo::GetInstance().Init(); // server::Metrics::GetInstance().Init(); diff --git a/cpp/unittest/metrics/prometheus_test.cpp b/cpp/unittest/metrics/prometheus_test.cpp index 885abed566..521e00fc5c 100644 --- a/cpp/unittest/metrics/prometheus_test.cpp +++ b/cpp/unittest/metrics/prometheus_test.cpp @@ -11,7 +11,7 @@ using namespace zilliz::milvus; -TEST(PrometheusTest, Prometheus_Test){ +TEST(PrometheusTest, PROMETHEUS_TEST){ server::PrometheusMetrics instance = server::PrometheusMetrics::GetInstance(); instance.Init(); instance.SetStartup(true); diff --git a/cpp/unittest/metrics/utils.cpp b/cpp/unittest/metrics/utils.cpp new file mode 100644 index 0000000000..81e924a87e --- /dev/null +++ b/cpp/unittest/metrics/utils.cpp @@ -0,0 +1,79 @@ +//////////////////////////////////////////////////////////////////////////////// +// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved +// Unauthorized copying of this file, via any medium is strictly prohibited. +// Proprietary and confidential. +//////////////////////////////////////////////////////////////////////////////// + +#include +#include +#include +#include + +#include "utils.h" +#include "db/Factories.h" +#include "db/Options.h" + +INITIALIZE_EASYLOGGINGPP + +using namespace zilliz::milvus; + +static std::string uri; + +class DBTestEnvironment : public ::testing::Environment { +public: + +// explicit DBTestEnvironment(std::string uri) : uri_(uri) {} + + static std::string getURI() { + return uri; + } + + void SetUp() override { + getURI(); + } + +}; + +void ASSERT_STATS(engine::Status& stat) { + ASSERT_TRUE(stat.ok()); + if(!stat.ok()) { + std::cout << stat.ToString() << std::endl; + } +} + + +void MetricTest::InitLog() { + el::Configurations defaultConf; + defaultConf.setToDefault(); + defaultConf.set(el::Level::Debug, + el::ConfigurationType::Format, "[%thread-%datetime-%level]: %msg (%fbase:%line)"); + el::Loggers::reconfigureLogger("default", defaultConf); +} + +engine::Options MetricTest::GetOptions() { + auto options = engine::OptionsFactory::Build(); + options.meta.path = "/tmp/milvus_test"; + options.meta.backend_uri = "sqlite://:@:/"; + return options; +} + +void MetricTest::SetUp() { + InitLog(); + auto options = GetOptions(); + db_ = engine::DBFactory::Build(options); +} + +void MetricTest::TearDown() { + delete db_; + boost::filesystem::remove_all("/tmp/milvus_test"); +} + +int main(int argc, char **argv) { + ::testing::InitGoogleTest(&argc, argv); + if (argc > 1) { + uri = argv[1]; + } +// std::cout << uri << std::endl; + ::testing::AddGlobalTestEnvironment(new DBTestEnvironment); + return RUN_ALL_TESTS(); +} diff --git a/cpp/unittest/metrics/utils.h b/cpp/unittest/metrics/utils.h new file mode 100644 index 0000000000..1badce00f2 --- /dev/null +++ b/cpp/unittest/metrics/utils.h @@ -0,0 +1,64 @@ +//////////////////////////////////////////////////////////////////////////////// +// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved +// Unauthorized copying of this file, via any medium is strictly prohibited. +// Proprietary and confidential. +//////////////////////////////////////////////////////////////////////////////// + +#pragma once + +#include +#include +//#include + +#include "db/DB.h" +#include "db/DBMetaImpl.h" +#include "db/MySQLMetaImpl.h" + + +#define TIMING + +#ifdef TIMING +#define INIT_TIMER auto start = std::chrono::high_resolution_clock::now(); +#define START_TIMER start = std::chrono::high_resolution_clock::now(); +#define STOP_TIMER(name) LOG(DEBUG) << "RUNTIME of " << name << ": " << \ + std::chrono::duration_cast( \ + std::chrono::high_resolution_clock::now()-start \ + ).count() << " ms "; +#else +#define INIT_TIMER +#define START_TIMER +#define STOP_TIMER(name) +#endif + +void ASSERT_STATS(zilliz::milvus::engine::Status& stat); + +//class TestEnv : public ::testing::Environment { +//public: +// +// static std::string getURI() { +// if (const char* uri = std::getenv("MILVUS_DBMETA_URI")) { +// return uri; +// } +// else { +// return ""; +// } +// } +// +// void SetUp() override { +// getURI(); +// } +// +//}; +// +//::testing::Environment* const test_env = +// ::testing::AddGlobalTestEnvironment(new TestEnv); + +class MetricTest : public ::testing::Test { +protected: + zilliz::milvus::engine::DB* db_; + + void InitLog(); + virtual void SetUp() override; + virtual void TearDown() override; + virtual zilliz::milvus::engine::Options GetOptions(); +}; \ No newline at end of file From 555c31c4eff2944312a05b6a6f71984f20ab5b59 Mon Sep 17 00:00:00 2001 From: groot Date: Sun, 30 Jun 2019 16:42:12 +0800 Subject: [PATCH 37/43] avoid unitest hang Former-commit-id: c1a58a96da3554b9e83795df706ba40a8de673a2 --- cpp/unittest/db/MySQLMetaImpl_test.cpp | 56 ++++++++++++++++++++++---- cpp/unittest/db/utils.cpp | 5 +++ 2 files changed, 54 insertions(+), 7 deletions(-) diff --git a/cpp/unittest/db/MySQLMetaImpl_test.cpp b/cpp/unittest/db/MySQLMetaImpl_test.cpp index 93e0ce9b28..a370fd1cd9 100644 --- a/cpp/unittest/db/MySQLMetaImpl_test.cpp +++ b/cpp/unittest/db/MySQLMetaImpl_test.cpp @@ -32,12 +32,19 @@ using namespace zilliz::milvus::engine; //} TEST_F(MySQLTest, core) { -// DBMetaOptions options; + DBMetaOptions options; // //dialect+driver://username:password@host:port/database // options.backend_uri = "mysql://root:1234@:/test"; // options.path = "/tmp/vecwise_test"; + try { + options = getDBMetaOptions(); + } catch(std::exception& ex) { + ASSERT_TRUE(false); + return; + } + int mode = Options::MODE::SINGLE; - meta::MySQLMetaImpl impl(getDBMetaOptions(), mode); + meta::MySQLMetaImpl impl(options, mode); // auto status = impl.Initialize(); // ASSERT_TRUE(status.ok()); @@ -192,9 +199,16 @@ TEST_F(MySQLTest, core) { } TEST_F(MySQLTest, GROUP_TEST) { + DBMetaOptions options; + try { + options = getDBMetaOptions(); + } catch(std::exception& ex) { + ASSERT_TRUE(false); + return; + } int mode = Options::MODE::SINGLE; - meta::MySQLMetaImpl impl(getDBMetaOptions(), mode); + meta::MySQLMetaImpl impl(options, mode); auto table_id = "meta_test_group"; @@ -228,9 +242,16 @@ TEST_F(MySQLTest, GROUP_TEST) { } TEST_F(MySQLTest, table_file_TEST) { + DBMetaOptions options; + try { + options = getDBMetaOptions(); + } catch(std::exception& ex) { + ASSERT_TRUE(false); + return; + } int mode = Options::MODE::SINGLE; - meta::MySQLMetaImpl impl(getDBMetaOptions(), mode); + meta::MySQLMetaImpl impl(options, mode); auto table_id = "meta_test_group"; @@ -296,7 +317,14 @@ TEST_F(MySQLTest, table_file_TEST) { TEST_F(MySQLTest, ARCHIVE_TEST_DAYS) { srand(time(0)); - DBMetaOptions options = getDBMetaOptions(); + DBMetaOptions options; + try { + options = getDBMetaOptions(); + } catch(std::exception& ex) { + ASSERT_TRUE(false); + return; + } + int days_num = rand() % 100; std::stringstream ss; ss << "days:" << days_num; @@ -350,7 +378,14 @@ TEST_F(MySQLTest, ARCHIVE_TEST_DAYS) { } TEST_F(MySQLTest, ARCHIVE_TEST_DISK) { - DBMetaOptions options = getDBMetaOptions(); + DBMetaOptions options; + try { + options = getDBMetaOptions(); + } catch(std::exception& ex) { + ASSERT_TRUE(false); + return; + } + options.archive_conf = ArchiveConf("delete", "disk:11"); int mode = Options::MODE::SINGLE; auto impl = meta::MySQLMetaImpl(options, mode); @@ -397,9 +432,16 @@ TEST_F(MySQLTest, ARCHIVE_TEST_DISK) { } TEST_F(MySQLTest, TABLE_FILES_TEST) { + DBMetaOptions options; + try { + options = getDBMetaOptions(); + } catch(std::exception& ex) { + ASSERT_TRUE(false); + return; + } int mode = Options::MODE::SINGLE; - auto impl = meta::MySQLMetaImpl(getDBMetaOptions(), mode); + auto impl = meta::MySQLMetaImpl(options, mode); auto table_id = "meta_test_group"; diff --git a/cpp/unittest/db/utils.cpp b/cpp/unittest/db/utils.cpp index 6b1fc1e407..70c0712549 100644 --- a/cpp/unittest/db/utils.cpp +++ b/cpp/unittest/db/utils.cpp @@ -91,6 +91,11 @@ zilliz::milvus::engine::DBMetaOptions MySQLTest::getDBMetaOptions() { zilliz::milvus::engine::DBMetaOptions options; options.path = "/tmp/milvus_test"; options.backend_uri = DBTestEnvironment::getURI(); + + if(options.backend_uri.empty()) { + throw std::exception(); + } + return options; } From 385875e196844aa94a68839a28ed0b60289addd2 Mon Sep 17 00:00:00 2001 From: groot Date: Sun, 30 Jun 2019 17:05:46 +0800 Subject: [PATCH 38/43] avoid unitest hang Former-commit-id: 97dc514f8f5cec2d6c1bbe85a89241c27a85b6ae --- .../db/{MySQLMetaImpl_test.cpp => mysql_meta_test.cpp} | 5 +++++ 1 file changed, 5 insertions(+) rename cpp/unittest/db/{MySQLMetaImpl_test.cpp => mysql_meta_test.cpp} (98%) diff --git a/cpp/unittest/db/MySQLMetaImpl_test.cpp b/cpp/unittest/db/mysql_meta_test.cpp similarity index 98% rename from cpp/unittest/db/MySQLMetaImpl_test.cpp rename to cpp/unittest/db/mysql_meta_test.cpp index a370fd1cd9..436086acb3 100644 --- a/cpp/unittest/db/MySQLMetaImpl_test.cpp +++ b/cpp/unittest/db/mysql_meta_test.cpp @@ -502,6 +502,11 @@ TEST_F(MySQLTest, TABLE_FILES_TEST) { ASSERT_EQ(dated_files[table_file.date_].size(), to_index_files_cnt+raw_files_cnt+index_files_cnt); + status = impl.FilesToSearch(table_id, meta::DatesT(), dated_files); + ASSERT_TRUE(status.ok()); + ASSERT_EQ(dated_files[table_file.date_].size(), + to_index_files_cnt+raw_files_cnt+index_files_cnt); + status = impl.DropAll(); ASSERT_TRUE(status.ok()); } From 26f182c7afdf9851ae1cac4a4cf41fd35804119d Mon Sep 17 00:00:00 2001 From: groot Date: Sun, 30 Jun 2019 18:53:43 +0800 Subject: [PATCH 39/43] refine unitest code Former-commit-id: 7a78a5d5bd966d34d9ccd6f52e3b1c533da3a1a0 --- cpp/unittest/db/db_tests.cpp | 258 +------------------------- cpp/unittest/db/mysql_db_test.cpp | 293 ++++++++++++++++++++++++++++++ 2 files changed, 299 insertions(+), 252 deletions(-) create mode 100644 cpp/unittest/db/mysql_db_test.cpp diff --git a/cpp/unittest/db/db_tests.cpp b/cpp/unittest/db/db_tests.cpp index 8e50b7403b..d505320e86 100644 --- a/cpp/unittest/db/db_tests.cpp +++ b/cpp/unittest/db/db_tests.cpp @@ -20,6 +20,8 @@ namespace { static const std::string TABLE_NAME = "test_group"; static constexpr int64_t TABLE_DIM = 256; + static constexpr int64_t VECTOR_COUNT = 250000; + static constexpr int64_t INSERT_LOOP = 100000; engine::meta::TableSchema BuildTableSchema() { engine::meta::TableSchema table_info; @@ -144,7 +146,7 @@ TEST_F(DBTest, DB_TEST) { } }); - int loop = 100000; + int loop = INSERT_LOOP; for (auto i=0; i xb(nb*TABLE_DIM); @@ -246,7 +248,7 @@ TEST_F(DBTest2, ARHIVE_DISK_CHECK) { std::vector xb; BuildVectors(nb, xb); - int loop = 100000; + int loop = INSERT_LOOP; for (auto i=0; iInsertVectors(TABLE_NAME, nb, xb.data(), vector_ids); std::this_thread::sleep_for(std::chrono::microseconds(1)); @@ -277,7 +279,7 @@ TEST_F(DBTest2, DELETE_TEST) { uint64_t size; db_->Size(size); - int64_t nb = 100000; + int64_t nb = INSERT_LOOP; std::vector xb; BuildVectors(nb, xb); @@ -293,251 +295,3 @@ TEST_F(DBTest2, DELETE_TEST) { ASSERT_TRUE(stat.ok()); ASSERT_FALSE(boost::filesystem::exists(table_info_get.location_)); }; - -TEST_F(MySQLDBTest, DB_TEST) { - - auto options = GetOptions(); - auto db_ = engine::DBFactory::Build(options); - - engine::meta::TableSchema table_info = BuildTableSchema(); - engine::Status stat = db_->CreateTable(table_info); - - engine::meta::TableSchema table_info_get; - table_info_get.table_id_ = TABLE_NAME; - stat = db_->DescribeTable(table_info_get); - ASSERT_STATS(stat); - ASSERT_EQ(table_info_get.dimension_, TABLE_DIM); - - engine::IDNumbers vector_ids; - engine::IDNumbers target_ids; - - int64_t nb = 50; - std::vector xb; - BuildVectors(nb, xb); - - int64_t qb = 5; - std::vector qxb; - BuildVectors(qb, qxb); - - std::thread search([&]() { - engine::QueryResults results; - int k = 10; - std::this_thread::sleep_for(std::chrono::seconds(2)); - - INIT_TIMER; - std::stringstream ss; - uint64_t count = 0; - uint64_t prev_count = 0; - - for (auto j=0; j<10; ++j) { - ss.str(""); - db_->Size(count); - prev_count = count; - - START_TIMER; - stat = db_->Query(TABLE_NAME, k, qb, qxb.data(), results); - ss << "Search " << j << " With Size " << count/engine::meta::M << " M"; - STOP_TIMER(ss.str()); - - ASSERT_STATS(stat); - for (auto k=0; k= prev_count); - std::this_thread::sleep_for(std::chrono::seconds(1)); - } - }); - - int loop = 100000; - - for (auto i=0; iInsertVectors(TABLE_NAME, qb, qxb.data(), target_ids); - ASSERT_EQ(target_ids.size(), qb); - } else { - db_->InsertVectors(TABLE_NAME, nb, xb.data(), vector_ids); - } - std::this_thread::sleep_for(std::chrono::microseconds(1)); - } - - search.join(); - - delete db_; - - auto dummyDB = engine::DBFactory::Build(options); - dummyDB->DropAll(); - delete dummyDB; -}; - -TEST_F(MySQLDBTest, SEARCH_TEST) { - auto options = GetOptions(); - auto db_ = engine::DBFactory::Build(options); - - engine::meta::TableSchema table_info = BuildTableSchema(); - engine::Status stat = db_->CreateTable(table_info); - - engine::meta::TableSchema table_info_get; - table_info_get.table_id_ = TABLE_NAME; - stat = db_->DescribeTable(table_info_get); - ASSERT_STATS(stat); - ASSERT_EQ(table_info_get.dimension_, TABLE_DIM); - - // prepare raw data - size_t nb = 250000; - size_t nq = 10; - size_t k = 5; - std::vector xb(nb*TABLE_DIM); - std::vector xq(nq*TABLE_DIM); - std::vector ids(nb); - - std::random_device rd; - std::mt19937 gen(rd()); - std::uniform_real_distribution<> dis_xt(-1.0, 1.0); - for (size_t i = 0; i < nb*TABLE_DIM; i++) { - xb[i] = dis_xt(gen); - if (i < nb){ - ids[i] = i; - } - } - for (size_t i = 0; i < nq*TABLE_DIM; i++) { - xq[i] = dis_xt(gen); - } - - // result data - //std::vector nns_gt(k*nq); - std::vector nns(k*nq); // nns = nearst neg search - //std::vector dis_gt(k*nq); - std::vector dis(k*nq); - - // insert data - const int batch_size = 100; - for (int j = 0; j < nb / batch_size; ++j) { - stat = db_->InsertVectors(TABLE_NAME, batch_size, xb.data()+batch_size*j*TABLE_DIM, ids); - if (j == 200){ sleep(1);} - ASSERT_STATS(stat); - } - - sleep(2); // wait until build index finish - - engine::QueryResults results; - stat = db_->Query(TABLE_NAME, k, nq, xq.data(), results); - ASSERT_STATS(stat); - - delete db_; - - auto dummyDB = engine::DBFactory::Build(options); - dummyDB->DropAll(); - delete dummyDB; - - // TODO(linxj): add groundTruth assert -}; - -TEST_F(MySQLDBTest, ARHIVE_DISK_CHECK) { - - auto options = GetOptions(); - options.meta.archive_conf = engine::ArchiveConf("delete", "disk:1"); - auto db_ = engine::DBFactory::Build(options); - - engine::meta::TableSchema table_info = BuildTableSchema(); - engine::Status stat = db_->CreateTable(table_info); - - std::vector table_schema_array; - stat = db_->AllTables(table_schema_array); - ASSERT_STATS(stat); - bool bfound = false; - for(auto& schema : table_schema_array) { - if(schema.table_id_ == TABLE_NAME) { - bfound = true; - break; - } - } - ASSERT_TRUE(bfound); - - engine::meta::TableSchema table_info_get; - table_info_get.table_id_ = TABLE_NAME; - stat = db_->DescribeTable(table_info_get); - ASSERT_STATS(stat); - ASSERT_EQ(table_info_get.dimension_, TABLE_DIM); - - engine::IDNumbers vector_ids; - engine::IDNumbers target_ids; - - uint64_t size; - db_->Size(size); - - int64_t nb = 10; - std::vector xb; - BuildVectors(nb, xb); - - int loop = 100000; - for (auto i=0; iInsertVectors(TABLE_NAME, nb, xb.data(), vector_ids); - std::this_thread::sleep_for(std::chrono::microseconds(1)); - } - - std::this_thread::sleep_for(std::chrono::seconds(1)); - - db_->Size(size); - LOG(DEBUG) << "size=" << size; - ASSERT_LE(size, 1 * engine::meta::G); - - delete db_; - - auto dummyDB = engine::DBFactory::Build(options); - dummyDB->DropAll(); - delete dummyDB; -}; - -TEST_F(MySQLDBTest, DELETE_TEST) { - - auto options = GetOptions(); - options.meta.archive_conf = engine::ArchiveConf("delete", "disk:1"); - auto db_ = engine::DBFactory::Build(options); - - engine::meta::TableSchema table_info = BuildTableSchema(); - engine::Status stat = db_->CreateTable(table_info); -// std::cout << stat.ToString() << std::endl; - - engine::meta::TableSchema table_info_get; - table_info_get.table_id_ = TABLE_NAME; - stat = db_->DescribeTable(table_info_get); - ASSERT_STATS(stat); - -// std::cout << "location: " << table_info_get.location_ << std::endl; - ASSERT_TRUE(boost::filesystem::exists(table_info_get.location_)); - - engine::IDNumbers vector_ids; - - uint64_t size; - db_->Size(size); - - int64_t nb = 100000; - std::vector xb; - BuildVectors(nb, xb); - - int loop = 20; - for (auto i=0; iInsertVectors(TABLE_NAME, nb, xb.data(), vector_ids); - std::this_thread::sleep_for(std::chrono::microseconds(1)); - } - - std::vector dates; - stat = db_->DeleteTable(TABLE_NAME, dates); -// std::cout << "5 sec start" << std::endl; - std::this_thread::sleep_for(std::chrono::seconds(5)); -// std::cout << "5 sec finish" << std::endl; - ASSERT_TRUE(stat.ok()); -// ASSERT_FALSE(boost::filesystem::exists(table_info_get.location_)); - - delete db_; - - auto dummyDB = engine::DBFactory::Build(options); - dummyDB->DropAll(); - delete dummyDB; -}; diff --git a/cpp/unittest/db/mysql_db_test.cpp b/cpp/unittest/db/mysql_db_test.cpp new file mode 100644 index 0000000000..db3c84751e --- /dev/null +++ b/cpp/unittest/db/mysql_db_test.cpp @@ -0,0 +1,293 @@ +//////////////////////////////////////////////////////////////////////////////// +// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved +// Unauthorized copying of this file, via any medium is strictly prohibited. +// Proprietary and confidential. +//////////////////////////////////////////////////////////////////////////////// +#include +#include +#include +#include + +#include "utils.h" +#include "db/DB.h" +#include "db/DBImpl.h" +#include "db/MetaConsts.h" +#include "db/Factories.h" + +using namespace zilliz::milvus; + +namespace { + +static const std::string TABLE_NAME = "test_group"; +static constexpr int64_t TABLE_DIM = 256; +static constexpr int64_t VECTOR_COUNT = 250000; +static constexpr int64_t INSERT_LOOP = 100000; + +engine::meta::TableSchema BuildTableSchema() { + engine::meta::TableSchema table_info; + table_info.dimension_ = TABLE_DIM; + table_info.table_id_ = TABLE_NAME; + table_info.engine_type_ = (int)engine::EngineType::FAISS_IDMAP; + return table_info; +} + +void BuildVectors(int64_t n, std::vector& vectors) { + vectors.clear(); + vectors.resize(n*TABLE_DIM); + float* data = vectors.data(); + for(int i = 0; i < n; i++) { + for(int j = 0; j < TABLE_DIM; j++) data[TABLE_DIM * i + j] = drand48(); + data[TABLE_DIM * i] += i / 2000.; + } +} + +} + + +TEST_F(MySQLDBTest, DB_TEST) { + + auto options = GetOptions(); + auto db_ = engine::DBFactory::Build(options); + + engine::meta::TableSchema table_info = BuildTableSchema(); + engine::Status stat = db_->CreateTable(table_info); + + engine::meta::TableSchema table_info_get; + table_info_get.table_id_ = TABLE_NAME; + stat = db_->DescribeTable(table_info_get); + ASSERT_STATS(stat); + ASSERT_EQ(table_info_get.dimension_, TABLE_DIM); + + engine::IDNumbers vector_ids; + engine::IDNumbers target_ids; + + int64_t nb = 50; + std::vector xb; + BuildVectors(nb, xb); + + int64_t qb = 5; + std::vector qxb; + BuildVectors(qb, qxb); + + std::thread search([&]() { + engine::QueryResults results; + int k = 10; + std::this_thread::sleep_for(std::chrono::seconds(2)); + + INIT_TIMER; + std::stringstream ss; + uint64_t count = 0; + uint64_t prev_count = 0; + + for (auto j=0; j<10; ++j) { + ss.str(""); + db_->Size(count); + prev_count = count; + + START_TIMER; + stat = db_->Query(TABLE_NAME, k, qb, qxb.data(), results); + ss << "Search " << j << " With Size " << count/engine::meta::M << " M"; + STOP_TIMER(ss.str()); + + ASSERT_STATS(stat); + for (auto k=0; k= prev_count); + std::this_thread::sleep_for(std::chrono::seconds(1)); + } + }); + + int loop = INSERT_LOOP; + + for (auto i=0; iInsertVectors(TABLE_NAME, qb, qxb.data(), target_ids); + ASSERT_EQ(target_ids.size(), qb); + } else { + db_->InsertVectors(TABLE_NAME, nb, xb.data(), vector_ids); + } + std::this_thread::sleep_for(std::chrono::microseconds(1)); + } + + search.join(); + + delete db_; + + auto dummyDB = engine::DBFactory::Build(options); + dummyDB->DropAll(); + delete dummyDB; +}; + +TEST_F(MySQLDBTest, SEARCH_TEST) { + auto options = GetOptions(); + auto db_ = engine::DBFactory::Build(options); + + engine::meta::TableSchema table_info = BuildTableSchema(); + engine::Status stat = db_->CreateTable(table_info); + + engine::meta::TableSchema table_info_get; + table_info_get.table_id_ = TABLE_NAME; + stat = db_->DescribeTable(table_info_get); + ASSERT_STATS(stat); + ASSERT_EQ(table_info_get.dimension_, TABLE_DIM); + + // prepare raw data + size_t nb = VECTOR_COUNT; + size_t nq = 10; + size_t k = 5; + std::vector xb(nb*TABLE_DIM); + std::vector xq(nq*TABLE_DIM); + std::vector ids(nb); + + std::random_device rd; + std::mt19937 gen(rd()); + std::uniform_real_distribution<> dis_xt(-1.0, 1.0); + for (size_t i = 0; i < nb*TABLE_DIM; i++) { + xb[i] = dis_xt(gen); + if (i < nb){ + ids[i] = i; + } + } + for (size_t i = 0; i < nq*TABLE_DIM; i++) { + xq[i] = dis_xt(gen); + } + + // result data + //std::vector nns_gt(k*nq); + std::vector nns(k*nq); // nns = nearst neg search + //std::vector dis_gt(k*nq); + std::vector dis(k*nq); + + // insert data + const int batch_size = 100; + for (int j = 0; j < nb / batch_size; ++j) { + stat = db_->InsertVectors(TABLE_NAME, batch_size, xb.data()+batch_size*j*TABLE_DIM, ids); + if (j == 200){ sleep(1);} + ASSERT_STATS(stat); + } + + sleep(2); // wait until build index finish + + engine::QueryResults results; + stat = db_->Query(TABLE_NAME, k, nq, xq.data(), results); + ASSERT_STATS(stat); + + delete db_; + + auto dummyDB = engine::DBFactory::Build(options); + dummyDB->DropAll(); + delete dummyDB; + + // TODO(linxj): add groundTruth assert +}; + +TEST_F(MySQLDBTest, ARHIVE_DISK_CHECK) { + + auto options = GetOptions(); + options.meta.archive_conf = engine::ArchiveConf("delete", "disk:1"); + auto db_ = engine::DBFactory::Build(options); + + engine::meta::TableSchema table_info = BuildTableSchema(); + engine::Status stat = db_->CreateTable(table_info); + + std::vector table_schema_array; + stat = db_->AllTables(table_schema_array); + ASSERT_STATS(stat); + bool bfound = false; + for(auto& schema : table_schema_array) { + if(schema.table_id_ == TABLE_NAME) { + bfound = true; + break; + } + } + ASSERT_TRUE(bfound); + + engine::meta::TableSchema table_info_get; + table_info_get.table_id_ = TABLE_NAME; + stat = db_->DescribeTable(table_info_get); + ASSERT_STATS(stat); + ASSERT_EQ(table_info_get.dimension_, TABLE_DIM); + + engine::IDNumbers vector_ids; + engine::IDNumbers target_ids; + + uint64_t size; + db_->Size(size); + + int64_t nb = 10; + std::vector xb; + BuildVectors(nb, xb); + + int loop = INSERT_LOOP; + for (auto i=0; iInsertVectors(TABLE_NAME, nb, xb.data(), vector_ids); + std::this_thread::sleep_for(std::chrono::microseconds(1)); + } + + std::this_thread::sleep_for(std::chrono::seconds(1)); + + db_->Size(size); + LOG(DEBUG) << "size=" << size; + ASSERT_LE(size, 1 * engine::meta::G); + + delete db_; + + auto dummyDB = engine::DBFactory::Build(options); + dummyDB->DropAll(); + delete dummyDB; +}; + +TEST_F(MySQLDBTest, DELETE_TEST) { + + auto options = GetOptions(); + options.meta.archive_conf = engine::ArchiveConf("delete", "disk:1"); + auto db_ = engine::DBFactory::Build(options); + + engine::meta::TableSchema table_info = BuildTableSchema(); + engine::Status stat = db_->CreateTable(table_info); +// std::cout << stat.ToString() << std::endl; + + engine::meta::TableSchema table_info_get; + table_info_get.table_id_ = TABLE_NAME; + stat = db_->DescribeTable(table_info_get); + ASSERT_STATS(stat); + +// std::cout << "location: " << table_info_get.location_ << std::endl; + ASSERT_TRUE(boost::filesystem::exists(table_info_get.location_)); + + engine::IDNumbers vector_ids; + + uint64_t size; + db_->Size(size); + + int64_t nb = INSERT_LOOP; + std::vector xb; + BuildVectors(nb, xb); + + int loop = 20; + for (auto i=0; iInsertVectors(TABLE_NAME, nb, xb.data(), vector_ids); + std::this_thread::sleep_for(std::chrono::microseconds(1)); + } + + std::vector dates; + stat = db_->DeleteTable(TABLE_NAME, dates); +// std::cout << "5 sec start" << std::endl; + std::this_thread::sleep_for(std::chrono::seconds(5)); +// std::cout << "5 sec finish" << std::endl; + ASSERT_TRUE(stat.ok()); +// ASSERT_FALSE(boost::filesystem::exists(table_info_get.location_)); + + delete db_; + + auto dummyDB = engine::DBFactory::Build(options); + dummyDB->DropAll(); + delete dummyDB; +}; From c659f0fcfbd8bfc6603629ebfd0c22913c7f0ca7 Mon Sep 17 00:00:00 2001 From: zhiru Date: Sun, 30 Jun 2019 19:00:53 +0800 Subject: [PATCH 40/43] update Former-commit-id: 64ac42d61ae8aa38599324b5f72e93e66507aa01 --- cpp/coverage.sh | 2 ++ cpp/src/CMakeLists.txt | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/cpp/coverage.sh b/cpp/coverage.sh index 54cf480703..cc509b611b 100755 --- a/cpp/coverage.sh +++ b/cpp/coverage.sh @@ -1,5 +1,7 @@ #!/bin/bash +export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$(pwd)/milvus/lib + LCOV_CMD="lcov" LCOV_GEN_CMD="genhtml" diff --git a/cpp/src/CMakeLists.txt b/cpp/src/CMakeLists.txt index 6c1bd12771..00e2e1e87d 100644 --- a/cpp/src/CMakeLists.txt +++ b/cpp/src/CMakeLists.txt @@ -187,6 +187,6 @@ install(FILES ${CMAKE_BINARY_DIR}/mysqlpp_ep-prefix/src/mysqlpp_ep/lib/${CMAKE_SHARED_LIBRARY_PREFIX}mysqlpp${CMAKE_SHARED_LIBRARY_SUFFIX} ${CMAKE_BINARY_DIR}/mysqlpp_ep-prefix/src/mysqlpp_ep/lib/${CMAKE_SHARED_LIBRARY_PREFIX}mysqlpp${CMAKE_SHARED_LIBRARY_SUFFIX}.3 ${CMAKE_BINARY_DIR}/mysqlpp_ep-prefix/src/mysqlpp_ep/lib/${CMAKE_SHARED_LIBRARY_PREFIX}mysqlpp${CMAKE_SHARED_LIBRARY_SUFFIX}.3.2.4 - DESTINATION bin/lib) #need to copy libmysqlpp.so + DESTINATION lib) #need to copy libmysqlpp.so add_subdirectory(sdk) From ea3fdf6603d3564a55f826619e6dd163d7120d29 Mon Sep 17 00:00:00 2001 From: groot Date: Sun, 30 Jun 2019 19:07:00 +0800 Subject: [PATCH 41/43] ignore c sdk Former-commit-id: 4bb374ff328198226ae3ab78e5901ed6264713e7 --- cpp/src/CMakeLists.txt | 2 +- cpp/src/sdk/CMakeLists.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cpp/src/CMakeLists.txt b/cpp/src/CMakeLists.txt index 6c1bd12771..4cde99bb0d 100644 --- a/cpp/src/CMakeLists.txt +++ b/cpp/src/CMakeLists.txt @@ -189,4 +189,4 @@ install(FILES ${CMAKE_BINARY_DIR}/mysqlpp_ep-prefix/src/mysqlpp_ep/lib/${CMAKE_SHARED_LIBRARY_PREFIX}mysqlpp${CMAKE_SHARED_LIBRARY_SUFFIX}.3.2.4 DESTINATION bin/lib) #need to copy libmysqlpp.so -add_subdirectory(sdk) +#add_subdirectory(sdk) diff --git a/cpp/src/sdk/CMakeLists.txt b/cpp/src/sdk/CMakeLists.txt index a43f0b85de..b51c2d5e09 100644 --- a/cpp/src/sdk/CMakeLists.txt +++ b/cpp/src/sdk/CMakeLists.txt @@ -32,4 +32,4 @@ target_link_libraries(milvus_sdk add_subdirectory(examples) -install(TARGETS milvus_sdk DESTINATION bin) +install(TARGETS milvus_sdk DESTINATION lib) From 5c65f2a865125a3e81ef35caaed0e71cc5dc3eda Mon Sep 17 00:00:00 2001 From: zhiru Date: Mon, 1 Jul 2019 15:29:12 +0800 Subject: [PATCH 42/43] update Former-commit-id: 2b2dae7736ed5a79b0bc610a1a51fd5361c8d5cb --- cpp/conf/server_config.template | 1 + cpp/coverage.sh | 31 +++- cpp/src/db/MySQLConnectionPool.h | 4 + cpp/src/db/MySQLMetaImpl.cpp | 23 ++- cpp/unittest/db/mysql_meta_test.cpp | 253 ++++------------------------ 5 files changed, 85 insertions(+), 227 deletions(-) diff --git a/cpp/conf/server_config.template b/cpp/conf/server_config.template index c2ed775601..b5ba2134b9 100644 --- a/cpp/conf/server_config.template +++ b/cpp/conf/server_config.template @@ -15,6 +15,7 @@ db_config: index_building_threshold: 1024 # index building trigger threshold, default: 1024, unit: MB archive_disk_threshold: 512 # triger archive action if storage size exceed this value, unit: GB archive_days_threshold: 30 # files older than x days will be archived, unit: day + sql_echo: on # print sql statement in debug log metric_config: is_startup: off # if monitoring start: on, off diff --git a/cpp/coverage.sh b/cpp/coverage.sh index cc509b611b..dd2b9418f5 100755 --- a/cpp/coverage.sh +++ b/cpp/coverage.sh @@ -1,7 +1,5 @@ #!/bin/bash -export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$(pwd)/milvus/lib - LCOV_CMD="lcov" LCOV_GEN_CMD="genhtml" @@ -14,6 +12,26 @@ DIR_LCOV_OUTPUT="lcov_out" DIR_GCNO="cmake_build" DIR_UNITTEST="milvus/bin" +MYSQL_USER_NAME=root +MYSQL_PASSWORD=Fantast1c +MYSQL_HOST='192.168.1.194' +MYSQL_PORT='3306' + +MYSQL_DB_NAME=`date +%s%N` + +function mysql_exc() +{ + cmd=$1 + mysql -h${MYSQL_HOST} -u${MYSQL_USER_NAME} -p${MYSQL_PASSWORD} -e "${cmd}" + if [ $? -ne 0 ]; then + echo "mysql $cmd run failed" + fi +} + +mysql_exc "CREATE DATABASE IF NOT EXISTS ${MYSQL_DB_NAME};" +mysql_exc "GRANT ALL PRIVILEGES ON ${MYSQL_DB_NAME}.* TO '${MYSQL_USER_NAME}'@'%';" +mysql_exc "FLUSH PRIVILEGES;" + # get baseline ${LCOV_CMD} -c -i -d ${DIR_GCNO} -o "${FILE_INFO_BASE}" if [ $? -ne 0 ]; then @@ -22,14 +40,13 @@ if [ $? -ne 0 ]; then fi for test in `ls ${DIR_UNITTEST}`; do + echo $test case ${test} in db_test) # set run args for db_test - args="mysql://root:Fantast1c@192.168.1.194:3306/test" - ;; + args="mysql://${MYSQL_USER_NAME}:${MYSQL_PASSWORD}@${MYSQL_HOST}:${MYSQL_PORT}/${MYSQL_DB_NAME}" *_test) args="" - ;; esac # run unittest ./${DIR_UNITTEST}/${test} "${args}" @@ -38,6 +55,8 @@ for test in `ls ${DIR_UNITTEST}`; do fi done +mysql_exc "DROP DATABASE IF EXISTS ${MYSQL_DB_NAME};" + # gen test converage ${LCOV_CMD} -d ${DIR_GCNO} -o "${FILE_INFO_MILVUS}" -c # merge coverage @@ -50,4 +69,4 @@ ${LCOV_CMD} -r "${FILE_INFO_OUTPUT}" -o "${FILE_INFO_OUTPUT_NEW}" \ "*/cmake_build/*_ep-prefix/*" \ # gen html report -${LCOV_GEN_CMD} "${FILE_INFO_OUTPUT_NEW}" --output-directory ${DIR_LCOV_OUTPUT}/ +${LCOV_GEN_CMD} "${FILE_INFO_OUTPUT_NEW}" --output-directory ${DIR_LCOV_OUTPUT}/ \ No newline at end of file diff --git a/cpp/src/db/MySQLConnectionPool.h b/cpp/src/db/MySQLConnectionPool.h index 8992ba274c..6a763a9729 100644 --- a/cpp/src/db/MySQLConnectionPool.h +++ b/cpp/src/db/MySQLConnectionPool.h @@ -69,6 +69,10 @@ public: max_idle_time_ = max_idle; } + std::string getDB() { + return db_; + } + protected: // Superclass overrides diff --git a/cpp/src/db/MySQLMetaImpl.cpp b/cpp/src/db/MySQLMetaImpl.cpp index 0b0cc01e5d..cb70168075 100644 --- a/cpp/src/db/MySQLMetaImpl.cpp +++ b/cpp/src/db/MySQLMetaImpl.cpp @@ -1733,17 +1733,30 @@ namespace meta { // ENGINE_LOG_WARNING << "MySQLMetaImpl::CleanUp: connection in use = " << mysql_connection_pool_->getConnectionsInUse(); // } - ENGINE_LOG_DEBUG << "Remove table file type as NEW"; Query cleanUpQuery = connectionPtr->query(); - cleanUpQuery << "DELETE FROM TableFiles WHERE file_type = " << std::to_string(TableFileSchema::NEW) << ";"; + cleanUpQuery << "SELECT table_name " << + "FROM information_schema.tables " << + "WHERE table_schema = " << quote << mysql_connection_pool_->getDB() << quote << " " << + "AND table_name = " << quote << "TableFiles" << quote << ";"; if (options_.sql_echo) { ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUp: " << cleanUpQuery.str(); } - if (!cleanUpQuery.exec()) { - ENGINE_LOG_ERROR << "QUERY ERROR WHEN CLEANING UP FILES"; - return Status::DBTransactionError("Clean up Error", cleanUpQuery.error()); + StoreQueryResult res = cleanUpQuery.store(); + assert(res); + if (!res.empty()) { + ENGINE_LOG_DEBUG << "Remove table file type as NEW"; + cleanUpQuery << "DELETE FROM TableFiles WHERE file_type = " << std::to_string(TableFileSchema::NEW) << ";"; + + if (options_.sql_echo) { + ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUp: " << cleanUpQuery.str(); + } + + if (!cleanUpQuery.exec()) { + ENGINE_LOG_ERROR << "QUERY ERROR WHEN CLEANING UP FILES"; + return Status::DBTransactionError("Clean up Error", cleanUpQuery.error()); + } } } catch (const BadQuery& er) { diff --git a/cpp/unittest/db/mysql_meta_test.cpp b/cpp/unittest/db/mysql_meta_test.cpp index 436086acb3..76d7846362 100644 --- a/cpp/unittest/db/mysql_meta_test.cpp +++ b/cpp/unittest/db/mysql_meta_test.cpp @@ -21,184 +21,7 @@ using namespace zilliz::milvus::engine; -//TEST_F(MySQLTest, InitializeTest) { -// DBMetaOptions options; -// //dialect+driver://username:password@host:port/database -// options.backend_uri = "mysql://root:1234@:/test"; -// meta::MySQLMetaImpl impl(options); -// auto status = impl.Initialize(); -// std::cout << status.ToString() << std::endl; -// ASSERT_TRUE(status.ok()); -//} - -TEST_F(MySQLTest, core) { - DBMetaOptions options; -// //dialect+driver://username:password@host:port/database -// options.backend_uri = "mysql://root:1234@:/test"; -// options.path = "/tmp/vecwise_test"; - try { - options = getDBMetaOptions(); - } catch(std::exception& ex) { - ASSERT_TRUE(false); - return; - } - - int mode = Options::MODE::SINGLE; - meta::MySQLMetaImpl impl(options, mode); -// auto status = impl.Initialize(); -// ASSERT_TRUE(status.ok()); - - meta::TableSchema schema1; - schema1.table_id_ = "test1"; - schema1.dimension_ = 123; - - auto status = impl.CreateTable(schema1); -// std::cout << status.ToString() << std::endl; - ASSERT_TRUE(status.ok()); - - meta::TableSchema schema2; - schema2.table_id_ = "test2"; - schema2.dimension_ = 321; - status = impl.CreateTable(schema2); -// std::cout << status.ToString() << std::endl; - ASSERT_TRUE(status.ok()); - - status = impl.CreateTable(schema2); -// std::cout << status.ToString() << std::endl; -// ASSERT_THROW(impl.CreateTable(schema), mysqlpp::BadQuery); - ASSERT_TRUE(status.ok()); - - status = impl.DeleteTable(schema2.table_id_); -// std::cout << status.ToString() << std::endl; - ASSERT_TRUE(status.ok()); - - size_t id1 = schema1.id_; - long created_on1 = schema1.created_on_; - status = impl.DescribeTable(schema1); - ASSERT_TRUE(status.ok()); - ASSERT_EQ(schema1.id_, id1); - ASSERT_EQ(schema1.table_id_, "test1"); - ASSERT_EQ(schema1.created_on_, created_on1); - ASSERT_EQ(schema1.files_cnt_, 0); - ASSERT_EQ(schema1.engine_type_, 1); - ASSERT_EQ(schema1.store_raw_data_, false); - - bool check; - status = impl.HasTable("test1", check); - ASSERT_TRUE(status.ok()); - ASSERT_EQ(check, true); - - std::vector table_schema_array; - status = impl.AllTables(table_schema_array); - ASSERT_TRUE(status.ok()); - ASSERT_EQ(table_schema_array.size(), 1); - meta::TableSchema resultSchema = table_schema_array[0]; - ASSERT_EQ(resultSchema.id_, id1); - ASSERT_EQ(resultSchema.table_id_, "test1"); - ASSERT_EQ(resultSchema.dimension_, 123); - ASSERT_EQ(resultSchema.files_cnt_, 0); - ASSERT_EQ(resultSchema.engine_type_, 1); - ASSERT_EQ(resultSchema.store_raw_data_, false); - - meta::TableFileSchema tableFileSchema; - tableFileSchema.table_id_ = "test1"; - - status = impl.CreateTableFile(tableFileSchema); -// std::cout << status.ToString() << std::endl; - ASSERT_TRUE(status.ok()); - - tableFileSchema.file_type_ = meta::TableFileSchema::TO_INDEX; - status = impl.UpdateTableFile(tableFileSchema); -// std::cout << status.ToString() << std::endl; - ASSERT_TRUE(status.ok()); - - meta::TableFilesSchema filesToIndex; - status = impl.FilesToIndex(filesToIndex); - ASSERT_TRUE(status.ok()); - ASSERT_EQ(filesToIndex.size(), 1); - meta::TableFileSchema fileToIndex = filesToIndex[0]; - ASSERT_EQ(fileToIndex.table_id_, "test1"); - ASSERT_EQ(fileToIndex.dimension_, 123); - -// meta::TableFilesSchema filesToIndex; -// status = impl.FilesToIndex(filesToIndex); -// ASSERT_TRUE(status.ok()); -// ASSERT_EQ(filesToIndex.size(), 0); - - meta::DatesT partition; - partition.push_back(tableFileSchema.date_); - meta::DatePartionedTableFilesSchema filesToSearch; - status = impl.FilesToSearch(tableFileSchema.table_id_, partition, filesToSearch); - ASSERT_TRUE(status.ok()); - ASSERT_EQ(filesToSearch.size(), 1); - ASSERT_EQ(filesToSearch[tableFileSchema.date_].size(), 1); - meta::TableFileSchema fileToSearch = filesToSearch[tableFileSchema.date_][0]; - ASSERT_EQ(fileToSearch.table_id_, "test1"); - ASSERT_EQ(fileToSearch.dimension_, 123); - - tableFileSchema.file_type_ = meta::TableFileSchema::RAW; - status = impl.UpdateTableFile(tableFileSchema); - ASSERT_TRUE(status.ok()); - - meta::DatePartionedTableFilesSchema filesToMerge; - status = impl.FilesToMerge(tableFileSchema.table_id_, filesToMerge); -// std::cout << status.ToString() << std::endl; - ASSERT_TRUE(status.ok()); - ASSERT_EQ(filesToMerge.size(), 1); - ASSERT_EQ(filesToMerge[tableFileSchema.date_].size(), 1); - meta::TableFileSchema fileToMerge = filesToMerge[tableFileSchema.date_][0]; - ASSERT_EQ(fileToMerge.table_id_, "test1"); - ASSERT_EQ(fileToMerge.dimension_, 123); - - meta::TableFilesSchema resultTableFilesSchema; - std::vector ids; - ids.push_back(tableFileSchema.id_); - status = impl.GetTableFiles(tableFileSchema.table_id_, ids, resultTableFilesSchema); - ASSERT_TRUE(status.ok()); - ASSERT_EQ(resultTableFilesSchema.size(), 1); - meta::TableFileSchema resultTableFileSchema = resultTableFilesSchema[0]; -// ASSERT_EQ(resultTableFileSchema.id_, tableFileSchema.id_); - ASSERT_EQ(resultTableFileSchema.table_id_, tableFileSchema.table_id_); - ASSERT_EQ(resultTableFileSchema.file_id_, tableFileSchema.file_id_); - ASSERT_EQ(resultTableFileSchema.file_type_, tableFileSchema.file_type_); - ASSERT_EQ(resultTableFileSchema.size_, tableFileSchema.size_); - ASSERT_EQ(resultTableFileSchema.date_, tableFileSchema.date_); - ASSERT_EQ(resultTableFileSchema.engine_type_, tableFileSchema.engine_type_); - ASSERT_EQ(resultTableFileSchema.dimension_, tableFileSchema.dimension_); - - tableFileSchema.size_ = 234; - meta::TableSchema schema3; - schema3.table_id_ = "test3"; - schema3.dimension_ = 321; - status = impl.CreateTable(schema3); - ASSERT_TRUE(status.ok()); - meta::TableFileSchema tableFileSchema2; - tableFileSchema2.table_id_ = "test3"; - tableFileSchema2.size_ = 345; - status = impl.CreateTableFile(tableFileSchema2); - ASSERT_TRUE(status.ok()); - meta::TableFilesSchema filesToUpdate; - filesToUpdate.emplace_back(tableFileSchema); - filesToUpdate.emplace_back(tableFileSchema2); - status = impl.UpdateTableFile(tableFileSchema); - ASSERT_TRUE(status.ok()); - - uint64_t resultSize; - status = impl.Size(resultSize); -// std::cout << status.ToString() << std::endl; - ASSERT_TRUE(status.ok()); - ASSERT_EQ(resultSize, tableFileSchema.size_ + tableFileSchema2.size_); - - uint64_t countResult; - status = impl.Count(tableFileSchema.table_id_, countResult); - ASSERT_TRUE(status.ok()); - - status = impl.DropAll(); - ASSERT_TRUE(status.ok()); - -} - -TEST_F(MySQLTest, GROUP_TEST) { +TEST_F(MySQLTest, TABLE_TEST) { DBMetaOptions options; try { options = getDBMetaOptions(); @@ -210,38 +33,37 @@ TEST_F(MySQLTest, GROUP_TEST) { int mode = Options::MODE::SINGLE; meta::MySQLMetaImpl impl(options, mode); - auto table_id = "meta_test_group"; + auto table_id = "meta_test_table"; - meta::TableSchema group; - group.table_id_ = table_id; - auto status = impl.CreateTable(group); + meta::TableSchema table; + table.table_id_ = table_id; + auto status = impl.CreateTable(table); ASSERT_TRUE(status.ok()); - auto gid = group.id_; - group.id_ = -1; - status = impl.DescribeTable(group); + auto gid = table.id_; + table.id_ = -1; + status = impl.DescribeTable(table); ASSERT_TRUE(status.ok()); - ASSERT_EQ(group.id_, gid); - ASSERT_EQ(group.table_id_, table_id); + ASSERT_EQ(table.id_, gid); + ASSERT_EQ(table.table_id_, table_id); - group.table_id_ = "not_found"; - status = impl.DescribeTable(group); + table.table_id_ = "not_found"; + status = impl.DescribeTable(table); ASSERT_TRUE(!status.ok()); - group.table_id_ = table_id; - status = impl.CreateTable(group); + table.table_id_ = table_id; + status = impl.CreateTable(table); ASSERT_TRUE(status.ok()); - group.table_id_ = ""; - status = impl.CreateTable(group); + table.table_id_ = ""; + status = impl.CreateTable(table); ASSERT_TRUE(status.ok()); - status = impl.DropAll(); ASSERT_TRUE(status.ok()); } -TEST_F(MySQLTest, table_file_TEST) { +TEST_F(MySQLTest, TABLE_FILE_TEST) { DBMetaOptions options; try { options = getDBMetaOptions(); @@ -253,17 +75,16 @@ TEST_F(MySQLTest, table_file_TEST) { int mode = Options::MODE::SINGLE; meta::MySQLMetaImpl impl(options, mode); - auto table_id = "meta_test_group"; + auto table_id = "meta_test_table"; - meta::TableSchema group; - group.table_id_ = table_id; - group.dimension_ = 256; - auto status = impl.CreateTable(group); + meta::TableSchema table; + table.table_id_ = table_id; + table.dimension_ = 256; + auto status = impl.CreateTable(table); meta::TableFileSchema table_file; - table_file.table_id_ = group.table_id_; + table_file.table_id_ = table.table_id_; status = impl.CreateTableFile(table_file); -// std::cout << status.ToString() << std::endl; ASSERT_TRUE(status.ok()); ASSERT_EQ(table_file.file_type_, meta::TableFileSchema::NEW); @@ -332,15 +153,15 @@ TEST_F(MySQLTest, ARCHIVE_TEST_DAYS) { int mode = Options::MODE::SINGLE; meta::MySQLMetaImpl impl(options, mode); - auto table_id = "meta_test_group"; + auto table_id = "meta_test_table"; - meta::TableSchema group; - group.table_id_ = table_id; - auto status = impl.CreateTable(group); + meta::TableSchema table; + table.table_id_ = table_id; + auto status = impl.CreateTable(table); meta::TableFilesSchema files; meta::TableFileSchema table_file; - table_file.table_id_ = group.table_id_; + table_file.table_id_ = table.table_id_; auto cnt = 100; long ts = utils::GetMicroSecTimeStamp(); @@ -391,13 +212,13 @@ TEST_F(MySQLTest, ARCHIVE_TEST_DISK) { auto impl = meta::MySQLMetaImpl(options, mode); auto table_id = "meta_test_group"; - meta::TableSchema group; - group.table_id_ = table_id; - auto status = impl.CreateTable(group); + meta::TableSchema table; + table.table_id_ = table_id; + auto status = impl.CreateTable(table); meta::TableFilesSchema files; meta::TableFileSchema table_file; - table_file.table_id_ = group.table_id_; + table_file.table_id_ = table.table_id_; auto cnt = 10; auto each_size = 2UL; @@ -445,9 +266,9 @@ TEST_F(MySQLTest, TABLE_FILES_TEST) { auto table_id = "meta_test_group"; - meta::TableSchema group; - group.table_id_ = table_id; - auto status = impl.CreateTable(group); + meta::TableSchema table; + table.table_id_ = table_id; + auto status = impl.CreateTable(table); int new_files_cnt = 4; int raw_files_cnt = 5; @@ -455,7 +276,7 @@ TEST_F(MySQLTest, TABLE_FILES_TEST) { int index_files_cnt = 7; meta::TableFileSchema table_file; - table_file.table_id_ = group.table_id_; + table_file.table_id_ = table.table_id_; for (auto i=0; i Date: Mon, 1 Jul 2019 15:38:35 +0800 Subject: [PATCH 43/43] Revert "Merge branch 'mysql-0.3.0' into 'branch-0.3.0'" This reverts merge request !143 Former-commit-id: 81a89490f668ae9342eaf1ecc48d1255879e44a6 --- cpp/conf/server_config.template | 1 - cpp/coverage.sh | 31 +--- cpp/src/db/MySQLConnectionPool.h | 4 - cpp/src/db/MySQLMetaImpl.cpp | 23 +-- cpp/unittest/db/mysql_meta_test.cpp | 253 ++++++++++++++++++++++++---- 5 files changed, 227 insertions(+), 85 deletions(-) diff --git a/cpp/conf/server_config.template b/cpp/conf/server_config.template index b5ba2134b9..c2ed775601 100644 --- a/cpp/conf/server_config.template +++ b/cpp/conf/server_config.template @@ -15,7 +15,6 @@ db_config: index_building_threshold: 1024 # index building trigger threshold, default: 1024, unit: MB archive_disk_threshold: 512 # triger archive action if storage size exceed this value, unit: GB archive_days_threshold: 30 # files older than x days will be archived, unit: day - sql_echo: on # print sql statement in debug log metric_config: is_startup: off # if monitoring start: on, off diff --git a/cpp/coverage.sh b/cpp/coverage.sh index dd2b9418f5..cc509b611b 100755 --- a/cpp/coverage.sh +++ b/cpp/coverage.sh @@ -1,5 +1,7 @@ #!/bin/bash +export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$(pwd)/milvus/lib + LCOV_CMD="lcov" LCOV_GEN_CMD="genhtml" @@ -12,26 +14,6 @@ DIR_LCOV_OUTPUT="lcov_out" DIR_GCNO="cmake_build" DIR_UNITTEST="milvus/bin" -MYSQL_USER_NAME=root -MYSQL_PASSWORD=Fantast1c -MYSQL_HOST='192.168.1.194' -MYSQL_PORT='3306' - -MYSQL_DB_NAME=`date +%s%N` - -function mysql_exc() -{ - cmd=$1 - mysql -h${MYSQL_HOST} -u${MYSQL_USER_NAME} -p${MYSQL_PASSWORD} -e "${cmd}" - if [ $? -ne 0 ]; then - echo "mysql $cmd run failed" - fi -} - -mysql_exc "CREATE DATABASE IF NOT EXISTS ${MYSQL_DB_NAME};" -mysql_exc "GRANT ALL PRIVILEGES ON ${MYSQL_DB_NAME}.* TO '${MYSQL_USER_NAME}'@'%';" -mysql_exc "FLUSH PRIVILEGES;" - # get baseline ${LCOV_CMD} -c -i -d ${DIR_GCNO} -o "${FILE_INFO_BASE}" if [ $? -ne 0 ]; then @@ -40,13 +22,14 @@ if [ $? -ne 0 ]; then fi for test in `ls ${DIR_UNITTEST}`; do - echo $test case ${test} in db_test) # set run args for db_test - args="mysql://${MYSQL_USER_NAME}:${MYSQL_PASSWORD}@${MYSQL_HOST}:${MYSQL_PORT}/${MYSQL_DB_NAME}" + args="mysql://root:Fantast1c@192.168.1.194:3306/test" + ;; *_test) args="" + ;; esac # run unittest ./${DIR_UNITTEST}/${test} "${args}" @@ -55,8 +38,6 @@ for test in `ls ${DIR_UNITTEST}`; do fi done -mysql_exc "DROP DATABASE IF EXISTS ${MYSQL_DB_NAME};" - # gen test converage ${LCOV_CMD} -d ${DIR_GCNO} -o "${FILE_INFO_MILVUS}" -c # merge coverage @@ -69,4 +50,4 @@ ${LCOV_CMD} -r "${FILE_INFO_OUTPUT}" -o "${FILE_INFO_OUTPUT_NEW}" \ "*/cmake_build/*_ep-prefix/*" \ # gen html report -${LCOV_GEN_CMD} "${FILE_INFO_OUTPUT_NEW}" --output-directory ${DIR_LCOV_OUTPUT}/ \ No newline at end of file +${LCOV_GEN_CMD} "${FILE_INFO_OUTPUT_NEW}" --output-directory ${DIR_LCOV_OUTPUT}/ diff --git a/cpp/src/db/MySQLConnectionPool.h b/cpp/src/db/MySQLConnectionPool.h index 6a763a9729..8992ba274c 100644 --- a/cpp/src/db/MySQLConnectionPool.h +++ b/cpp/src/db/MySQLConnectionPool.h @@ -69,10 +69,6 @@ public: max_idle_time_ = max_idle; } - std::string getDB() { - return db_; - } - protected: // Superclass overrides diff --git a/cpp/src/db/MySQLMetaImpl.cpp b/cpp/src/db/MySQLMetaImpl.cpp index cb70168075..0b0cc01e5d 100644 --- a/cpp/src/db/MySQLMetaImpl.cpp +++ b/cpp/src/db/MySQLMetaImpl.cpp @@ -1733,30 +1733,17 @@ namespace meta { // ENGINE_LOG_WARNING << "MySQLMetaImpl::CleanUp: connection in use = " << mysql_connection_pool_->getConnectionsInUse(); // } + ENGINE_LOG_DEBUG << "Remove table file type as NEW"; Query cleanUpQuery = connectionPtr->query(); - cleanUpQuery << "SELECT table_name " << - "FROM information_schema.tables " << - "WHERE table_schema = " << quote << mysql_connection_pool_->getDB() << quote << " " << - "AND table_name = " << quote << "TableFiles" << quote << ";"; + cleanUpQuery << "DELETE FROM TableFiles WHERE file_type = " << std::to_string(TableFileSchema::NEW) << ";"; if (options_.sql_echo) { ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUp: " << cleanUpQuery.str(); } - StoreQueryResult res = cleanUpQuery.store(); - assert(res); - if (!res.empty()) { - ENGINE_LOG_DEBUG << "Remove table file type as NEW"; - cleanUpQuery << "DELETE FROM TableFiles WHERE file_type = " << std::to_string(TableFileSchema::NEW) << ";"; - - if (options_.sql_echo) { - ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUp: " << cleanUpQuery.str(); - } - - if (!cleanUpQuery.exec()) { - ENGINE_LOG_ERROR << "QUERY ERROR WHEN CLEANING UP FILES"; - return Status::DBTransactionError("Clean up Error", cleanUpQuery.error()); - } + if (!cleanUpQuery.exec()) { + ENGINE_LOG_ERROR << "QUERY ERROR WHEN CLEANING UP FILES"; + return Status::DBTransactionError("Clean up Error", cleanUpQuery.error()); } } catch (const BadQuery& er) { diff --git a/cpp/unittest/db/mysql_meta_test.cpp b/cpp/unittest/db/mysql_meta_test.cpp index 76d7846362..436086acb3 100644 --- a/cpp/unittest/db/mysql_meta_test.cpp +++ b/cpp/unittest/db/mysql_meta_test.cpp @@ -21,7 +21,184 @@ using namespace zilliz::milvus::engine; -TEST_F(MySQLTest, TABLE_TEST) { +//TEST_F(MySQLTest, InitializeTest) { +// DBMetaOptions options; +// //dialect+driver://username:password@host:port/database +// options.backend_uri = "mysql://root:1234@:/test"; +// meta::MySQLMetaImpl impl(options); +// auto status = impl.Initialize(); +// std::cout << status.ToString() << std::endl; +// ASSERT_TRUE(status.ok()); +//} + +TEST_F(MySQLTest, core) { + DBMetaOptions options; +// //dialect+driver://username:password@host:port/database +// options.backend_uri = "mysql://root:1234@:/test"; +// options.path = "/tmp/vecwise_test"; + try { + options = getDBMetaOptions(); + } catch(std::exception& ex) { + ASSERT_TRUE(false); + return; + } + + int mode = Options::MODE::SINGLE; + meta::MySQLMetaImpl impl(options, mode); +// auto status = impl.Initialize(); +// ASSERT_TRUE(status.ok()); + + meta::TableSchema schema1; + schema1.table_id_ = "test1"; + schema1.dimension_ = 123; + + auto status = impl.CreateTable(schema1); +// std::cout << status.ToString() << std::endl; + ASSERT_TRUE(status.ok()); + + meta::TableSchema schema2; + schema2.table_id_ = "test2"; + schema2.dimension_ = 321; + status = impl.CreateTable(schema2); +// std::cout << status.ToString() << std::endl; + ASSERT_TRUE(status.ok()); + + status = impl.CreateTable(schema2); +// std::cout << status.ToString() << std::endl; +// ASSERT_THROW(impl.CreateTable(schema), mysqlpp::BadQuery); + ASSERT_TRUE(status.ok()); + + status = impl.DeleteTable(schema2.table_id_); +// std::cout << status.ToString() << std::endl; + ASSERT_TRUE(status.ok()); + + size_t id1 = schema1.id_; + long created_on1 = schema1.created_on_; + status = impl.DescribeTable(schema1); + ASSERT_TRUE(status.ok()); + ASSERT_EQ(schema1.id_, id1); + ASSERT_EQ(schema1.table_id_, "test1"); + ASSERT_EQ(schema1.created_on_, created_on1); + ASSERT_EQ(schema1.files_cnt_, 0); + ASSERT_EQ(schema1.engine_type_, 1); + ASSERT_EQ(schema1.store_raw_data_, false); + + bool check; + status = impl.HasTable("test1", check); + ASSERT_TRUE(status.ok()); + ASSERT_EQ(check, true); + + std::vector table_schema_array; + status = impl.AllTables(table_schema_array); + ASSERT_TRUE(status.ok()); + ASSERT_EQ(table_schema_array.size(), 1); + meta::TableSchema resultSchema = table_schema_array[0]; + ASSERT_EQ(resultSchema.id_, id1); + ASSERT_EQ(resultSchema.table_id_, "test1"); + ASSERT_EQ(resultSchema.dimension_, 123); + ASSERT_EQ(resultSchema.files_cnt_, 0); + ASSERT_EQ(resultSchema.engine_type_, 1); + ASSERT_EQ(resultSchema.store_raw_data_, false); + + meta::TableFileSchema tableFileSchema; + tableFileSchema.table_id_ = "test1"; + + status = impl.CreateTableFile(tableFileSchema); +// std::cout << status.ToString() << std::endl; + ASSERT_TRUE(status.ok()); + + tableFileSchema.file_type_ = meta::TableFileSchema::TO_INDEX; + status = impl.UpdateTableFile(tableFileSchema); +// std::cout << status.ToString() << std::endl; + ASSERT_TRUE(status.ok()); + + meta::TableFilesSchema filesToIndex; + status = impl.FilesToIndex(filesToIndex); + ASSERT_TRUE(status.ok()); + ASSERT_EQ(filesToIndex.size(), 1); + meta::TableFileSchema fileToIndex = filesToIndex[0]; + ASSERT_EQ(fileToIndex.table_id_, "test1"); + ASSERT_EQ(fileToIndex.dimension_, 123); + +// meta::TableFilesSchema filesToIndex; +// status = impl.FilesToIndex(filesToIndex); +// ASSERT_TRUE(status.ok()); +// ASSERT_EQ(filesToIndex.size(), 0); + + meta::DatesT partition; + partition.push_back(tableFileSchema.date_); + meta::DatePartionedTableFilesSchema filesToSearch; + status = impl.FilesToSearch(tableFileSchema.table_id_, partition, filesToSearch); + ASSERT_TRUE(status.ok()); + ASSERT_EQ(filesToSearch.size(), 1); + ASSERT_EQ(filesToSearch[tableFileSchema.date_].size(), 1); + meta::TableFileSchema fileToSearch = filesToSearch[tableFileSchema.date_][0]; + ASSERT_EQ(fileToSearch.table_id_, "test1"); + ASSERT_EQ(fileToSearch.dimension_, 123); + + tableFileSchema.file_type_ = meta::TableFileSchema::RAW; + status = impl.UpdateTableFile(tableFileSchema); + ASSERT_TRUE(status.ok()); + + meta::DatePartionedTableFilesSchema filesToMerge; + status = impl.FilesToMerge(tableFileSchema.table_id_, filesToMerge); +// std::cout << status.ToString() << std::endl; + ASSERT_TRUE(status.ok()); + ASSERT_EQ(filesToMerge.size(), 1); + ASSERT_EQ(filesToMerge[tableFileSchema.date_].size(), 1); + meta::TableFileSchema fileToMerge = filesToMerge[tableFileSchema.date_][0]; + ASSERT_EQ(fileToMerge.table_id_, "test1"); + ASSERT_EQ(fileToMerge.dimension_, 123); + + meta::TableFilesSchema resultTableFilesSchema; + std::vector ids; + ids.push_back(tableFileSchema.id_); + status = impl.GetTableFiles(tableFileSchema.table_id_, ids, resultTableFilesSchema); + ASSERT_TRUE(status.ok()); + ASSERT_EQ(resultTableFilesSchema.size(), 1); + meta::TableFileSchema resultTableFileSchema = resultTableFilesSchema[0]; +// ASSERT_EQ(resultTableFileSchema.id_, tableFileSchema.id_); + ASSERT_EQ(resultTableFileSchema.table_id_, tableFileSchema.table_id_); + ASSERT_EQ(resultTableFileSchema.file_id_, tableFileSchema.file_id_); + ASSERT_EQ(resultTableFileSchema.file_type_, tableFileSchema.file_type_); + ASSERT_EQ(resultTableFileSchema.size_, tableFileSchema.size_); + ASSERT_EQ(resultTableFileSchema.date_, tableFileSchema.date_); + ASSERT_EQ(resultTableFileSchema.engine_type_, tableFileSchema.engine_type_); + ASSERT_EQ(resultTableFileSchema.dimension_, tableFileSchema.dimension_); + + tableFileSchema.size_ = 234; + meta::TableSchema schema3; + schema3.table_id_ = "test3"; + schema3.dimension_ = 321; + status = impl.CreateTable(schema3); + ASSERT_TRUE(status.ok()); + meta::TableFileSchema tableFileSchema2; + tableFileSchema2.table_id_ = "test3"; + tableFileSchema2.size_ = 345; + status = impl.CreateTableFile(tableFileSchema2); + ASSERT_TRUE(status.ok()); + meta::TableFilesSchema filesToUpdate; + filesToUpdate.emplace_back(tableFileSchema); + filesToUpdate.emplace_back(tableFileSchema2); + status = impl.UpdateTableFile(tableFileSchema); + ASSERT_TRUE(status.ok()); + + uint64_t resultSize; + status = impl.Size(resultSize); +// std::cout << status.ToString() << std::endl; + ASSERT_TRUE(status.ok()); + ASSERT_EQ(resultSize, tableFileSchema.size_ + tableFileSchema2.size_); + + uint64_t countResult; + status = impl.Count(tableFileSchema.table_id_, countResult); + ASSERT_TRUE(status.ok()); + + status = impl.DropAll(); + ASSERT_TRUE(status.ok()); + +} + +TEST_F(MySQLTest, GROUP_TEST) { DBMetaOptions options; try { options = getDBMetaOptions(); @@ -33,37 +210,38 @@ TEST_F(MySQLTest, TABLE_TEST) { int mode = Options::MODE::SINGLE; meta::MySQLMetaImpl impl(options, mode); - auto table_id = "meta_test_table"; + auto table_id = "meta_test_group"; - meta::TableSchema table; - table.table_id_ = table_id; - auto status = impl.CreateTable(table); + meta::TableSchema group; + group.table_id_ = table_id; + auto status = impl.CreateTable(group); ASSERT_TRUE(status.ok()); - auto gid = table.id_; - table.id_ = -1; - status = impl.DescribeTable(table); + auto gid = group.id_; + group.id_ = -1; + status = impl.DescribeTable(group); ASSERT_TRUE(status.ok()); - ASSERT_EQ(table.id_, gid); - ASSERT_EQ(table.table_id_, table_id); + ASSERT_EQ(group.id_, gid); + ASSERT_EQ(group.table_id_, table_id); - table.table_id_ = "not_found"; - status = impl.DescribeTable(table); + group.table_id_ = "not_found"; + status = impl.DescribeTable(group); ASSERT_TRUE(!status.ok()); - table.table_id_ = table_id; - status = impl.CreateTable(table); + group.table_id_ = table_id; + status = impl.CreateTable(group); ASSERT_TRUE(status.ok()); - table.table_id_ = ""; - status = impl.CreateTable(table); + group.table_id_ = ""; + status = impl.CreateTable(group); ASSERT_TRUE(status.ok()); + status = impl.DropAll(); ASSERT_TRUE(status.ok()); } -TEST_F(MySQLTest, TABLE_FILE_TEST) { +TEST_F(MySQLTest, table_file_TEST) { DBMetaOptions options; try { options = getDBMetaOptions(); @@ -75,16 +253,17 @@ TEST_F(MySQLTest, TABLE_FILE_TEST) { int mode = Options::MODE::SINGLE; meta::MySQLMetaImpl impl(options, mode); - auto table_id = "meta_test_table"; + auto table_id = "meta_test_group"; - meta::TableSchema table; - table.table_id_ = table_id; - table.dimension_ = 256; - auto status = impl.CreateTable(table); + meta::TableSchema group; + group.table_id_ = table_id; + group.dimension_ = 256; + auto status = impl.CreateTable(group); meta::TableFileSchema table_file; - table_file.table_id_ = table.table_id_; + table_file.table_id_ = group.table_id_; status = impl.CreateTableFile(table_file); +// std::cout << status.ToString() << std::endl; ASSERT_TRUE(status.ok()); ASSERT_EQ(table_file.file_type_, meta::TableFileSchema::NEW); @@ -153,15 +332,15 @@ TEST_F(MySQLTest, ARCHIVE_TEST_DAYS) { int mode = Options::MODE::SINGLE; meta::MySQLMetaImpl impl(options, mode); - auto table_id = "meta_test_table"; + auto table_id = "meta_test_group"; - meta::TableSchema table; - table.table_id_ = table_id; - auto status = impl.CreateTable(table); + meta::TableSchema group; + group.table_id_ = table_id; + auto status = impl.CreateTable(group); meta::TableFilesSchema files; meta::TableFileSchema table_file; - table_file.table_id_ = table.table_id_; + table_file.table_id_ = group.table_id_; auto cnt = 100; long ts = utils::GetMicroSecTimeStamp(); @@ -212,13 +391,13 @@ TEST_F(MySQLTest, ARCHIVE_TEST_DISK) { auto impl = meta::MySQLMetaImpl(options, mode); auto table_id = "meta_test_group"; - meta::TableSchema table; - table.table_id_ = table_id; - auto status = impl.CreateTable(table); + meta::TableSchema group; + group.table_id_ = table_id; + auto status = impl.CreateTable(group); meta::TableFilesSchema files; meta::TableFileSchema table_file; - table_file.table_id_ = table.table_id_; + table_file.table_id_ = group.table_id_; auto cnt = 10; auto each_size = 2UL; @@ -266,9 +445,9 @@ TEST_F(MySQLTest, TABLE_FILES_TEST) { auto table_id = "meta_test_group"; - meta::TableSchema table; - table.table_id_ = table_id; - auto status = impl.CreateTable(table); + meta::TableSchema group; + group.table_id_ = table_id; + auto status = impl.CreateTable(group); int new_files_cnt = 4; int raw_files_cnt = 5; @@ -276,7 +455,7 @@ TEST_F(MySQLTest, TABLE_FILES_TEST) { int index_files_cnt = 7; meta::TableFileSchema table_file; - table_file.table_id_ = table.table_id_; + table_file.table_id_ = group.table_id_; for (auto i=0; i