diff --git a/cpp/src/db/MySQLMetaImpl.h b/cpp/src/db/MySQLMetaImpl.h new file mode 100644 index 0000000000..e099fbb262 --- /dev/null +++ b/cpp/src/db/MySQLMetaImpl.h @@ -0,0 +1,80 @@ +/******************************************************************************* + * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved + * Unauthorized copying of this file, via any medium is strictly prohibited. + * Proprietary and confidential. + ******************************************************************************/ +#pragma once + +#include "Meta.h" +#include "Options.h" + +namespace zilliz { +namespace milvus { +namespace engine { +namespace meta { + +// auto StoragePrototype(const std::string& path); + + class MySQLMetaImpl : public Meta { + public: + MySQLMetaImpl(const DBMetaOptions& options_); + + virtual Status CreateTable(TableSchema& table_schema) override; + virtual Status DeleteTable(const std::string& table_id) override; + virtual Status DescribeTable(TableSchema& group_info_) override; + virtual Status HasTable(const std::string& table_id, bool& has_or_not) override; + virtual Status AllTables(std::vector& table_schema_array) override; + + virtual Status CreateTableFile(TableFileSchema& file_schema) override; + virtual Status DropPartitionsByDates(const std::string& table_id, + const DatesT& dates) override; + + virtual Status GetTableFile(TableFileSchema& file_schema) override; + + virtual Status UpdateTableFile(TableFileSchema& file_schema) override; + + virtual Status UpdateTableFiles(TableFilesSchema& files) override; + + virtual Status FilesToSearch(const std::string& table_id, + const DatesT& partition, + DatePartionedTableFilesSchema& files) override; + + virtual Status FilesToMerge(const std::string& table_id, + DatePartionedTableFilesSchema& files) override; + + virtual Status FilesToDelete(const std::string& table_id, + const DatesT& partition, + DatePartionedTableFilesSchema& files) override; + + virtual Status FilesToIndex(TableFilesSchema&) override; + + virtual Status Archive() override; + + virtual Status Size(uint64_t& result) override; + + virtual Status CleanUp() override; + + virtual Status CleanUpFilesWithTTL(uint16_t seconds) override; + + virtual Status DropAll() override; + + virtual Status Count(const std::string& table_id, uint64_t& result) override; + + virtual ~MySQLMetaImpl(); + + private: + Status NextFileId(std::string& file_id); + Status NextTableId(std::string& table_id); + Status DiscardFiles(long to_discard_size); + std::string GetTablePath(const std::string& table_id); + std::string GetTableDatePartitionPath(const std::string& table_id, DateT& date); + void GetTableFilePath(TableFileSchema& group_file); + Status Initialize(); + + const DBMetaOptions options_; + }; // DBMetaImpl + +} // namespace meta +} // namespace engine +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/db/Options.h b/cpp/src/db/Options.h index ec757de003..6eddd26269 100644 --- a/cpp/src/db/Options.h +++ b/cpp/src/db/Options.h @@ -37,12 +37,6 @@ struct DBMetaOptions { ArchiveConf archive_conf = ArchiveConf("delete"); }; // DBMetaOptions -struct MySQLMetaOptions { - std::string path; - -}; - - struct Options { Options(); uint16_t memory_sync_interval = 1; diff --git a/cpp/unittest/faiss_wrapper/wrapper_test.cpp b/cpp/unittest/faiss_wrapper/wrapper_test.cpp index 67a6c3cde8..c1b4ff2cb0 100644 --- a/cpp/unittest/faiss_wrapper/wrapper_test.cpp +++ b/cpp/unittest/faiss_wrapper/wrapper_test.cpp @@ -9,130 +9,144 @@ #include "wrapper/Operand.h" #include "wrapper/Index.h" #include "wrapper/IndexBuilder.h" - +#include using namespace zilliz::milvus::engine; +TEST(xxx, Wrapper_Test){ + // std::string xxx = "dialect+driver://username:password@host:port/database"; -TEST(operand_test, Wrapper_Test) { - using std::cout; - using std::endl; + //mysql://scott:tiger@localhost/mydatabase - auto opd = std::make_shared(); - opd->index_type = "IVF"; - opd->preproc = "OPQ"; - opd->postproc = "PQ"; - opd->metric_type = "L2"; - opd->d = 64; + std::string littel_xx = "dixx://"; + std::regex xx_regex("([a-zA-Z0-9_-\\.]*):\\/\\/([a-zA-Z0-9_-\\.]*)\\:([a-zA-Z0-9_-\\.]*)\\@([a-zA-Z0-9_-\\.]*)\\:([0-9]*)\\/([a-zA-Z0-9_-\\.]*)"); + std::smatch pieces_match; + std::regex_match(littel_xx, pieces_match, xx_regex); - auto opd_str = operand_to_str(opd); - auto new_opd = str_to_operand(opd_str); - - // TODO: fix all place where using opd to build index. - assert(new_opd->get_index_type(10000) == opd->get_index_type(10000)); + for(auto &x : pieces_match) { + std::cout << "hehhe: " << x.str() << std::endl; + } } -TEST(build_test, Wrapper_Test) { - // dimension of the vectors to index - int d = 3; - - // make a set of nt training vectors in the unit cube - size_t nt = 10000; - - // a reasonable number of cetroids to index nb vectors - int ncentroids = 16; - - std::random_device rd; - std::mt19937 gen(rd()); - - std::vector xb; - std::vector ids; - - //prepare train data - std::uniform_real_distribution<> dis_xt(-1.0, 1.0); - std::vector xt(nt * d); - for (size_t i = 0; i < nt * d; i++) { - xt[i] = dis_xt(gen); - } - - //train the index - auto opd = std::make_shared(); - opd->index_type = "IVF"; - opd->d = d; - opd->ncent = ncentroids; - IndexBuilderPtr index_builder_1 = GetIndexBuilder(opd); - auto index_1 = index_builder_1->build_all(0, xb, ids, nt, xt); - ASSERT_TRUE(index_1 != nullptr); - - // size of the database we plan to index - size_t nb = 100000; - - //prepare raw data - xb.resize(nb); - ids.resize(nb); - for (size_t i = 0; i < nb; i++) { - xb[i] = dis_xt(gen); - ids[i] = i; - } - index_1->add_with_ids(nb, xb.data(), ids.data()); - - //search in first quadrant - int nq = 1, k = 10; - std::vector xq = {0.5, 0.5, 0.5}; - float *result_dists = new float[k]; - long *result_ids = new long[k]; - index_1->search(nq, xq.data(), k, result_dists, result_ids); - - for (int i = 0; i < k; i++) { - if (result_ids[i] < 0) { - ASSERT_TRUE(false); - break; - } - - long id = result_ids[i]; - std::cout << "No." << id << " [" << xb[id * 3] << ", " << xb[id * 3 + 1] << ", " - << xb[id * 3 + 2] << "] distance = " << result_dists[i] << std::endl; - - //makesure result vector is in first quadrant - ASSERT_TRUE(xb[id * 3] > 0.0); - ASSERT_TRUE(xb[id * 3 + 1] > 0.0); - ASSERT_TRUE(xb[id * 3 + 2] > 0.0); - } - - delete[] result_dists; - delete[] result_ids; -} - -TEST(gpu_build_test, Wrapper_Test) { - using std::vector; - - int d = 256; - int nb = 3 * 1000 * 100; - int nq = 100; - vector xb(d * nb); - vector xq(d * nq); - vector ids(nb); - - std::random_device rd; - std::mt19937 gen(rd()); - std::uniform_real_distribution<> dis_xt(-1.0, 1.0); - for (auto &e : xb) { e = float(dis_xt(gen)); } - for (auto &e : xq) { e = float(dis_xt(gen)); } - for (int i = 0; i < nb; ++i) { ids[i] = i; } - - auto opd = std::make_shared(); - opd->index_type = "IVF"; - opd->d = d; - opd->ncent = 256; - - IndexBuilderPtr index_builder_1 = GetIndexBuilder(opd); - auto index_1 = index_builder_1->build_all(nb, xb.data(), ids.data()); - assert(index_1->ntotal == nb); - assert(index_1->dim == d); - - // sanity check: search 5 first vectors of xb - int k = 1; - vector I(5 * k); - vector D(5 * k); - index_1->search(5, xb.data(), k, D.data(), I.data()); - for (int i = 0; i < 5; ++i) { assert(i == I[i]); } -} +//TEST(operand_test, Wrapper_Test) { +// using std::cout; +// using std::endl; +// +// auto opd = std::make_shared(); +// opd->index_type = "IVF"; +// opd->preproc = "OPQ"; +// opd->postproc = "PQ"; +// opd->metric_type = "L2"; +// opd->d = 64; +// +// auto opd_str = operand_to_str(opd); +// auto new_opd = str_to_operand(opd_str); +// +// // TODO: fix all place where using opd to build index. +// assert(new_opd->get_index_type(10000) == opd->get_index_type(10000)); +//} +// +//TEST(build_test, Wrapper_Test) { +// // dimension of the vectors to index +// int d = 3; +// +// // make a set of nt training vectors in the unit cube +// size_t nt = 10000; +// +// // a reasonable number of cetroids to index nb vectors +// int ncentroids = 16; +// +// std::random_device rd; +// std::mt19937 gen(rd()); +// +// std::vector xb; +// std::vector ids; +// +// //prepare train data +// std::uniform_real_distribution<> dis_xt(-1.0, 1.0); +// std::vector xt(nt * d); +// for (size_t i = 0; i < nt * d; i++) { +// xt[i] = dis_xt(gen); +// } +// +// //train the index +// auto opd = std::make_shared(); +// opd->index_type = "IVF"; +// opd->d = d; +// opd->ncent = ncentroids; +// IndexBuilderPtr index_builder_1 = GetIndexBuilder(opd); +// auto index_1 = index_builder_1->build_all(0, xb, ids, nt, xt); +// ASSERT_TRUE(index_1 != nullptr); +// +// // size of the database we plan to index +// size_t nb = 100000; +// +// //prepare raw data +// xb.resize(nb); +// ids.resize(nb); +// for (size_t i = 0; i < nb; i++) { +// xb[i] = dis_xt(gen); +// ids[i] = i; +// } +// index_1->add_with_ids(nb, xb.data(), ids.data()); +// +// //search in first quadrant +// int nq = 1, k = 10; +// std::vector xq = {0.5, 0.5, 0.5}; +// float *result_dists = new float[k]; +// long *result_ids = new long[k]; +// index_1->search(nq, xq.data(), k, result_dists, result_ids); +// +// for (int i = 0; i < k; i++) { +// if (result_ids[i] < 0) { +// ASSERT_TRUE(false); +// break; +// } +// +// long id = result_ids[i]; +// std::cout << "No." << id << " [" << xb[id * 3] << ", " << xb[id * 3 + 1] << ", " +// << xb[id * 3 + 2] << "] distance = " << result_dists[i] << std::endl; +// +// //makesure result vector is in first quadrant +// ASSERT_TRUE(xb[id * 3] > 0.0); +// ASSERT_TRUE(xb[id * 3 + 1] > 0.0); +// ASSERT_TRUE(xb[id * 3 + 2] > 0.0); +// } +// +// delete[] result_dists; +// delete[] result_ids; +//} +// +//TEST(gpu_build_test, Wrapper_Test) { +// using std::vector; +// +// int d = 256; +// int nb = 3 * 1000 * 100; +// int nq = 100; +// vector xb(d * nb); +// vector xq(d * nq); +// vector ids(nb); +// +// std::random_device rd; +// std::mt19937 gen(rd()); +// std::uniform_real_distribution<> dis_xt(-1.0, 1.0); +// for (auto &e : xb) { e = float(dis_xt(gen)); } +// for (auto &e : xq) { e = float(dis_xt(gen)); } +// for (int i = 0; i < nb; ++i) { ids[i] = i; } +// +// auto opd = std::make_shared(); +// opd->index_type = "IVF"; +// opd->d = d; +// opd->ncent = 256; +// +// IndexBuilderPtr index_builder_1 = GetIndexBuilder(opd); +// auto index_1 = index_builder_1->build_all(nb, xb.data(), ids.data()); +// assert(index_1->ntotal == nb); +// assert(index_1->dim == d); +// +// // sanity check: search 5 first vectors of xb +// int k = 1; +// vector I(5 * k); +// vector D(5 * k); +// index_1->search(5, xb.data(), k, D.data(), I.data()); +// for (int i = 0; i < 5; ++i) { assert(i == I[i]); } +//}