admin/milvus
1
0
Fork 0
mirror of https://gitee.com/milvus-io/milvus.git synced 2026-01-02 00:45:30 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'branch-0.3.0' into 'branch-0.3.0'

Browse Source 
MS-64 Different table can have different index type

See merge request megasearch/vecwise_engine!64

Former-commit-id: 1fa03f45102d55efb05a6c96203c041551280785
This commit is contained in:
jinhai 2019-06-09 16:06:45 +08:00
commit 9cb9fdc7b3
43 changed files with 1577 additions and 4253 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
cpp/CHANGELOG.md
View File

@ -12,6 +12,7 @@ Please mark all change in change log and use the ticket from JIRA.
- MS-57 - Implement index load/search pipeline
- MS-56 - Add version information when server is started
- Ms-64 - Different table can have different index type
## Task

2
cpp/src/CMakeLists.txt
View File

@ -85,7 +85,6 @@ set(third_party_libs
prometheus-cpp-pull
prometheus-cpp-core
civetweb
rocksdb
boost_system_static
boost_filesystem_static
boost_serialization_static
@ -165,7 +164,6 @@ endif ()
set(server_libs
vecwise_engine
librocksdb.a
libthrift.a
pthread
libyaml-cpp.a

53
cpp/src/db/DBImpl.cpp
View File

@ -50,7 +50,7 @@ void CollectQueryMetrics(double total_time, size_t nq) {
server::Metrics::GetInstance().QueryVectorResponsePerSecondGaugeSet(double (nq) / total_time);
}
void CollectFileMetrics(meta::TableFileSchema::FILE_TYPE file_type, size_t file_size, double total_time) {
void CollectFileMetrics(int file_type, size_t file_size, double total_time) {
switch(file_type) {
case meta::TableFileSchema::RAW:
case meta::TableFileSchema::TO_INDEX: {
@ -143,16 +143,16 @@ Status DBImpl::QuerySync(const std::string& table_id, size_t k, size_t nq,
meta::TableFilesSchema raw_files;
for (auto &day_files : files) {
for (auto &file : day_files.second) {
file.file_type == meta::TableFileSchema::INDEX ?
file.file_type_ == meta::TableFileSchema::INDEX ?
index_files.push_back(file) : raw_files.push_back(file);
}
}
int dim = 0;
if (!index_files.empty()) {
dim = index_files[0].dimension;
dim = index_files[0].dimension_;
} else if (!raw_files.empty()) {
dim = raw_files[0].dimension;
dim = raw_files[0].dimension_;
} else {
LOG(DEBUG) << "no files to search";
return Status::OK();
@ -185,12 +185,12 @@ Status DBImpl::QuerySync(const std::string& table_id, size_t k, size_t nq,
auto search_in_index = [&](meta::TableFilesSchema& file_vec) -> void {
for (auto &file : file_vec) {
ExecutionEnginePtr index = EngineFactory::Build(file.dimension, file.location, (EngineType)file.engine_type_);
ExecutionEnginePtr index = EngineFactory::Build(file.dimension_, file.location_, (EngineType)file.engine_type_);
index->Load();
auto file_size = index->PhysicalSize();
search_set_size += file_size;
LOG(DEBUG) << "Search file_type " << file.file_type << " Of Size: "
LOG(DEBUG) << "Search file_type " << file.file_type_ << " Of Size: "
<< file_size/(1024*1024) << " M";
int inner_k = index->Count() < k ? index->Count() : k;
@ -198,7 +198,7 @@ Status DBImpl::QuerySync(const std::string& table_id, size_t k, size_t nq,
index->Search(nq, vectors, inner_k, output_distence, output_ids);
auto end_time = METRICS_NOW_TIME;
auto total_time = METRICS_MICROSECONDS(start_time, end_time);
CollectFileMetrics((meta::TableFileSchema::FILE_TYPE)file.file_type, file_size, total_time);
CollectFileMetrics(file.file_type_, file_size, total_time);
cluster(output_ids, output_distence, inner_k); // cluster to each query
memset(output_distence, 0, k * nq * sizeof(float));
memset(output_ids, 0, k * nq * sizeof(long));
@ -348,8 +348,8 @@ void DBImpl::BackgroundCall() {
Status DBImpl::MergeFiles(const std::string& table_id, const meta::DateT& date,
const meta::TableFilesSchema& files) {
meta::TableFileSchema table_file;
table_file.table_id = table_id;
table_file.date = date;
table_file.table_id_ = table_id;
table_file.date_ = date;
Status status = pMeta_->CreateTableFile(table_file);
if (!status.ok()) {
@ -357,7 +357,8 @@ Status DBImpl::MergeFiles(const std::string& table_id, const meta::DateT& date,
return status;
}
ExecutionEnginePtr index = EngineFactory::Build(table_file.dimension, table_file.location, (EngineType)table_file.engine_type_);
ExecutionEnginePtr index =
EngineFactory::Build(table_file.dimension_, table_file.location_, (EngineType)table_file.engine_type_);
meta::TableFilesSchema updated;
long index_size = 0;
@ -365,15 +366,15 @@ Status DBImpl::MergeFiles(const std::string& table_id, const meta::DateT& date,
for (auto& file : files) {
auto start_time = METRICS_NOW_TIME;
index->Merge(file.location);
index->Merge(file.location_);
auto file_schema = file;
auto end_time = METRICS_NOW_TIME;
auto total_time = METRICS_MICROSECONDS(start_time,end_time);
server::Metrics::GetInstance().MemTableMergeDurationSecondsHistogramObserve(total_time);
file_schema.file_type = meta::TableFileSchema::TO_DELETE;
file_schema.file_type_ = meta::TableFileSchema::TO_DELETE;
updated.push_back(file_schema);
LOG(DEBUG) << "Merging file " << file_schema.file_id;
LOG(DEBUG) << "Merging file " << file_schema.file_id_;
index_size = index->Size();
if (index_size >= options_.index_trigger_size) break;
@ -383,14 +384,14 @@ Status DBImpl::MergeFiles(const std::string& table_id, const meta::DateT& date,
index->Serialize();
if (index_size >= options_.index_trigger_size) {
table_file.file_type = meta::TableFileSchema::TO_INDEX;
table_file.file_type_ = meta::TableFileSchema::TO_INDEX;
} else {
table_file.file_type = meta::TableFileSchema::RAW;
table_file.file_type_ = meta::TableFileSchema::RAW;
}
table_file.size = index_size;
table_file.size_ = index_size;
updated.push_back(table_file);
status = pMeta_->UpdateTableFiles(updated);
LOG(DEBUG) << "New merged file " << table_file.file_id <<
LOG(DEBUG) << "New merged file " << table_file.file_id_ <<
" of size=" << index->PhysicalSize()/(1024*1024) << " M";
index->Cache();
@ -427,34 +428,34 @@ Status DBImpl::BackgroundMergeFiles(const std::string& table_id) {
Status DBImpl::BuildIndex(const meta::TableFileSchema& file) {
meta::TableFileSchema table_file;
table_file.table_id = file.table_id;
table_file.date = file.date;
table_file.table_id_ = file.table_id_;
table_file.date_ = file.date_;
Status status = pMeta_->CreateTableFile(table_file);
if (!status.ok()) {
return status;
}
ExecutionEnginePtr to_index = EngineFactory::Build(file.dimension, file.location, (EngineType)file.engine_type_);
ExecutionEnginePtr to_index = EngineFactory::Build(file.dimension_, file.location_, (EngineType)file.engine_type_);
to_index->Load();
auto start_time = METRICS_NOW_TIME;
auto index = to_index->BuildIndex(table_file.location);
auto index = to_index->BuildIndex(table_file.location_);
auto end_time = METRICS_NOW_TIME;
auto total_time = METRICS_MICROSECONDS(start_time, end_time);
server::Metrics::GetInstance().BuildIndexDurationSecondsHistogramObserve(total_time);
table_file.file_type = meta::TableFileSchema::INDEX;
table_file.size = index->Size();
table_file.file_type_ = meta::TableFileSchema::INDEX;
table_file.size_ = index->Size();
auto to_remove = file;
to_remove.file_type = meta::TableFileSchema::TO_DELETE;
to_remove.file_type_ = meta::TableFileSchema::TO_DELETE;
meta::TableFilesSchema update_files = {to_remove, table_file};
pMeta_->UpdateTableFiles(update_files);
LOG(DEBUG) << "New index file " << table_file.file_id << " of size "
LOG(DEBUG) << "New index file " << table_file.file_id_ << " of size "
<< index->PhysicalSize()/(1024*1024) << " M"
<< " from file " << to_remove.file_id;
<< " from file " << to_remove.file_id_;
index->Cache();
pMeta_->Archive();

429
cpp/src/db/DBMetaImpl.cpp
View File

@ -30,29 +30,30 @@ using namespace sqlite_orm;
inline auto StoragePrototype(const std::string &path) {
return make_storage(path,
make_table("Table",
make_column("id", &TableSchema::id, primary_key()),
make_column("table_id", &TableSchema::table_id, unique()),
make_column("dimension", &TableSchema::dimension),
make_column("created_on", &TableSchema::created_on),
make_column("files_cnt", &TableSchema::files_cnt, default_value(0)),
make_column("engine_type", &TableSchema::engine_type_)),
make_column("id", &TableSchema::id_, primary_key()),
make_column("table_id", &TableSchema::table_id_, unique()),
make_column("dimension", &TableSchema::dimension_),
make_column("created_on", &TableSchema::created_on_),
make_column("files_cnt", &TableSchema::files_cnt_, default_value(0)),
make_column("engine_type", &TableSchema::engine_type_),
make_column("store_raw_data", &TableSchema::store_raw_data_)),
make_table("TableFile",
make_column("id", &TableFileSchema::id, primary_key()),
make_column("table_id", &TableFileSchema::table_id),
make_column("id", &TableFileSchema::id_, primary_key()),
make_column("table_id", &TableFileSchema::table_id_),
make_column("engine_type", &TableFileSchema::engine_type_),
make_column("file_id", &TableFileSchema::file_id),
make_column("file_type", &TableFileSchema::file_type),
make_column("size", &TableFileSchema::size, default_value(0)),
make_column("updated_time", &TableFileSchema::updated_time),
make_column("created_on", &TableFileSchema::created_on),
make_column("date", &TableFileSchema::date))
make_column("file_id", &TableFileSchema::file_id_),
make_column("file_type", &TableFileSchema::file_type_),
make_column("size", &TableFileSchema::size_, default_value(0)),
make_column("updated_time", &TableFileSchema::updated_time_),
make_column("created_on", &TableFileSchema::created_on_),
make_column("date", &TableFileSchema::date_))
);
}
using ConnectorT = decltype(StoragePrototype(""));
static std::unique_ptr<ConnectorT> ConnectorPtr;
using ConditionT = decltype(c(&TableFileSchema::id) == 1UL);
using ConditionT = decltype(c(&TableFileSchema::id_) == 1UL);
std::string DBMetaImpl::GetTablePath(const std::string &table_id) {
return options_.path + "/tables/" + table_id;
@ -65,13 +66,13 @@ std::string DBMetaImpl::GetTableDatePartitionPath(const std::string &table_id, D
}
void DBMetaImpl::GetTableFilePath(TableFileSchema &group_file) {
if (group_file.date == EmptyDate) {
group_file.date = Meta::GetDate();
if (group_file.date_ == EmptyDate) {
group_file.date_ = Meta::GetDate();
}
std::stringstream ss;
ss << GetTableDatePartitionPath(group_file.table_id, group_file.date)
<< "/" << group_file.file_id;
group_file.location = ss.str();
ss << GetTableDatePartitionPath(group_file.table_id_, group_file.date_)
<< "/" << group_file.file_id_;
group_file.location_ = ss.str();
}
Status DBMetaImpl::NextTableId(std::string &table_id) {
@ -123,7 +124,7 @@ Status DBMetaImpl::DropPartitionsByDates(const std::string &table_id,
}
TableSchema table_schema;
table_schema.table_id = table_id;
table_schema.table_id_ = table_id;
auto status = DescribeTable(table_schema);
if (!status.ok()) {
return status;
@ -140,11 +141,11 @@ Status DBMetaImpl::DropPartitionsByDates(const std::string &table_id,
try {
ConnectorPtr->update_all(
set(
c(&TableFileSchema::file_type) = (int) TableFileSchema::TO_DELETE
c(&TableFileSchema::file_type_) = (int) TableFileSchema::TO_DELETE
),
where(
c(&TableFileSchema::table_id) == table_id and
in(&TableFileSchema::date, dates)
c(&TableFileSchema::table_id_) == table_id and
in(&TableFileSchema::date_, dates)
));
} catch (std::exception &e) {
LOG(DEBUG) << e.what();
@ -155,17 +156,17 @@ Status DBMetaImpl::DropPartitionsByDates(const std::string &table_id,
Status DBMetaImpl::CreateTable(TableSchema &table_schema) {
server::Metrics::GetInstance().MetaAccessTotalIncrement();
if (table_schema.table_id == "") {
NextTableId(table_schema.table_id);
if (table_schema.table_id_ == "") {
NextTableId(table_schema.table_id_);
}
table_schema.files_cnt = 0;
table_schema.id = -1;
table_schema.created_on = utils::GetMicroSecTimeStamp();
table_schema.files_cnt_ = 0;
table_schema.id_ = -1;
table_schema.created_on_ = utils::GetMicroSecTimeStamp();
auto start_time = METRICS_NOW_TIME;
{
try {
auto id = ConnectorPtr->insert(table_schema);
table_schema.id = id;
table_schema.id_ = id;
} catch (...) {
return Status::DBTransactionError("Add Table Error");
}
@ -174,7 +175,7 @@ Status DBMetaImpl::CreateTable(TableSchema &table_schema) {
auto total_time = METRICS_MICROSECONDS(start_time, end_time);
server::Metrics::GetInstance().MetaAccessDurationSecondsHistogramObserve(total_time);
auto group_path = GetTablePath(table_schema.table_id);
auto group_path = GetTablePath(table_schema.table_id_);
if (!boost::filesystem::is_directory(group_path)) {
auto ret = boost::filesystem::create_directories(group_path);
@ -191,21 +192,25 @@ Status DBMetaImpl::DescribeTable(TableSchema &table_schema) {
try {
server::Metrics::GetInstance().MetaAccessTotalIncrement();
auto start_time = METRICS_NOW_TIME;
auto groups = ConnectorPtr->select(columns(&TableSchema::id,
&TableSchema::table_id,
&TableSchema::files_cnt,
&TableSchema::dimension),
where(c(&TableSchema::table_id) == table_schema.table_id));
auto groups = ConnectorPtr->select(columns(&TableSchema::id_,
&TableSchema::table_id_,
&TableSchema::files_cnt_,
&TableSchema::dimension_,
&TableSchema::engine_type_,
&TableSchema::store_raw_data_),
where(c(&TableSchema::table_id_) == table_schema.table_id_));
auto end_time = METRICS_NOW_TIME;
auto total_time = METRICS_MICROSECONDS(start_time, end_time);
server::Metrics::GetInstance().MetaAccessDurationSecondsHistogramObserve(total_time);
assert(groups.size() <= 1);
if (groups.size() == 1) {
table_schema.id = std::get<0>(groups[0]);
table_schema.files_cnt = std::get<2>(groups[0]);
table_schema.dimension = std::get<3>(groups[0]);
table_schema.id_ = std::get<0>(groups[0]);
table_schema.files_cnt_ = std::get<2>(groups[0]);
table_schema.dimension_ = std::get<3>(groups[0]);
table_schema.engine_type_ = std::get<4>(groups[0]);
table_schema.store_raw_data_ = std::get<5>(groups[0]);
} else {
return Status::NotFound("Table " + table_schema.table_id + " not found");
return Status::NotFound("Table " + table_schema.table_id_ + " not found");
}
} catch (std::exception &e) {
LOG(DEBUG) << e.what();
@ -220,8 +225,8 @@ Status DBMetaImpl::HasTable(const std::string &table_id, bool &has_or_not) {
server::Metrics::GetInstance().MetaAccessTotalIncrement();
auto start_time = METRICS_NOW_TIME;
auto tables = ConnectorPtr->select(columns(&TableSchema::id),
where(c(&TableSchema::table_id) == table_id));
auto tables = ConnectorPtr->select(columns(&TableSchema::id_),
where(c(&TableSchema::table_id_) == table_id));
auto end_time = METRICS_NOW_TIME;
auto total_time = METRICS_MICROSECONDS(start_time, end_time);
server::Metrics::GetInstance().MetaAccessDurationSecondsHistogramObserve(total_time);
@ -239,22 +244,23 @@ Status DBMetaImpl::HasTable(const std::string &table_id, bool &has_or_not) {
}
Status DBMetaImpl::CreateTableFile(TableFileSchema &file_schema) {
if (file_schema.date == EmptyDate) {
file_schema.date = Meta::GetDate();
if (file_schema.date_ == EmptyDate) {
file_schema.date_ = Meta::GetDate();
}
TableSchema table_schema;
table_schema.table_id = file_schema.table_id;
table_schema.table_id_ = file_schema.table_id_;
auto status = DescribeTable(table_schema);
if (!status.ok()) {
return status;
}
NextFileId(file_schema.file_id);
file_schema.file_type = TableFileSchema::NEW;
file_schema.dimension = table_schema.dimension;
file_schema.size = 0;
file_schema.created_on = utils::GetMicroSecTimeStamp();
file_schema.updated_time = file_schema.created_on;
NextFileId(file_schema.file_id_);
file_schema.file_type_ = TableFileSchema::NEW;
file_schema.dimension_ = table_schema.dimension_;
file_schema.size_ = 0;
file_schema.created_on_ = utils::GetMicroSecTimeStamp();
file_schema.updated_time_ = file_schema.created_on_;
file_schema.engine_type_ = table_schema.engine_type_;
GetTableFilePath(file_schema);
{
@ -262,7 +268,7 @@ Status DBMetaImpl::CreateTableFile(TableFileSchema &file_schema) {
server::Metrics::GetInstance().MetaAccessTotalIncrement();
auto start_time = METRICS_NOW_TIME;
auto id = ConnectorPtr->insert(file_schema);
file_schema.id = id;
file_schema.id_ = id;
auto end_time = METRICS_NOW_TIME;
auto total_time = METRICS_MICROSECONDS(start_time, end_time);
server::Metrics::GetInstance().MetaAccessDurationSecondsHistogramObserve(total_time);
@ -271,7 +277,7 @@ Status DBMetaImpl::CreateTableFile(TableFileSchema &file_schema) {
}
}
auto partition_path = GetTableDatePartitionPath(file_schema.table_id, file_schema.date);
auto partition_path = GetTableDatePartitionPath(file_schema.table_id_, file_schema.date_);
if (!boost::filesystem::is_directory(partition_path)) {
auto ret = boost::filesystem::create_directory(partition_path);
@ -290,13 +296,14 @@ Status DBMetaImpl::FilesToIndex(TableFilesSchema &files) {
try {
server::Metrics::GetInstance().MetaAccessTotalIncrement();
auto start_time = METRICS_NOW_TIME;
auto selected = ConnectorPtr->select(columns(&TableFileSchema::id,
&TableFileSchema::table_id,
&TableFileSchema::file_id,
&TableFileSchema::file_type,
&TableFileSchema::size,
&TableFileSchema::date),
where(c(&TableFileSchema::file_type)
auto selected = ConnectorPtr->select(columns(&TableFileSchema::id_,
&TableFileSchema::table_id_,
&TableFileSchema::file_id_,
&TableFileSchema::file_type_,
&TableFileSchema::size_,
&TableFileSchema::date_,
&TableFileSchema::engine_type_),
where(c(&TableFileSchema::file_type_)
== (int) TableFileSchema::TO_INDEX));
auto end_time = METRICS_NOW_TIME;
auto total_time = METRICS_MICROSECONDS(start_time, end_time);
@ -306,24 +313,26 @@ Status DBMetaImpl::FilesToIndex(TableFilesSchema &files) {
TableFileSchema table_file;
for (auto &file : selected) {
table_file.id = std::get<0>(file);
table_file.table_id = std::get<1>(file);
table_file.file_id = std::get<2>(file);
table_file.file_type = std::get<3>(file);
table_file.size = std::get<4>(file);
table_file.date = std::get<5>(file);
table_file.id_ = std::get<0>(file);
table_file.table_id_ = std::get<1>(file);
table_file.file_id_ = std::get<2>(file);
table_file.file_type_ = std::get<3>(file);
table_file.size_ = std::get<4>(file);
table_file.date_ = std::get<5>(file);
table_file.engine_type_ = std::get<6>(file);
GetTableFilePath(table_file);
auto groupItr = groups.find(table_file.table_id);
auto groupItr = groups.find(table_file.table_id_);
if (groupItr == groups.end()) {
TableSchema table_schema;
table_schema.table_id = table_file.table_id;
table_schema.table_id_ = table_file.table_id_;
auto status = DescribeTable(table_schema);
if (!status.ok()) {
return status;
}
groups[table_file.table_id] = table_schema;
groups[table_file.table_id_] = table_schema;
}
table_file.dimension = groups[table_file.table_id].dimension;
table_file.dimension_ = groups[table_file.table_id_].dimension_;
files.push_back(table_file);
}
} catch (std::exception &e) {
@ -343,23 +352,24 @@ Status DBMetaImpl::FilesToSearch(const std::string &table_id,
server::Metrics::GetInstance().MetaAccessTotalIncrement();
auto start_time = METRICS_NOW_TIME;
if (partition.empty()) {
auto selected = ConnectorPtr->select(columns(&TableFileSchema::id,
&TableFileSchema::table_id,
&TableFileSchema::file_id,
&TableFileSchema::file_type,
&TableFileSchema::size,
&TableFileSchema::date),
where(c(&TableFileSchema::table_id) == table_id and
(c(&TableFileSchema::file_type) == (int) TableFileSchema::RAW or
c(&TableFileSchema::file_type)
auto selected = ConnectorPtr->select(columns(&TableFileSchema::id_,
&TableFileSchema::table_id_,
&TableFileSchema::file_id_,
&TableFileSchema::file_type_,
&TableFileSchema::size_,
&TableFileSchema::date_,
&TableFileSchema::engine_type_),
where(c(&TableFileSchema::table_id_) == table_id and
(c(&TableFileSchema::file_type_) == (int) TableFileSchema::RAW or
c(&TableFileSchema::file_type_)
== (int) TableFileSchema::TO_INDEX or
c(&TableFileSchema::file_type)
c(&TableFileSchema::file_type_)
== (int) TableFileSchema::INDEX)));
auto end_time = METRICS_NOW_TIME;
auto total_time = METRICS_MICROSECONDS(start_time, end_time);
server::Metrics::GetInstance().MetaAccessDurationSecondsHistogramObserve(total_time);
TableSchema table_schema;
table_schema.table_id = table_id;
table_schema.table_id_ = table_id;
auto status = DescribeTable(table_schema);
if (!status.ok()) {
return status;
@ -368,40 +378,41 @@ Status DBMetaImpl::FilesToSearch(const std::string &table_id,
TableFileSchema table_file;
for (auto &file : selected) {
table_file.id = std::get<0>(file);
table_file.table_id = std::get<1>(file);
table_file.file_id = std::get<2>(file);
table_file.file_type = std::get<3>(file);
table_file.size = std::get<4>(file);
table_file.date = std::get<5>(file);
table_file.dimension = table_schema.dimension;
table_file.id_ = std::get<0>(file);
table_file.table_id_ = std::get<1>(file);
table_file.file_id_ = std::get<2>(file);
table_file.file_type_ = std::get<3>(file);
table_file.size_ = std::get<4>(file);
table_file.date_ = std::get<5>(file);
table_file.engine_type_ = std::get<6>(file);
table_file.dimension_ = table_schema.dimension_;
GetTableFilePath(table_file);
auto dateItr = files.find(table_file.date);
auto dateItr = files.find(table_file.date_);
if (dateItr == files.end()) {
files[table_file.date] = TableFilesSchema();
files[table_file.date_] = TableFilesSchema();
}
files[table_file.date].push_back(table_file);
files[table_file.date_].push_back(table_file);
}
}
else {
auto selected = ConnectorPtr->select(columns(&TableFileSchema::id,
&TableFileSchema::table_id,
&TableFileSchema::file_id,
&TableFileSchema::file_type,
&TableFileSchema::size,
&TableFileSchema::date),
where(c(&TableFileSchema::table_id) == table_id and
in(&TableFileSchema::date, partition) and
(c(&TableFileSchema::file_type) == (int) TableFileSchema::RAW or
c(&TableFileSchema::file_type)
auto selected = ConnectorPtr->select(columns(&TableFileSchema::id_,
&TableFileSchema::table_id_,
&TableFileSchema::file_id_,
&TableFileSchema::file_type_,
&TableFileSchema::size_,
&TableFileSchema::date_),
where(c(&TableFileSchema::table_id_) == table_id and
in(&TableFileSchema::date_, partition) and
(c(&TableFileSchema::file_type_) == (int) TableFileSchema::RAW or
c(&TableFileSchema::file_type_)
== (int) TableFileSchema::TO_INDEX or
c(&TableFileSchema::file_type)
c(&TableFileSchema::file_type_)
== (int) TableFileSchema::INDEX)));
auto end_time = METRICS_NOW_TIME;
auto total_time = METRICS_MICROSECONDS(start_time, end_time);
server::Metrics::GetInstance().MetaAccessDurationSecondsHistogramObserve(total_time);
TableSchema table_schema;
table_schema.table_id = table_id;
table_schema.table_id_ = table_id;
auto status = DescribeTable(table_schema);
if (!status.ok()) {
return status;
@ -410,19 +421,19 @@ Status DBMetaImpl::FilesToSearch(const std::string &table_id,
TableFileSchema table_file;
for (auto &file : selected) {
table_file.id = std::get<0>(file);
table_file.table_id = std::get<1>(file);
table_file.file_id = std::get<2>(file);
table_file.file_type = std::get<3>(file);
table_file.size = std::get<4>(file);
table_file.date = std::get<5>(file);
table_file.dimension = table_schema.dimension;
table_file.id_ = std::get<0>(file);
table_file.table_id_ = std::get<1>(file);
table_file.file_id_ = std::get<2>(file);
table_file.file_type_ = std::get<3>(file);
table_file.size_ = std::get<4>(file);
table_file.date_ = std::get<5>(file);
table_file.dimension_ = table_schema.dimension_;
GetTableFilePath(table_file);
auto dateItr = files.find(table_file.date);
auto dateItr = files.find(table_file.date_);
if (dateItr == files.end()) {
files[table_file.date] = TableFilesSchema();
files[table_file.date_] = TableFilesSchema();
}
files[table_file.date].push_back(table_file);
files[table_file.date_].push_back(table_file);
}
}
@ -441,19 +452,19 @@ Status DBMetaImpl::FilesToMerge(const std::string &table_id,
try {
server::Metrics::GetInstance().MetaAccessTotalIncrement();
auto start_time = METRICS_NOW_TIME;
auto selected = ConnectorPtr->select(columns(&TableFileSchema::id,
&TableFileSchema::table_id,
&TableFileSchema::file_id,
&TableFileSchema::file_type,
&TableFileSchema::size,
&TableFileSchema::date),
where(c(&TableFileSchema::file_type) == (int) TableFileSchema::RAW and
c(&TableFileSchema::table_id) == table_id));
auto selected = ConnectorPtr->select(columns(&TableFileSchema::id_,
&TableFileSchema::table_id_,
&TableFileSchema::file_id_,
&TableFileSchema::file_type_,
&TableFileSchema::size_,
&TableFileSchema::date_),
where(c(&TableFileSchema::file_type_) == (int) TableFileSchema::RAW and
c(&TableFileSchema::table_id_) == table_id));
auto end_time = METRICS_NOW_TIME;
auto total_time = METRICS_MICROSECONDS(start_time, end_time);
server::Metrics::GetInstance().MetaAccessDurationSecondsHistogramObserve(total_time);
TableSchema table_schema;
table_schema.table_id = table_id;
table_schema.table_id_ = table_id;
auto status = DescribeTable(table_schema);
if (!status.ok()) {
@ -462,19 +473,19 @@ Status DBMetaImpl::FilesToMerge(const std::string &table_id,
TableFileSchema table_file;
for (auto &file : selected) {
table_file.id = std::get<0>(file);
table_file.table_id = std::get<1>(file);
table_file.file_id = std::get<2>(file);
table_file.file_type = std::get<3>(file);
table_file.size = std::get<4>(file);
table_file.date = std::get<5>(file);
table_file.dimension = table_schema.dimension;
table_file.id_ = std::get<0>(file);
table_file.table_id_ = std::get<1>(file);
table_file.file_id_ = std::get<2>(file);
table_file.file_type_ = std::get<3>(file);
table_file.size_ = std::get<4>(file);
table_file.date_ = std::get<5>(file);
table_file.dimension_ = table_schema.dimension_;
GetTableFilePath(table_file);
auto dateItr = files.find(table_file.date);
auto dateItr = files.find(table_file.date_);
if (dateItr == files.end()) {
files[table_file.date] = TableFilesSchema();
files[table_file.date_] = TableFilesSchema();
}
files[table_file.date].push_back(table_file);
files[table_file.date_].push_back(table_file);
}
} catch (std::exception &e) {
LOG(DEBUG) << e.what();
@ -487,26 +498,26 @@ Status DBMetaImpl::FilesToMerge(const std::string &table_id,
Status DBMetaImpl::GetTableFile(TableFileSchema &file_schema) {
try {
auto files = ConnectorPtr->select(columns(&TableFileSchema::id,
&TableFileSchema::table_id,
&TableFileSchema::file_id,
&TableFileSchema::file_type,
&TableFileSchema::size,
&TableFileSchema::date),
where(c(&TableFileSchema::file_id) == file_schema.file_id and
c(&TableFileSchema::table_id) == file_schema.table_id
auto files = ConnectorPtr->select(columns(&TableFileSchema::id_,
&TableFileSchema::table_id_,
&TableFileSchema::file_id_,
&TableFileSchema::file_type_,
&TableFileSchema::size_,
&TableFileSchema::date_),
where(c(&TableFileSchema::file_id_) == file_schema.file_id_ and
c(&TableFileSchema::table_id_) == file_schema.table_id_
));
assert(files.size() <= 1);
if (files.size() == 1) {
file_schema.id = std::get<0>(files[0]);
file_schema.table_id = std::get<1>(files[0]);
file_schema.file_id = std::get<2>(files[0]);
file_schema.file_type = std::get<3>(files[0]);
file_schema.size = std::get<4>(files[0]);
file_schema.date = std::get<5>(files[0]);
file_schema.id_ = std::get<0>(files[0]);
file_schema.table_id_ = std::get<1>(files[0]);
file_schema.file_id_ = std::get<2>(files[0]);
file_schema.file_type_ = std::get<3>(files[0]);
file_schema.size_ = std::get<4>(files[0]);
file_schema.date_ = std::get<5>(files[0]);
} else {
return Status::NotFound("Table:" + file_schema.table_id +
" File:" + file_schema.file_id + " not found");
return Status::NotFound("Table:" + file_schema.table_id_ +
" File:" + file_schema.file_id_ + " not found");
}
} catch (std::exception &e) {
LOG(DEBUG) << e.what();
@ -532,11 +543,11 @@ Status DBMetaImpl::Archive() {
try {
ConnectorPtr->update_all(
set(
c(&TableFileSchema::file_type) = (int) TableFileSchema::TO_DELETE
c(&TableFileSchema::file_type_) = (int) TableFileSchema::TO_DELETE
),
where(
c(&TableFileSchema::created_on) < (long) (now - usecs) and
c(&TableFileSchema::file_type) != (int) TableFileSchema::TO_DELETE
c(&TableFileSchema::created_on_) < (long) (now - usecs) and
c(&TableFileSchema::file_type_) != (int) TableFileSchema::TO_DELETE
));
} catch (std::exception &e) {
LOG(DEBUG) << e.what();
@ -558,9 +569,9 @@ Status DBMetaImpl::Archive() {
Status DBMetaImpl::Size(long &result) {
result = 0;
try {
auto selected = ConnectorPtr->select(columns(sum(&TableFileSchema::size)),
auto selected = ConnectorPtr->select(columns(sum(&TableFileSchema::size_)),
where(
c(&TableFileSchema::file_type) != (int) TableFileSchema::TO_DELETE
c(&TableFileSchema::file_type_) != (int) TableFileSchema::TO_DELETE
));
for (auto &sub_query : selected) {
@ -583,11 +594,11 @@ Status DBMetaImpl::DiscardFiles(long to_discard_size) {
return Status::OK();
}
try {
auto selected = ConnectorPtr->select(columns(&TableFileSchema::id,
&TableFileSchema::size),
where(c(&TableFileSchema::file_type)
auto selected = ConnectorPtr->select(columns(&TableFileSchema::id_,
&TableFileSchema::size_),
where(c(&TableFileSchema::file_type_)
!= (int) TableFileSchema::TO_DELETE),
order_by(&TableFileSchema::id),
order_by(&TableFileSchema::id_),
limit(10));
std::vector<int> ids;
@ -595,11 +606,11 @@ Status DBMetaImpl::DiscardFiles(long to_discard_size) {
for (auto &file : selected) {
if (to_discard_size <= 0) break;
table_file.id = std::get<0>(file);
table_file.size = std::get<1>(file);
ids.push_back(table_file.id);
LOG(DEBUG) << "Discard table_file.id=" << table_file.file_id << " table_file.size=" << table_file.size;
to_discard_size -= table_file.size;
table_file.id_ = std::get<0>(file);
table_file.size_ = std::get<1>(file);
ids.push_back(table_file.id_);
LOG(DEBUG) << "Discard table_file.id=" << table_file.file_id_ << " table_file.size=" << table_file.size_;
to_discard_size -= table_file.size_;
}
if (ids.size() == 0) {
@ -608,10 +619,10 @@ Status DBMetaImpl::DiscardFiles(long to_discard_size) {
ConnectorPtr->update_all(
set(
c(&TableFileSchema::file_type) = (int) TableFileSchema::TO_DELETE
c(&TableFileSchema::file_type_) = (int) TableFileSchema::TO_DELETE
),
where(
in(&TableFileSchema::id, ids)
in(&TableFileSchema::id_, ids)
));
} catch (std::exception &e) {
@ -624,7 +635,7 @@ Status DBMetaImpl::DiscardFiles(long to_discard_size) {
}
Status DBMetaImpl::UpdateTableFile(TableFileSchema &file_schema) {
file_schema.updated_time = utils::GetMicroSecTimeStamp();
file_schema.updated_time_ = utils::GetMicroSecTimeStamp();
try {
server::Metrics::GetInstance().MetaAccessTotalIncrement();
auto start_time = METRICS_NOW_TIME;
@ -634,7 +645,7 @@ Status DBMetaImpl::UpdateTableFile(TableFileSchema &file_schema) {
server::Metrics::GetInstance().MetaAccessDurationSecondsHistogramObserve(total_time);
} catch (std::exception &e) {
LOG(DEBUG) << e.what();
LOG(DEBUG) << "table_id= " << file_schema.table_id << " file_id=" << file_schema.file_id;
LOG(DEBUG) << "table_id= " << file_schema.table_id_ << " file_id=" << file_schema.file_id_;
throw e;
}
return Status::OK();
@ -646,7 +657,7 @@ Status DBMetaImpl::UpdateTableFiles(TableFilesSchema &files) {
auto start_time = METRICS_NOW_TIME;
auto commited = ConnectorPtr->transaction([&]() mutable {
for (auto &file : files) {
file.updated_time = utils::GetMicroSecTimeStamp();
file.updated_time_ = utils::GetMicroSecTimeStamp();
ConnectorPtr->update(file);
}
auto end_time = METRICS_NOW_TIME;
@ -667,33 +678,33 @@ Status DBMetaImpl::UpdateTableFiles(TableFilesSchema &files) {
Status DBMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) {
auto now = utils::GetMicroSecTimeStamp();
try {
auto selected = ConnectorPtr->select(columns(&TableFileSchema::id,
&TableFileSchema::table_id,
&TableFileSchema::file_id,
&TableFileSchema::file_type,
&TableFileSchema::size,
&TableFileSchema::date),
auto selected = ConnectorPtr->select(columns(&TableFileSchema::id_,
&TableFileSchema::table_id_,
&TableFileSchema::file_id_,
&TableFileSchema::file_type_,
&TableFileSchema::size_,
&TableFileSchema::date_),
where(
c(&TableFileSchema::file_type) == (int) TableFileSchema::TO_DELETE
c(&TableFileSchema::file_type_) == (int) TableFileSchema::TO_DELETE
and
c(&TableFileSchema::updated_time)
c(&TableFileSchema::updated_time_)
> now - seconds * US_PS));
TableFilesSchema updated;
TableFileSchema table_file;
for (auto &file : selected) {
table_file.id = std::get<0>(file);
table_file.table_id = std::get<1>(file);
table_file.file_id = std::get<2>(file);
table_file.file_type = std::get<3>(file);
table_file.size = std::get<4>(file);
table_file.date = std::get<5>(file);
table_file.id_ = std::get<0>(file);
table_file.table_id_ = std::get<1>(file);
table_file.file_id_ = std::get<2>(file);
table_file.file_type_ = std::get<3>(file);
table_file.size_ = std::get<4>(file);
table_file.date_ = std::get<5>(file);
GetTableFilePath(table_file);
if (table_file.file_type == TableFileSchema::TO_DELETE) {
boost::filesystem::remove(table_file.location);
if (table_file.file_type_ == TableFileSchema::TO_DELETE) {
boost::filesystem::remove(table_file.location_);
}
ConnectorPtr->remove<TableFileSchema>(table_file.id);
ConnectorPtr->remove<TableFileSchema>(table_file.id_);
/* LOG(DEBUG) << "Removing deleted id=" << table_file.id << " location=" << table_file.location << std::endl; */
}
} catch (std::exception &e) {
@ -706,33 +717,33 @@ Status DBMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) {
Status DBMetaImpl::CleanUp() {
try {
auto selected = ConnectorPtr->select(columns(&TableFileSchema::id,
&TableFileSchema::table_id,
&TableFileSchema::file_id,
&TableFileSchema::file_type,
&TableFileSchema::size,
&TableFileSchema::date),
auto selected = ConnectorPtr->select(columns(&TableFileSchema::id_,
&TableFileSchema::table_id_,
&TableFileSchema::file_id_,
&TableFileSchema::file_type_,
&TableFileSchema::size_,
&TableFileSchema::date_),
where(
c(&TableFileSchema::file_type) == (int) TableFileSchema::TO_DELETE
c(&TableFileSchema::file_type_) == (int) TableFileSchema::TO_DELETE
or
c(&TableFileSchema::file_type)
c(&TableFileSchema::file_type_)
== (int) TableFileSchema::NEW));
TableFilesSchema updated;
TableFileSchema table_file;
for (auto &file : selected) {
table_file.id = std::get<0>(file);
table_file.table_id = std::get<1>(file);
table_file.file_id = std::get<2>(file);
table_file.file_type = std::get<3>(file);
table_file.size = std::get<4>(file);
table_file.date = std::get<5>(file);
table_file.id_ = std::get<0>(file);
table_file.table_id_ = std::get<1>(file);
table_file.file_id_ = std::get<2>(file);
table_file.file_type_ = std::get<3>(file);
table_file.size_ = std::get<4>(file);
table_file.date_ = std::get<5>(file);
GetTableFilePath(table_file);
if (table_file.file_type == TableFileSchema::TO_DELETE) {
boost::filesystem::remove(table_file.location);
if (table_file.file_type_ == TableFileSchema::TO_DELETE) {
boost::filesystem::remove(table_file.location_);
}
ConnectorPtr->remove<TableFileSchema>(table_file.id);
ConnectorPtr->remove<TableFileSchema>(table_file.id_);
/* LOG(DEBUG) << "Removing id=" << table_file.id << " location=" << table_file.location << std::endl; */
}
} catch (std::exception &e) {
@ -749,19 +760,19 @@ Status DBMetaImpl::Count(const std::string &table_id, long &result) {
server::Metrics::GetInstance().MetaAccessTotalIncrement();
auto start_time = METRICS_NOW_TIME;
auto selected = ConnectorPtr->select(columns(&TableFileSchema::size,
&TableFileSchema::date),
where((c(&TableFileSchema::file_type) == (int) TableFileSchema::RAW or
c(&TableFileSchema::file_type) == (int) TableFileSchema::TO_INDEX
auto selected = ConnectorPtr->select(columns(&TableFileSchema::size_,
&TableFileSchema::date_),
where((c(&TableFileSchema::file_type_) == (int) TableFileSchema::RAW or
c(&TableFileSchema::file_type_) == (int) TableFileSchema::TO_INDEX
or
c(&TableFileSchema::file_type) == (int) TableFileSchema::INDEX)
c(&TableFileSchema::file_type_) == (int) TableFileSchema::INDEX)
and
c(&TableFileSchema::table_id) == table_id));
c(&TableFileSchema::table_id_) == table_id));
auto end_time = METRICS_NOW_TIME;
auto total_time = METRICS_MICROSECONDS(start_time, end_time);
server::Metrics::GetInstance().MetaAccessDurationSecondsHistogramObserve(total_time);
TableSchema table_schema;
table_schema.table_id = table_id;
table_schema.table_id_ = table_id;
auto status = DescribeTable(table_schema);
if (!status.ok()) {
@ -773,7 +784,7 @@ Status DBMetaImpl::Count(const std::string &table_id, long &result) {
result += std::get<0>(file);
}
result /= table_schema.dimension;
result /= table_schema.dimension_;
} catch (std::exception &e) {
LOG(DEBUG) << e.what();

1
cpp/src/db/ExecutionEngine.h
View File

@ -15,6 +15,7 @@ namespace vecwise {
namespace engine {
enum class EngineType {
INVALID = 0,
FAISS_IDMAP = 1,
FAISS_IVFFLAT,
};

14
cpp/src/db/MemManager.cpp
View File

@ -24,7 +24,7 @@ MemVectors::MemVectors(const std::shared_ptr<meta::Meta>& meta_ptr,
options_(options),
schema_(schema),
pIdGenerator_(new SimpleIDGenerator()),
pEE_(EngineFactory::Build(schema_.dimension, schema_.location, (EngineType)schema_.engine_type_)) {
pEE_(EngineFactory::Build(schema_.dimension_, schema_.location_, (EngineType)schema_.engine_type_)) {
}
void MemVectors::Add(size_t n_, const float* vectors_, IDNumbers& vector_ids_) {
@ -41,23 +41,23 @@ size_t MemVectors::ApproximateSize() const {
}
Status MemVectors::Serialize(std::string& table_id) {
table_id = schema_.table_id;
table_id = schema_.table_id_;
auto size = ApproximateSize();
auto start_time = METRICS_NOW_TIME;
pEE_->Serialize();
auto end_time = METRICS_NOW_TIME;
auto total_time = METRICS_MICROSECONDS(start_time, end_time);
schema_.size = size;
schema_.size_ = size;
server::Metrics::GetInstance().DiskStoreIOSpeedGaugeSet(size/total_time);
schema_.file_type = (size >= options_.index_trigger_size) ?
schema_.file_type_ = (size >= options_.index_trigger_size) ?
meta::TableFileSchema::TO_INDEX : meta::TableFileSchema::RAW;
auto status = pMeta_->UpdateTableFile(schema_);
LOG(DEBUG) << "New " << ((schema_.file_type == meta::TableFileSchema::RAW) ? "raw" : "to_index")
<< " file " << schema_.file_id << " of size " << pEE_->Size() / meta::M << " M";
LOG(DEBUG) << "New " << ((schema_.file_type_ == meta::TableFileSchema::RAW) ? "raw" : "to_index")
<< " file " << schema_.file_id_ << " of size " << pEE_->Size() / meta::M << " M";
pEE_->Cache();
@ -82,7 +82,7 @@ MemManager::MemVectorsPtr MemManager::GetMemByTable(
}
meta::TableFileSchema table_file;
table_file.table_id = table_id;
table_file.table_id_ = table_id;
auto status = pMeta_->CreateTableFile(table_file);
if (!status.ok()) {
return nullptr;

2
cpp/src/db/MemManager.h
View File

@ -42,7 +42,7 @@ public:
~MemVectors();
const std::string& Location() const { return schema_.location; }
const std::string& Location() const { return schema_.location_; }
private:
MemVectors() = delete;

33
cpp/src/db/MetaTypes.h
View File

@ -21,13 +21,14 @@ const DateT EmptyDate = -1;
typedef std::vector<DateT> DatesT;
struct TableSchema {
size_t id;
std::string table_id;
size_t files_cnt = 0;
uint16_t dimension;
std::string location;
long created_on;
size_t id_;
std::string table_id_;
size_t files_cnt_ = 0;
uint16_t dimension_;
std::string location_;
long created_on_;
int engine_type_ = (int)EngineType::FAISS_IDMAP;
bool store_raw_data_ = false;
}; // TableSchema
struct TableFileSchema {
@ -39,17 +40,17 @@ struct TableFileSchema {
TO_DELETE,
} FILE_TYPE;
size_t id;
std::string table_id;
size_t id_;
std::string table_id_;
int engine_type_ = (int)EngineType::FAISS_IDMAP;
std::string file_id;
int file_type = NEW;
size_t size;
DateT date = EmptyDate;
uint16_t dimension;
std::string location;
long updated_time;
long created_on;
std::string file_id_;
int file_type_ = NEW;
size_t size_;
DateT date_ = EmptyDate;
uint16_t dimension_;
std::string location_;
long updated_time_;
long created_on_;
}; // TableFileSchema
typedef std::vector<TableFileSchema> TableFilesSchema;

8
cpp/src/db/scheduler/ScheduleStrategy.cpp
View File

@ -24,21 +24,21 @@ public:
SearchContext::Id2IndexMap index_files = search_context->GetIndexMap();
//some index loader alread exists
for(auto& loader : loader_list) {
if(index_files.find(loader->file_->id) != index_files.end()){
if(index_files.find(loader->file_->id_) != index_files.end()){
SERVER_LOG_INFO << "Append SearchContext to exist IndexLoaderContext";
index_files.erase(loader->file_->id);
index_files.erase(loader->file_->id_);
loader->search_contexts_.push_back(search_context);
}
}
//index_files still contains some index files, create new loader
for(auto& pair : index_files) {
SERVER_LOG_INFO << "Create new IndexLoaderContext for: " << pair.second->location;
SERVER_LOG_INFO << "Create new IndexLoaderContext for: " << pair.second->location_;
IndexLoaderContextPtr new_loader = std::make_shared<IndexLoaderContext>();
new_loader->search_contexts_.push_back(search_context);
new_loader->file_ = pair.second;
auto index = zilliz::vecwise::cache::CpuCacheMgr::GetInstance()->GetIndex(pair.second->location);
auto index = zilliz::vecwise::cache::CpuCacheMgr::GetInstance()->GetIndex(pair.second->location_);
if(index != nullptr) {
//if the index file has been in memory, increase its priority
loader_list.push_front(new_loader);

6
cpp/src/db/scheduler/SearchContext.cpp
View File

@ -26,13 +26,13 @@ SearchContext::SearchContext(uint64_t topk, uint64_t nq, const float* vectors)
bool
SearchContext::AddIndexFile(TableFileSchemaPtr& index_file) {
std::unique_lock <std::mutex> lock(mtx_);
if(index_file == nullptr || map_index_files_.find(index_file->id) != map_index_files_.end()) {
if(index_file == nullptr || map_index_files_.find(index_file->id_) != map_index_files_.end()) {
return false;
}
SERVER_LOG_INFO << "SearchContext " << identity_ << " add index file: " << index_file->id;
SERVER_LOG_INFO << "SearchContext " << identity_ << " add index file: " << index_file->id_;
map_index_files_[index_file->id] = index_file;
map_index_files_[index_file->id_] = index_file;
return true;
}

91
cpp/src/db/scheduler/SearchScheduler.cpp
View File

@ -16,6 +16,44 @@ namespace zilliz {
namespace vecwise {
namespace engine {
namespace {
void CollectFileMetrics(int file_type, size_t file_size) {
switch(file_type) {
case meta::TableFileSchema::RAW:
case meta::TableFileSchema::TO_INDEX: {
server::Metrics::GetInstance().RawFileSizeHistogramObserve(file_size);
server::Metrics::GetInstance().RawFileSizeTotalIncrement(file_size);
server::Metrics::GetInstance().RawFileSizeGaugeSet(file_size);
break;
}
default: {
server::Metrics::GetInstance().IndexFileSizeHistogramObserve(file_size);
server::Metrics::GetInstance().IndexFileSizeTotalIncrement(file_size);
server::Metrics::GetInstance().IndexFileSizeGaugeSet(file_size);
break;
}
}
}
void CollectDurationMetrics(int index_type, double total_time) {
switch(index_type) {
case meta::TableFileSchema::RAW: {
server::Metrics::GetInstance().SearchRawDataDurationSecondsHistogramObserve(total_time);
break;
}
case meta::TableFileSchema::TO_INDEX: {
server::Metrics::GetInstance().SearchRawDataDurationSecondsHistogramObserve(total_time);
break;
}
default: {
server::Metrics::GetInstance().SearchIndexDataDurationSecondsHistogramObserve(total_time);
break;
}
}
}
}
SearchScheduler::SearchScheduler()
: thread_pool_(2),
stopped_(true) {
@ -76,47 +114,27 @@ SearchScheduler::IndexLoadWorker() {
break;//exit
}
SERVER_LOG_INFO << "Loading index(" << context->file_->id << ") from location: " << context->file_->location;
SERVER_LOG_INFO << "Loading index(" << context->file_->id_ << ") from location: " << context->file_->location_;
server::TimeRecorder rc("Load index");
//load index
ExecutionEnginePtr index_ptr = EngineFactory::Build(context->file_->dimension,
context->file_->location,
//step 1: load index
ExecutionEnginePtr index_ptr = EngineFactory::Build(context->file_->dimension_,
context->file_->location_,
(EngineType)context->file_->engine_type_);
index_ptr->Load();
rc.Record("load index file to memory");
size_t file_size = index_ptr->PhysicalSize();
LOG(DEBUG) << "Index file type " << context->file_->file_type << " Of Size: "
LOG(DEBUG) << "Index file type " << context->file_->file_type_ << " Of Size: "
<< file_size/(1024*1024) << " M";
//metric
switch(context->file_->file_type) {
case meta::TableFileSchema::RAW: {
server::Metrics::GetInstance().RawFileSizeHistogramObserve(file_size);
server::Metrics::GetInstance().RawFileSizeTotalIncrement(file_size);
server::Metrics::GetInstance().RawFileSizeGaugeSet(file_size);
break;
}
case meta::TableFileSchema::TO_INDEX: {
server::Metrics::GetInstance().RawFileSizeHistogramObserve(file_size);
server::Metrics::GetInstance().RawFileSizeTotalIncrement(file_size);
server::Metrics::GetInstance().RawFileSizeGaugeSet(file_size);
break;
}
default: {
server::Metrics::GetInstance().IndexFileSizeHistogramObserve(file_size);
server::Metrics::GetInstance().IndexFileSizeTotalIncrement(file_size);
server::Metrics::GetInstance().IndexFileSizeGaugeSet(file_size);
break;
}
}
CollectFileMetrics(context->file_->file_type_, file_size);
//put search task to another queue
//step 2: put search task into another queue
SearchTaskPtr task_ptr = std::make_shared<SearchTask>();
task_ptr->index_id_ = context->file_->id;
task_ptr->index_type_ = context->file_->file_type;
task_ptr->index_id_ = context->file_->id_;
task_ptr->index_type_ = context->file_->file_type_;
task_ptr->index_engine_ = index_ptr;
task_ptr->search_contexts_.swap(context->search_contexts_);
search_queue.Put(task_ptr);
@ -143,20 +161,7 @@ SearchScheduler::SearchWorker() {
task_ptr->DoSearch();
auto end_time = METRICS_NOW_TIME;
auto total_time = METRICS_MICROSECONDS(start_time, end_time);
switch(task_ptr->index_type_) {
case meta::TableFileSchema::RAW: {
server::Metrics::GetInstance().SearchRawDataDurationSecondsHistogramObserve(total_time);
break;
}
case meta::TableFileSchema::TO_INDEX: {
server::Metrics::GetInstance().SearchRawDataDurationSecondsHistogramObserve(total_time);
break;
}
default: {
server::Metrics::GetInstance().SearchIndexDataDurationSecondsHistogramObserve(total_time);
break;
}
}
CollectDurationMetrics(task_ptr->index_type_, total_time);
}
return true;

82
cpp/src/sdk/examples/simple/src/ClientTest.cpp
View File

@ -16,9 +16,6 @@ namespace {
std::string GetTableName();
static const std::string TABLE_NAME = GetTableName();
static const std::string VECTOR_COLUMN_NAME = "face_vector";
static const std::string ID_COLUMN_NAME = "aid";
static const std::string CITY_COLUMN_NAME = "city";
static constexpr int64_t TABLE_DIMENSION = 512;
static constexpr int64_t TOTAL_ROW_COUNT = 100000;
static constexpr int64_t TOP_K = 10;
@ -29,9 +26,9 @@ namespace {
void PrintTableSchema(const megasearch::TableSchema& tb_schema) {
BLOCK_SPLITER
std::cout << "Table name: " << tb_schema.table_name << std::endl;
std::cout << "Table vectors: " << tb_schema.vector_column_array.size() << std::endl;
std::cout << "Table attributes: " << tb_schema.attribute_column_array.size() << std::endl;
std::cout << "Table partitions: " << tb_schema.partition_column_name_array.size() << std::endl;
std::cout << "Table index type: " << (int)tb_schema.index_type << std::endl;
std::cout << "Table dimension: " << tb_schema.dimension << std::endl;
std::cout << "Table store raw data: " << tb_schema.store_raw_vector << std::endl;
BLOCK_SPLITER
}
@ -58,9 +55,6 @@ namespace {
<< " search result:" << std::endl;
for(auto& item : result.query_result_arrays) {
std::cout << "\t" << std::to_string(item.id) << "\tscore:" << std::to_string(item.score);
for(auto& attribute : item.column_map) {
std::cout << "\t" << attribute.first << ":" << attribute.second;
}
std::cout << std::endl;
}
}
@ -88,73 +82,30 @@ namespace {
TableSchema BuildTableSchema() {
TableSchema tb_schema;
VectorColumn col1;
col1.name = VECTOR_COLUMN_NAME;
col1.dimension = TABLE_DIMENSION;
col1.store_raw_vector = true;
tb_schema.vector_column_array.emplace_back(col1);
Column col2 = {ColumnType::int8, ID_COLUMN_NAME};
tb_schema.attribute_column_array.emplace_back(col2);
Column col3 = {ColumnType::int16, CITY_COLUMN_NAME};
tb_schema.attribute_column_array.emplace_back(col3);
tb_schema.table_name = TABLE_NAME;
tb_schema.index_type = IndexType::gpu_ivfflat;
tb_schema.dimension = TABLE_DIMENSION;
tb_schema.store_raw_vector = true;
return tb_schema;
}
void BuildVectors(int64_t from, int64_t to,
std::vector<RowRecord>* vector_record_array,
std::vector<QueryRecord>* query_record_array) {
std::vector<RowRecord>& vector_record_array) {
if(to <= from){
return;
}
if(vector_record_array) {
vector_record_array->clear();
}
if(query_record_array) {
query_record_array->clear();
}
static const std::map<int64_t , std::string> CITY_MAP = {
{0, "Beijing"},
{1, "Shanhai"},
{2, "Hangzhou"},
{3, "Guangzhou"},
{4, "Shenzheng"},
{5, "Wuhan"},
{6, "Chengdu"},
{7, "Chongqin"},
{8, "Tianjing"},
{9, "Hongkong"},
};
vector_record_array.clear();
for (int64_t k = from; k < to; k++) {
std::vector<float> f_p;
f_p.resize(TABLE_DIMENSION);
RowRecord record;
record.data.resize(TABLE_DIMENSION);
for(int64_t i = 0; i < TABLE_DIMENSION; i++) {
f_p[i] = (float)(i + k);
record.data[i] = (float)(i + k);
}
if(vector_record_array) {
RowRecord record;
record.vector_map.insert(std::make_pair(VECTOR_COLUMN_NAME, f_p));
record.attribute_map[ID_COLUMN_NAME] = std::to_string(k);
record.attribute_map[CITY_COLUMN_NAME] = CITY_MAP.at(k%CITY_MAP.size());
vector_record_array->emplace_back(record);
}
if(query_record_array) {
QueryRecord record;
record.vector_map.insert(std::make_pair(VECTOR_COLUMN_NAME, f_p));
record.selected_column_array.push_back(ID_COLUMN_NAME);
record.selected_column_array.push_back(CITY_COLUMN_NAME);
query_record_array->emplace_back(record);
}
vector_record_array.emplace_back(record);
}
}
}
@ -205,7 +156,7 @@ ClientTest::Test(const std::string& address, const std::string& port) {
{//add vectors
std::vector<RowRecord> record_array;
BuildVectors(0, TOTAL_ROW_COUNT, &record_array, nullptr);
BuildVectors(0, TOTAL_ROW_COUNT, record_array);
std::vector<int64_t> record_ids;
Status stat = conn->AddVector(TABLE_NAME, record_array, record_ids);
std::cout << "AddVector function call status: " << stat.ToString() << std::endl;
@ -215,11 +166,12 @@ ClientTest::Test(const std::string& address, const std::string& port) {
{//search vectors
std::cout << "Waiting data persist. Sleep 10 seconds ..." << std::endl;
sleep(10);
std::vector<QueryRecord> record_array;
BuildVectors(SEARCH_TARGET, SEARCH_TARGET + 10, nullptr, &record_array);
std::vector<RowRecord> record_array;
BuildVectors(SEARCH_TARGET, SEARCH_TARGET + 10, record_array);
std::vector<Range> query_range_array;
std::vector<TopKQueryResult> topk_query_result_array;
Status stat = conn->SearchVector(TABLE_NAME, record_array, topk_query_result_array, TOP_K);
Status stat = conn->SearchVector(TABLE_NAME, record_array, query_range_array, TOP_K, topk_query_result_array);
std::cout << "SearchVector function call status: " << stat.ToString() << std::endl;
PrintSearchResult(topk_query_result_array);
}

146
cpp/src/sdk/include/MegaSearch.h
View File

@ -4,7 +4,6 @@
#include <string>
#include <vector>
#include <map>
#include <memory>
/** \brief MegaSearch SDK namespace
@ -12,129 +11,70 @@
namespace megasearch {
/**
* @brief Column Type
*/
enum class ColumnType {
invalid,
int8,
int16,
int32,
int64,
float32,
float64,
date,
vector
};
/**
* @brief Index Type
*/
enum class IndexType {
raw,
ivfflat
invalid = 0,
cpu_idmap,
gpu_ivfflat,
};
/**
* @brief Connect API parameter
*/
struct ConnectParam {
std::string ip_address; ///< Server IP address
std::string port; ///< Server PORT
};
/**
* @brief Table column description
*/
struct Column {
ColumnType type = ColumnType::invalid; ///< Column Type: enum ColumnType
std::string name; ///< Column name
};
/**
* @brief Table vector column description
*/
struct VectorColumn : public Column {
VectorColumn() { type = ColumnType::vector; }
int64_t dimension = 0; ///< Vector dimension
IndexType index_type = IndexType::raw; ///< Index type
bool store_raw_vector = false; ///< Is vector self stored in the table
std::string ip_address; ///< Server IP address
std::string port; ///< Server PORT
};
/**
* @brief Table Schema
*/
struct TableSchema {
std::string table_name; ///< Table name
std::vector<VectorColumn> vector_column_array; ///< Vector column description
std::vector<Column> attribute_column_array; ///< Columns description
std::vector<std::string> partition_column_name_array; ///< Partition column name
std::string table_name; ///< Table name
IndexType index_type = IndexType::invalid; ///< Index type
int64_t dimension = 0; ///< Vector dimension, must be a positive value
bool store_raw_vector = false; ///< Is vector raw data stored in the table
};
/**
* @brief Range information
* for DATE partition, the format is like: 'year-month-day'
*/
struct Range {
std::string start_value; ///< Range start
std::string end_value; ///< Range stop
};
/**
* @brief Create table partition parameters
*/
struct CreateTablePartitionParam {
std::string table_name; ///< Table name, vector/float32/float64 type column is not allowed for partition
std::string partition_name; ///< Partition name, created partition name
std::map<std::string, Range> range_map; ///< Column name to PartitionRange map
};
/**
* @brief Delete table partition parameters
*/
struct DeleteTablePartitionParam {
std::string table_name; ///< Table name
std::vector<std::string> partition_name_array; ///< Partition name array
std::string start_value; ///< Range start
std::string end_value; ///< Range stop
};
/**
* @brief Record inserted
*/
struct RowRecord {
std::map<std::string, std::vector<float>> vector_map; ///< Vector columns
std::map<std::string, std::string> attribute_map; ///< Other attribute columns
};
/**
* @brief Query record
*/
struct QueryRecord {
std::map<std::string, std::vector<float>> vector_map; ///< Query vectors
std::vector<std::string> selected_column_array; ///< Output column array
std::map<std::string, std::vector<Range>> partition_filter_column_map; ///< Range used to select partitions
std::vector<float> data; ///< Vector raw data
};
/**
* @brief Query result
*/
struct QueryResult {
int64_t id; ///< Output result
double score; ///< Vector similarity score: 0 ~ 100
std::map<std::string, std::string> column_map; ///< Other column
int64_t id; ///< Output result
double score; ///< Vector similarity score: 0 ~ 100
};
/**
* @brief TopK query result
*/
struct TopKQueryResult {
std::vector<QueryResult> query_result_arrays; ///< TopK query result
std::vector<QueryResult> query_result_arrays; ///< TopK query result
};
/**
* @brief SDK main class
*/
class Connection {
public:
public:
/**
* @brief CreateConnection
@ -228,30 +168,6 @@ class Connection {
virtual Status DeleteTable(const std::string &table_name) = 0;
/**
* @brief Create table partition
*
* This method is used to create table partition.
*
* @param param, use to provide partition information to be created.
*
* @return Indicate if table partition is created successfully.
*/
virtual Status CreateTablePartition(const CreateTablePartitionParam &param) = 0;
/**
* @brief Delete table partition
*
* This method is used to delete table partition.
*
* @param param, use to provide partition information to be deleted.
*
* @return Indicate if table partition is delete successfully.
*/
virtual Status DeleteTablePartition(const DeleteTablePartitionParam &param) = 0;
/**
* @brief Add vector to table
*
@ -264,8 +180,8 @@ class Connection {
* @return Indicate if vector array are inserted successfully
*/
virtual Status AddVector(const std::string &table_name,
const std::vector<RowRecord> &record_array,
std::vector<int64_t> &id_array) = 0;
const std::vector<RowRecord> &record_array,
std::vector<int64_t> &id_array) = 0;
/**
@ -275,15 +191,17 @@ class Connection {
*
* @param table_name, table_name is queried.
* @param query_record_array, all vector are going to be queried.
* @param topk_query_result_array, result array.
* @param query_range_array, time ranges, if not specified, will search in whole table
* @param topk, how many similarity vectors will be searched.
* @param topk_query_result_array, result array.
*
* @return Indicate if query is successful.
*/
virtual Status SearchVector(const std::string &table_name,
const std::vector<QueryRecord> &query_record_array,
std::vector<TopKQueryResult> &topk_query_result_array,
int64_t topk) = 0;
const std::vector<RowRecord> &query_record_array,
const std::vector<Range> &query_range_array,
int64_t topk,
std::vector<TopKQueryResult> &topk_query_result_array) = 0;
/**
* @brief Show table description
@ -297,6 +215,18 @@ class Connection {
*/
virtual Status DescribeTable(const std::string &table_name, TableSchema &table_schema) = 0;
/**
* @brief Get table row count
*
* This method is used to get table row count.
*
* @param table_name, table's name.
* @param row_count, table total row count.
*
* @return Indicate if this operation is successful.
*/
virtual Status GetTableRowCount(const std::string &table_name, int64_t &row_count) = 0;
/**
* @brief Show all tables in database
*

15
cpp/src/sdk/include/Status.h
View File

@ -12,10 +12,13 @@ namespace megasearch {
*/
enum class StatusCode {
OK = 0,
Invalid = 1,
UnknownError = 2,
NotSupported = 3,
NotConnected = 4
// system error section
UnknownError = 1,
NotSupported,
NotConnected,
// function error section
InvalidAgument = 1000,
};
/**
@ -171,7 +174,7 @@ class Status {
*/
template<typename... Args>
static Status Invalid(Args &&... args) {
return Status(StatusCode::Invalid,
return Status(StatusCode::InvalidAgument,
MessageBuilder(std::forward<Args>(args)...));
}
@ -221,7 +224,7 @@ class Status {
* @return, if the status indicates invalid.
*
*/
bool IsInvalid() const { return code() == StatusCode::Invalid; }
bool IsInvalid() const { return code() == StatusCode::InvalidAgument; }
/**
* @brief IsUnknownError

151
cpp/src/sdk/src/client/ClientProxy.cpp
View File

@ -89,27 +89,9 @@ ClientProxy::CreateTable(const TableSchema &param) {
try {
thrift::TableSchema schema;
schema.__set_table_name(param.table_name);
std::vector<thrift::VectorColumn> vector_column_array;
for(auto& column : param.vector_column_array) {
thrift::VectorColumn col;
col.__set_dimension(column.dimension);
col.__set_index_type(ConvertUtil::IndexType2Str(column.index_type));
col.__set_store_raw_vector(column.store_raw_vector);
vector_column_array.emplace_back(col);
}
schema.__set_vector_column_array(vector_column_array);
std::vector<thrift::Column> attribute_column_array;
for(auto& column : param.attribute_column_array) {
thrift::Column col;
col.__set_name(col.name);
col.__set_type(col.type);
attribute_column_array.emplace_back(col);
}
schema.__set_attribute_column_array(attribute_column_array);
schema.__set_partition_column_name_array(param.partition_column_name_array);
schema.__set_index_type((int)param.index_type);
schema.__set_dimension(param.dimension);
schema.__set_store_raw_vector(param.store_raw_vector);
ClientPtr()->interface()->CreateTable(schema);
@ -120,55 +102,6 @@ ClientProxy::CreateTable(const TableSchema &param) {
return Status::OK();
}
Status
ClientProxy::CreateTablePartition(const CreateTablePartitionParam &param) {
if(!IsConnected()) {
return Status(StatusCode::NotConnected, "not connected to server");
}
try {
thrift::CreateTablePartitionParam partition_param;
partition_param.__set_table_name(param.table_name);
partition_param.__set_partition_name(param.partition_name);
std::map<std::string, thrift::Range> range_map;
for(auto& pair : param.range_map) {
thrift::Range range;
range.__set_start_value(pair.second.start_value);
range.__set_end_value(pair.second.end_value);
range_map.insert(std::make_pair(pair.first, range));
}
partition_param.__set_range_map(range_map);
ClientPtr()->interface()->CreateTablePartition(partition_param);
} catch ( std::exception& ex) {
return Status(StatusCode::UnknownError, "failed to create table partition: " + std::string(ex.what()));
}
return Status::OK();
}
Status
ClientProxy::DeleteTablePartition(const DeleteTablePartitionParam &param) {
if(!IsConnected()) {
return Status(StatusCode::NotConnected, "not connected to server");
}
try {
thrift::DeleteTablePartitionParam partition_param;
partition_param.__set_table_name(param.table_name);
partition_param.__set_partition_name_array(param.partition_name_array);
ClientPtr()->interface()->DeleteTablePartition(partition_param);
} catch ( std::exception& ex) {
return Status(StatusCode::UnknownError, "failed to delete table partition: " + std::string(ex.what()));
}
return Status::OK();
}
Status
ClientProxy::DeleteTable(const std::string &table_name) {
if(!IsConnected()) {
@ -197,17 +130,13 @@ ClientProxy::AddVector(const std::string &table_name,
std::vector<thrift::RowRecord> thrift_records;
for(auto& record : record_array) {
thrift::RowRecord thrift_record;
thrift_record.__set_attribute_map(record.attribute_map);
for(auto& pair : record.vector_map) {
size_t dim = pair.second.size();
std::string& thrift_vector = thrift_record.vector_map[pair.first];
thrift_vector.resize(dim * sizeof(double));
double *dbl = (double *) (const_cast<char *>(thrift_vector.data()));
for (size_t i = 0; i < dim; i++) {
dbl[i] = (double) (pair.second[i]);
}
thrift_record.vector_data.resize(record.data.size() * sizeof(double));
double *dbl = (double *) (const_cast<char *>(thrift_record.vector_data.data()));
for (size_t i = 0; i < record.data.size(); i++) {
dbl[i] = (double) (record.data[i]);
}
thrift_records.emplace_back(thrift_record);
}
ClientPtr()->interface()->AddVector(id_array, table_name, thrift_records);
@ -221,33 +150,31 @@ ClientProxy::AddVector(const std::string &table_name,
Status
ClientProxy::SearchVector(const std::string &table_name,
const std::vector<QueryRecord> &query_record_array,
std::vector<TopKQueryResult> &topk_query_result_array,
int64_t topk) {
const std::vector<RowRecord> &query_record_array,
const std::vector<Range> &query_range_array,
int64_t topk,
std::vector<TopKQueryResult> &topk_query_result_array) {
if(!IsConnected()) {
return Status(StatusCode::NotConnected, "not connected to server");
}
try {
std::vector<thrift::QueryRecord> thrift_records;
std::vector<thrift::RowRecord> thrift_records;
for(auto& record : query_record_array) {
thrift::QueryRecord thrift_record;
thrift_record.__set_selected_column_array(record.selected_column_array);
thrift::RowRecord thrift_record;
for(auto& pair : record.vector_map) {
size_t dim = pair.second.size();
std::string& thrift_vector = thrift_record.vector_map[pair.first];
thrift_vector.resize(dim * sizeof(double));
double *dbl = (double *) (const_cast<char *>(thrift_vector.data()));
for (size_t i = 0; i < dim; i++) {
dbl[i] = (double) (pair.second[i]);
}
thrift_record.vector_data.resize(record.data.size() * sizeof(double));
double *dbl = (double *) (const_cast<char *>(thrift_record.vector_data.data()));
for (size_t i = 0; i < record.data.size(); i++) {
dbl[i] = (double) (record.data[i]);
}
thrift_records.emplace_back(thrift_record);
}
std::vector<thrift::Range> thrift_ranges;
std::vector<thrift::TopKQueryResult> result_array;
ClientPtr()->interface()->SearchVector(result_array, table_name, thrift_records, topk);
ClientPtr()->interface()->SearchVector(result_array, table_name, thrift_records, thrift_ranges, topk);
for(auto& thrift_topk_result : result_array) {
TopKQueryResult result;
@ -255,7 +182,6 @@ ClientProxy::SearchVector(const std::string &table_name,
for(auto& thrift_query_result : thrift_topk_result.query_result_arrays) {
QueryResult query_result;
query_result.id = thrift_query_result.id;
query_result.column_map = thrift_query_result.column_map;
query_result.score = thrift_query_result.score;
result.query_result_arrays.emplace_back(query_result);
}
@ -281,24 +207,7 @@ ClientProxy::DescribeTable(const std::string &table_name, TableSchema &table_sch
ClientPtr()->interface()->DescribeTable(thrift_schema, table_name);
table_schema.table_name = thrift_schema.table_name;
table_schema.partition_column_name_array = thrift_schema.partition_column_name_array;
for(auto& thrift_col : thrift_schema.attribute_column_array) {
Column col;
col.name = col.name;
col.type = col.type;
table_schema.attribute_column_array.emplace_back(col);
}
for(auto& thrift_col : thrift_schema.vector_column_array) {
VectorColumn col;
col.store_raw_vector = thrift_col.store_raw_vector;
col.index_type = ConvertUtil::Str2IndexType(thrift_col.index_type);
col.dimension = thrift_col.dimension;
col.name = thrift_col.base.name;
col.type = (ColumnType)thrift_col.base.type;
table_schema.vector_column_array.emplace_back(col);
}
table_schema.index_type = (IndexType)thrift_schema.index_type;
} catch ( std::exception& ex) {
return Status(StatusCode::UnknownError, "failed to describe table: " + std::string(ex.what()));
@ -307,6 +216,22 @@ ClientProxy::DescribeTable(const std::string &table_name, TableSchema &table_sch
return Status::OK();
}
Status
ClientProxy::GetTableRowCount(const std::string &table_name, int64_t &row_count) {
if(!IsConnected()) {
return Status(StatusCode::NotConnected, "not connected to server");
}
try {
row_count = ClientPtr()->interface()->GetTableRowCount(table_name);
} catch ( std::exception& ex) {
return Status(StatusCode::UnknownError, "failed to show tables: " + std::string(ex.what()));
}
return Status::OK();
}
Status
ClientProxy::ShowTables(std::vector<std::string> &table_array) {
if(!IsConnected()) {

13
cpp/src/sdk/src/client/ClientProxy.h
View File

@ -25,21 +25,20 @@ public:
virtual Status DeleteTable(const std::string &table_name) override;
virtual Status CreateTablePartition(const CreateTablePartitionParam &param) override;
virtual Status DeleteTablePartition(const DeleteTablePartitionParam &param) override;
virtual Status AddVector(const std::string &table_name,
const std::vector<RowRecord> &record_array,
std::vector<int64_t> &id_array) override;
virtual Status SearchVector(const std::string &table_name,
const std::vector<QueryRecord> &query_record_array,
std::vector<TopKQueryResult> &topk_query_result_array,
int64_t topk) override;
const std::vector<RowRecord> &query_record_array,
const std::vector<Range> &query_range_array,
int64_t topk,
std::vector<TopKQueryResult> &topk_query_result_array) override;
virtual Status DescribeTable(const std::string &table_name, TableSchema &table_schema) override;
virtual Status GetTableRowCount(const std::string &table_name, int64_t &row_count) override;
virtual Status ShowTables(std::vector<std::string> &table_array) override;
virtual std::string ClientVersion() const override;

2
cpp/src/sdk/src/client/ThriftClient.cpp
View File

@ -58,7 +58,7 @@ ThriftClient::Connect(const std::string& address, int32_t port, const std::strin
protocol_ptr.reset(new TCompactProtocol(transport_ptr));
} else {
//CLIENT_LOG_ERROR << "Service protocol: " << protocol << " is not supported currently";
return Status(StatusCode::Invalid, "unsupported protocol");
return Status(StatusCode::InvalidAgument, "unsupported protocol");
}
transport_ptr->open();

24
cpp/src/sdk/src/interface/ConnectionImpl.cpp
View File

@ -55,16 +55,6 @@ ConnectionImpl::CreateTable(const TableSchema &param) {
return client_proxy_->CreateTable(param);
}
Status
ConnectionImpl::CreateTablePartition(const CreateTablePartitionParam &param) {
return client_proxy_->CreateTablePartition(param);
}
Status
ConnectionImpl::DeleteTablePartition(const DeleteTablePartitionParam &param) {
return client_proxy_->DeleteTablePartition(param);
}
Status
ConnectionImpl::DeleteTable(const std::string &table_name) {
return client_proxy_->DeleteTable(table_name);
@ -79,10 +69,11 @@ ConnectionImpl::AddVector(const std::string &table_name,
Status
ConnectionImpl::SearchVector(const std::string &table_name,
const std::vector<QueryRecord> &query_record_array,
std::vector<TopKQueryResult> &topk_query_result_array,
int64_t topk) {
return client_proxy_->SearchVector(table_name, query_record_array, topk_query_result_array, topk);
const std::vector<RowRecord> &query_record_array,
const std::vector<Range> &query_range_array,
int64_t topk,
std::vector<TopKQueryResult> &topk_query_result_array) {
return client_proxy_->SearchVector(table_name, query_record_array, query_range_array, topk, topk_query_result_array);
}
Status
@ -90,6 +81,11 @@ ConnectionImpl::DescribeTable(const std::string &table_name, TableSchema &table_
return client_proxy_->DescribeTable(table_name, table_schema);
}
Status
ConnectionImpl::GetTableRowCount(const std::string &table_name, int64_t &row_count) {
return client_proxy_->GetTableRowCount(table_name, row_count);
}
Status
ConnectionImpl::ShowTables(std::vector<std::string> &table_array) {
return client_proxy_->ShowTables(table_array);

13
cpp/src/sdk/src/interface/ConnectionImpl.h
View File

@ -27,21 +27,20 @@ public:
virtual Status DeleteTable(const std::string &table_name) override;
virtual Status CreateTablePartition(const CreateTablePartitionParam &param) override;
virtual Status DeleteTablePartition(const DeleteTablePartitionParam &param) override;
virtual Status AddVector(const std::string &table_name,
const std::vector<RowRecord> &record_array,
std::vector<int64_t> &id_array) override;
virtual Status SearchVector(const std::string &table_name,
const std::vector<QueryRecord> &query_record_array,
std::vector<TopKQueryResult> &topk_query_result_array,
int64_t topk) override;
const std::vector<RowRecord> &query_record_array,
const std::vector<Range> &query_range_array,
int64_t topk,
std::vector<TopKQueryResult> &topk_query_result_array) override;
virtual Status DescribeTable(const std::string &table_name, TableSchema &table_schema) override;
virtual Status GetTableRowCount(const std::string &table_name, int64_t &row_count) override;
virtual Status ShowTables(std::vector<std::string> &table_array) override;
virtual std::string ClientVersion() const override;

2
cpp/src/sdk/src/interface/Status.cpp
View File

@ -95,7 +95,7 @@ std::string Status::CodeAsString() const {
switch (code()) {
case StatusCode::OK: type = "OK";
break;
case StatusCode::Invalid: type = "Invalid";
case StatusCode::InvalidAgument: type = "Invalid agument";
break;
case StatusCode::UnknownError: type = "Unknown error";
break;

12
cpp/src/sdk/src/util/ConvertUtil.cpp
View File

@ -15,13 +15,13 @@ static const std::string INDEX_IVFFLAT = "ivfflat";
std::string ConvertUtil::IndexType2Str(megasearch::IndexType index) {
static const std::map<megasearch::IndexType, std::string> s_index2str = {
{megasearch::IndexType::raw, INDEX_RAW},
{megasearch::IndexType::ivfflat, INDEX_IVFFLAT}
{megasearch::IndexType::cpu_idmap, INDEX_RAW},
{megasearch::IndexType::gpu_ivfflat, INDEX_IVFFLAT}
};
const auto& iter = s_index2str.find(index);
if(iter == s_index2str.end()) {
throw Exception(StatusCode::Invalid, "Invalid index type");
throw Exception(StatusCode::InvalidAgument, "Invalid index type");
}
return iter->second;
@ -29,13 +29,13 @@ std::string ConvertUtil::IndexType2Str(megasearch::IndexType index) {
megasearch::IndexType ConvertUtil::Str2IndexType(const std::string& type) {
static const std::map<std::string, megasearch::IndexType> s_str2index = {
{INDEX_RAW, megasearch::IndexType::raw},
{INDEX_IVFFLAT, megasearch::IndexType::ivfflat}
{INDEX_RAW, megasearch::IndexType::cpu_idmap},
{INDEX_IVFFLAT, megasearch::IndexType::gpu_ivfflat}
};
const auto& iter = s_str2index.find(type);
if(iter == s_str2index.end()) {
throw Exception(StatusCode::Invalid, "Invalid index type");
throw Exception(StatusCode::InvalidAgument, "Invalid index type");
}
return iter->second;

35
cpp/src/server/MegasearchHandler.cpp
View File

@ -30,18 +30,6 @@ MegasearchServiceHandler::DeleteTable(const std::string &table_name) {
MegasearchScheduler::ExecTask(task_ptr);
}
void
MegasearchServiceHandler::CreateTablePartition(const thrift::CreateTablePartitionParam &param) {
BaseTaskPtr task_ptr = CreateTablePartitionTask::Create(param);
MegasearchScheduler::ExecTask(task_ptr);
}
void
MegasearchServiceHandler::DeleteTablePartition(const thrift::DeleteTablePartitionParam &param) {
BaseTaskPtr task_ptr = DeleteTablePartitionTask::Create(param);
MegasearchScheduler::ExecTask(task_ptr);
}
void
MegasearchServiceHandler::AddVector(std::vector<int64_t> &_return,
const std::string &table_name,
@ -51,11 +39,12 @@ MegasearchServiceHandler::AddVector(std::vector<int64_t> &_return,
}
void
MegasearchServiceHandler::SearchVector(std::vector<thrift::TopKQueryResult> &_return,
const std::string &table_name,
const std::vector<thrift::QueryRecord> &query_record_array,
const int64_t topk) {
BaseTaskPtr task_ptr = SearchVectorTask::Create(table_name, query_record_array, topk, _return);
MegasearchServiceHandler::SearchVector(std::vector<megasearch::thrift::TopKQueryResult> & _return,
const std::string& table_name,
const std::vector<megasearch::thrift::RowRecord> & query_record_array,
const std::vector<megasearch::thrift::Range> & query_range_array,
const int64_t topk) {
BaseTaskPtr task_ptr = SearchVectorTask::Create(table_name, query_record_array, query_range_array, topk, _return);
MegasearchScheduler::ExecTask(task_ptr);
}
@ -65,6 +54,18 @@ MegasearchServiceHandler::DescribeTable(thrift::TableSchema &_return, const std:
MegasearchScheduler::ExecTask(task_ptr);
}
int64_t
MegasearchServiceHandler::GetTableRowCount(const std::string& table_name) {
int64_t row_count = 0;
{
BaseTaskPtr task_ptr = GetTableRowCountTask::Create(table_name, row_count);
MegasearchScheduler::ExecTask(task_ptr);
task_ptr->WaitToFinish();
}
return row_count;
}
void
MegasearchServiceHandler::ShowTables(std::vector<std::string> &_return) {
BaseTaskPtr task_ptr = ShowTablesTask::Create(_return);

66
cpp/src/server/MegasearchHandler.h
View File

@ -19,15 +19,15 @@ public:
MegasearchServiceHandler();
/**
* @brief Create table method
*
* This method is used to create table
*
* @param param, use to provide table information to be created.
*
*
* @param param
*/
* @brief Create table method
*
* This method is used to create table
*
* @param param, use to provide table information to be created.
*
*
* @param param
*/
void CreateTable(const megasearch::thrift::TableSchema& param);
/**
@ -42,30 +42,6 @@ public:
*/
void DeleteTable(const std::string& table_name);
/**
* @brief Create table partition
*
* This method is used to create table partition.
*
* @param param, use to provide partition information to be created.
*
*
* @param param
*/
void CreateTablePartition(const megasearch::thrift::CreateTablePartitionParam& param);
/**
* @brief Delete table partition
*
* This method is used to delete table partition.
*
* @param param, use to provide partition information to be deleted.
*
*
* @param param
*/
void DeleteTablePartition(const megasearch::thrift::DeleteTablePartitionParam& param);
/**
* @brief Add vector array to table
*
@ -90,25 +66,28 @@ public:
*
* @param table_name, table_name is queried.
* @param query_record_array, all vector are going to be queried.
* @param query_range_array, optional ranges for conditional search. If not specified, search whole table
* @param topk, how many similarity vectors will be searched.
*
* @return query result array.
*
* @param table_name
* @param query_record_array
* @param query_range_array
* @param topk
*/
void SearchVector(std::vector<megasearch::thrift::TopKQueryResult> & _return,
const std::string& table_name,
const std::vector<megasearch::thrift::QueryRecord> & query_record_array,
const std::vector<megasearch::thrift::RowRecord> & query_record_array,
const std::vector<megasearch::thrift::Range> & query_range_array,
const int64_t topk);
/**
* @brief Show table information
* @brief Get table schema
*
* This method is used to show table information.
* This method is used to get table schema.
*
* @param table_name, which table is show.
* @param table_name, target table name.
*
* @return table schema
*
@ -116,6 +95,19 @@ public:
*/
void DescribeTable(megasearch::thrift::TableSchema& _return, const std::string& table_name);
/**
* @brief Get table row count
*
* This method is used to get table row count.
*
* @param table_name, target table name.
*
* @return table row count
*
* @param table_name
*/
int64_t GetTableRowCount(const std::string& table_name);
/**
* @brief List all tables in database
*

4
cpp/src/server/MegasearchScheduler.cpp
View File

@ -24,7 +24,7 @@ namespace {
{SERVER_FILE_NOT_FOUND, thrift::ErrorCode::ILLEGAL_ARGUMENT},
{SERVER_NOT_IMPLEMENT, thrift::ErrorCode::ILLEGAL_ARGUMENT},
{SERVER_BLOCKING_QUEUE_EMPTY, thrift::ErrorCode::ILLEGAL_ARGUMENT},
{SERVER_GROUP_NOT_EXIST, thrift::ErrorCode::TABLE_NOT_EXISTS},
{SERVER_TABLE_NOT_EXIST, thrift::ErrorCode::TABLE_NOT_EXISTS},
{SERVER_INVALID_TIME_RANGE, thrift::ErrorCode::ILLEGAL_RANGE},
{SERVER_INVALID_VECTOR_DIMENSION, thrift::ErrorCode::ILLEGAL_DIMENSION},
};
@ -40,7 +40,7 @@ namespace {
{SERVER_FILE_NOT_FOUND, "file not found"},
{SERVER_NOT_IMPLEMENT, "not implemented"},
{SERVER_BLOCKING_QUEUE_EMPTY, "queue empty"},
{SERVER_GROUP_NOT_EXIST, "group not exist"},
{SERVER_TABLE_NOT_EXIST, "table not exist"},
{SERVER_INVALID_TIME_RANGE, "invalid time range"},
{SERVER_INVALID_VECTOR_DIMENSION, "invalid vector dimension"},
};

197
cpp/src/server/MegasearchTask.cpp
View File

@ -5,15 +5,13 @@
******************************************************************************/
#include "MegasearchTask.h"
#include "ServerConfig.h"
#include "VecIdMapper.h"
#include "utils/CommonUtil.h"
#include "utils/Log.h"
#include "utils/TimeRecorder.h"
#include "utils/ThreadPool.h"
#include "db/DB.h"
#include "db/Env.h"
#include "db/Meta.h"
#include "version.h"
namespace zilliz {
namespace vecwise {
@ -64,9 +62,18 @@ namespace {
return db_wrapper.DB();
}
ThreadPool& GetThreadPool() {
static ThreadPool pool(6);
return pool;
engine::EngineType EngineType(int type) {
static std::map<int, engine::EngineType> map_type = {
{0, engine::EngineType::INVALID},
{1, engine::EngineType::FAISS_IDMAP},
{2, engine::EngineType::FAISS_IVFFLAT},
};
if(map_type.find(type) == map_type.end()) {
return engine::EngineType::INVALID;
}
return map_type[type];
}
}
@ -85,16 +92,20 @@ ServerError CreateTableTask::OnExecute() {
TimeRecorder rc("CreateTableTask");
try {
if(schema_.vector_column_array.empty()) {
if(schema_.table_name.empty() || schema_.dimension == 0 || schema_.index_type == 0) {
return SERVER_INVALID_ARGUMENT;
}
IVecIdMapper::GetInstance()->AddGroup(schema_.table_name);
//step 1: construct table schema
engine::meta::TableSchema table_info;
table_info.dimension = (uint16_t)schema_.vector_column_array[0].dimension;
table_info.table_id = schema_.table_name;
table_info.dimension_ = (uint16_t)schema_.dimension;
table_info.table_id_ = schema_.table_name;
table_info.engine_type_ = (int)EngineType(schema_.index_type);
table_info.store_raw_data_ = schema_.store_raw_vector;
//step 2: create table
engine::Status stat = DB()->CreateTable(table_info);
if(!stat.ok()) {//could exist
if(!stat.ok()) {//table could exist
error_msg_ = "Engine failed: " + stat.ToString();
SERVER_LOG_ERROR << error_msg_;
return SERVER_SUCCESS;
@ -129,10 +140,10 @@ ServerError DescribeTableTask::OnExecute() {
try {
engine::meta::TableSchema table_info;
table_info.table_id = table_name_;
table_info.table_id_ = table_name_;
engine::Status stat = DB()->DescribeTable(table_info);
if(!stat.ok()) {
error_code_ = SERVER_GROUP_NOT_EXIST;
error_code_ = SERVER_TABLE_NOT_EXIST;
error_msg_ = "Engine failed: " + stat.ToString();
SERVER_LOG_ERROR << error_msg_;
return error_code_;
@ -168,46 +179,6 @@ ServerError DeleteTableTask::OnExecute() {
error_msg_ = "delete table not implemented";
SERVER_LOG_ERROR << error_msg_;
IVecIdMapper::GetInstance()->DeleteGroup(table_name_);
return SERVER_NOT_IMPLEMENT;
}
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
CreateTablePartitionTask::CreateTablePartitionTask(const thrift::CreateTablePartitionParam &param)
: BaseTask(DDL_DML_TASK_GROUP),
param_(param) {
}
BaseTaskPtr CreateTablePartitionTask::Create(const thrift::CreateTablePartitionParam &param) {
return std::shared_ptr<BaseTask>(new CreateTablePartitionTask(param));
}
ServerError CreateTablePartitionTask::OnExecute() {
error_code_ = SERVER_NOT_IMPLEMENT;
error_msg_ = "create table partition not implemented";
SERVER_LOG_ERROR << error_msg_;
return SERVER_NOT_IMPLEMENT;
}
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
DeleteTablePartitionTask::DeleteTablePartitionTask(const thrift::DeleteTablePartitionParam &param)
: BaseTask(DDL_DML_TASK_GROUP),
param_(param) {
}
BaseTaskPtr DeleteTablePartitionTask::Create(const thrift::DeleteTablePartitionParam &param) {
return std::shared_ptr<BaseTask>(new DeleteTablePartitionTask(param));
}
ServerError DeleteTablePartitionTask::OnExecute() {
error_code_ = SERVER_NOT_IMPLEMENT;
error_msg_ = "delete table partition not implemented";
SERVER_LOG_ERROR << error_msg_;
return SERVER_NOT_IMPLEMENT;
}
@ -223,7 +194,6 @@ BaseTaskPtr ShowTablesTask::Create(std::vector<std::string>& tables) {
}
ServerError ShowTablesTask::OnExecute() {
IVecIdMapper::GetInstance()->AllGroups(tables_);
return SERVER_SUCCESS;
}
@ -253,31 +223,33 @@ ServerError AddVectorTask::OnExecute() {
return SERVER_SUCCESS;
}
//step 1: check table existence
engine::meta::TableSchema table_info;
table_info.table_id = table_name_;
table_info.table_id_ = table_name_;
engine::Status stat = DB()->DescribeTable(table_info);
if(!stat.ok()) {
error_code_ = SERVER_GROUP_NOT_EXIST;
error_code_ = SERVER_TABLE_NOT_EXIST;
error_msg_ = "Engine failed: " + stat.ToString();
SERVER_LOG_ERROR << error_msg_;
return error_code_;
}
rc.Record("get group info");
rc.Record("check validation");
//step 2: prepare float data
uint64_t vec_count = (uint64_t)record_array_.size();
uint64_t group_dim = table_info.dimension;
uint64_t group_dim = table_info.dimension_;
std::vector<float> vec_f;
vec_f.resize(vec_count*group_dim);//allocate enough memory
for(uint64_t i = 0; i < vec_count; i++) {
const auto& record = record_array_[i];
if(record.vector_map.empty()) {
if(record.vector_data.empty()) {
error_code_ = SERVER_INVALID_ARGUMENT;
error_msg_ = "No vector provided in record";
SERVER_LOG_ERROR << error_msg_;
return error_code_;
}
uint64_t vec_dim = record.vector_map.begin()->second.size()/sizeof(double);//how many double value?
uint64_t vec_dim = record.vector_data.size()/sizeof(double);//how many double value?
if(vec_dim != group_dim) {
SERVER_LOG_ERROR << "Invalid vector dimension: " << vec_dim
<< " vs. group dimension:" << group_dim;
@ -287,7 +259,7 @@ ServerError AddVectorTask::OnExecute() {
}
//convert double array to float array(thrift has no float type)
const double* d_p = reinterpret_cast<const double*>(record.vector_map.begin()->second.data());
const double* d_p = reinterpret_cast<const double*>(record.vector_data.data());
for(uint64_t d = 0; d < vec_dim; d++) {
vec_f[i*vec_dim + d] = (float)(d_p[d]);
}
@ -295,6 +267,7 @@ ServerError AddVectorTask::OnExecute() {
rc.Record("prepare vectors data");
//step 3: insert vectors
stat = DB()->InsertVectors(table_name_, vec_count, vec_f.data(), record_ids_);
rc.Record("add vectors to engine");
if(!stat.ok()) {
@ -309,22 +282,8 @@ ServerError AddVectorTask::OnExecute() {
return SERVER_UNEXPECTED_ERROR;
}
//persist attributes
for(uint64_t i = 0; i < vec_count; i++) {
const auto &record = record_array_[i];
//any attributes?
if(record.attribute_map.empty()) {
continue;
}
std::string nid = std::to_string(record_ids_[i]);
std::string attrib_str;
AttributeSerializer::Encode(record.attribute_map, attrib_str);
IVecIdMapper::GetInstance()->Put(nid, attrib_str, table_name_);
}
rc.Record("persist vector attributes");
rc.Record("do insert");
rc.Elapse("totally cost");
} catch (std::exception& ex) {
error_code_ = SERVER_UNEXPECTED_ERROR;
@ -338,28 +297,33 @@ ServerError AddVectorTask::OnExecute() {
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
SearchVectorTask::SearchVectorTask(const std::string& table_name,
const std::vector<thrift::RowRecord> & query_record_array,
const std::vector<megasearch::thrift::Range> & query_range_array,
const int64_t top_k,
const std::vector<thrift::QueryRecord>& record_array,
std::vector<thrift::TopKQueryResult>& result_array)
: BaseTask(DQL_TASK_GROUP),
table_name_(table_name),
record_array_(query_record_array),
range_array_(query_range_array),
top_k_(top_k),
record_array_(record_array),
result_array_(result_array) {
}
BaseTaskPtr SearchVectorTask::Create(const std::string& table_name,
const std::vector<thrift::QueryRecord>& record_array,
const std::vector<thrift::RowRecord> & query_record_array,
const std::vector<megasearch::thrift::Range> & query_range_array,
const int64_t top_k,
std::vector<thrift::TopKQueryResult>& result_array) {
return std::shared_ptr<BaseTask>(new SearchVectorTask(table_name, top_k, record_array, result_array));
return std::shared_ptr<BaseTask>(new SearchVectorTask(table_name,
query_record_array, query_range_array, top_k, result_array));
}
ServerError SearchVectorTask::OnExecute() {
try {
TimeRecorder rc("SearchVectorTask");
//step 1: check validation
if(top_k_ <= 0 || record_array_.empty()) {
error_code_ = SERVER_INVALID_ARGUMENT;
error_msg_ = "Invalid topk value, or query record array is empty";
@ -367,40 +331,44 @@ ServerError SearchVectorTask::OnExecute() {
return error_code_;
}
//step 2: check table existence
engine::meta::TableSchema table_info;
table_info.table_id = table_name_;
table_info.table_id_ = table_name_;
engine::Status stat = DB()->DescribeTable(table_info);
if(!stat.ok()) {
error_code_ = SERVER_GROUP_NOT_EXIST;
error_code_ = SERVER_TABLE_NOT_EXIST;
error_msg_ = "Engine failed: " + stat.ToString();
SERVER_LOG_ERROR << error_msg_;
return error_code_;
}
rc.Record("check validation");
//step 3: prepare float data
std::vector<float> vec_f;
uint64_t record_count = (uint64_t)record_array_.size();
vec_f.resize(record_count*table_info.dimension);
vec_f.resize(record_count*table_info.dimension_);
for(uint64_t i = 0; i < record_array_.size(); i++) {
const auto& record = record_array_[i];
if (record.vector_map.empty()) {
if (record.vector_data.empty()) {
error_code_ = SERVER_INVALID_ARGUMENT;
error_msg_ = "Query record has no vector";
SERVER_LOG_ERROR << error_msg_;
return error_code_;
}
uint64_t vec_dim = record.vector_map.begin()->second.size() / sizeof(double);//how many double value?
if (vec_dim != table_info.dimension) {
uint64_t vec_dim = record.vector_data.size() / sizeof(double);//how many double value?
if (vec_dim != table_info.dimension_) {
SERVER_LOG_ERROR << "Invalid vector dimension: " << vec_dim
<< " vs. group dimension:" << table_info.dimension;
<< " vs. group dimension:" << table_info.dimension_;
error_code_ = SERVER_INVALID_VECTOR_DIMENSION;
error_msg_ = "Engine failed: " + stat.ToString();
return error_code_;
}
//convert double array to float array(thrift has no float type)
const double* d_p = reinterpret_cast<const double*>(record.vector_map.begin()->second.data());
const double* d_p = reinterpret_cast<const double*>(record.vector_data.data());
for(uint64_t d = 0; d < vec_dim; d++) {
vec_f[i*vec_dim + d] = (float)(d_p[d]);
}
@ -408,6 +376,8 @@ ServerError SearchVectorTask::OnExecute() {
rc.Record("prepare vector data");
//step 4: search vectors
std::vector<DB_DATE> dates;
engine::QueryResults results;
stat = DB()->Query(table_name_, (size_t)top_k_, record_count, vec_f.data(), dates, results);
@ -422,7 +392,9 @@ ServerError SearchVectorTask::OnExecute() {
return SERVER_UNEXPECTED_ERROR;
}
//construct result array
rc.Record("do search");
//step 5: construct result array
for(uint64_t i = 0; i < record_count; i++) {
auto& result = results[i];
const auto& record = record_array_[i];
@ -432,23 +404,6 @@ ServerError SearchVectorTask::OnExecute() {
thrift::QueryResult thrift_result;
thrift_result.__set_id(id);
//need get attributes?
if(record.selected_column_array.empty()) {
thrift_topk_result.query_result_arrays.emplace_back(thrift_result);
continue;
}
std::string nid = std::to_string(id);
std::string attrib_str;
IVecIdMapper::GetInstance()->Get(nid, attrib_str, table_name_);
AttribMap attrib_map;
AttributeSerializer::Decode(attrib_str, attrib_map);
for(auto& attribute : record.selected_column_array) {
thrift_result.column_map[attribute] = attrib_map[attribute];
}
thrift_topk_result.query_result_arrays.emplace_back(thrift_result);
}
@ -466,6 +421,32 @@ ServerError SearchVectorTask::OnExecute() {
return SERVER_SUCCESS;
}
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
GetTableRowCountTask::GetTableRowCountTask(const std::string& table_name, int64_t& row_count)
: BaseTask(DQL_TASK_GROUP),
table_name_(table_name),
row_count_(row_count) {
}
BaseTaskPtr GetTableRowCountTask::Create(const std::string& table_name, int64_t& row_count) {
return std::shared_ptr<BaseTask>(new GetTableRowCountTask(table_name, row_count));
}
ServerError GetTableRowCountTask::OnExecute() {
if(table_name_.empty()) {
error_code_ = SERVER_UNEXPECTED_ERROR;
error_msg_ = "Table name cannot be empty";
SERVER_LOG_ERROR << error_msg_;
return error_code_;
}
error_code_ = SERVER_NOT_IMPLEMENT;
error_msg_ = "Not implemented";
SERVER_LOG_ERROR << error_msg_;
return error_code_;
}
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
PingTask::PingTask(const std::string& cmd, std::string& result)
: BaseTask(PING_TASK_GROUP),
@ -480,7 +461,7 @@ BaseTaskPtr PingTask::Create(const std::string& cmd, std::string& result) {
ServerError PingTask::OnExecute() {
if(cmd_ == "version") {
result_ = "v1.2.0";//currently hardcode
result_ = MEGASEARCH_VERSION;
}
return SERVER_SUCCESS;

54
cpp/src/server/MegasearchTask.h
View File

@ -65,36 +65,6 @@ private:
std::string table_name_;
};
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
class CreateTablePartitionTask : public BaseTask {
public:
static BaseTaskPtr Create(const thrift::CreateTablePartitionParam &param);
protected:
CreateTablePartitionTask(const thrift::CreateTablePartitionParam &param);
ServerError OnExecute() override;
private:
const thrift::CreateTablePartitionParam &param_;
};
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
class DeleteTablePartitionTask : public BaseTask {
public:
static BaseTaskPtr Create(const thrift::DeleteTablePartitionParam &param);
protected:
DeleteTablePartitionTask(const thrift::DeleteTablePartitionParam &param);
ServerError OnExecute() override;
private:
const thrift::DeleteTablePartitionParam &param_;
};
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
class ShowTablesTask : public BaseTask {
public:
@ -133,14 +103,16 @@ private:
class SearchVectorTask : public BaseTask {
public:
static BaseTaskPtr Create(const std::string& table_name,
const std::vector<thrift::QueryRecord>& record_array,
const std::vector<thrift::RowRecord> & query_record_array,
const std::vector<megasearch::thrift::Range> & query_range_array,
const int64_t top_k,
std::vector<thrift::TopKQueryResult>& result_array);
protected:
SearchVectorTask(const std::string& table_name,
const std::vector<thrift::RowRecord> & query_record_array,
const std::vector<megasearch::thrift::Range> & query_range_array,
const int64_t top_k,
const std::vector<thrift::QueryRecord>& record_array,
std::vector<thrift::TopKQueryResult>& result_array);
ServerError OnExecute() override;
@ -148,10 +120,26 @@ protected:
private:
std::string table_name_;
int64_t top_k_;
const std::vector<thrift::QueryRecord>& record_array_;
const std::vector<thrift::RowRecord>& record_array_;
const std::vector<megasearch::thrift::Range>& range_array_;
std::vector<thrift::TopKQueryResult>& result_array_;
};
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
class GetTableRowCountTask : public BaseTask {
public:
static BaseTaskPtr Create(const std::string& table_name, int64_t& row_count);
protected:
GetTableRowCountTask(const std::string& table_name, int64_t& row_count);
ServerError OnExecute() override;
private:
std::string table_name_;
int64_t& row_count_;
};
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
class PingTask : public BaseTask {
public:

327
cpp/src/server/RocksIdMapper.cpp
View File

@ -1,327 +0,0 @@
/*******************************************************************************
* Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved
* Unauthorized copying of this file, via any medium is strictly prohibited.
* Proprietary and confidential.
******************************************************************************/
#include "RocksIdMapper.h"
#include "ServerConfig.h"
#include "utils/Log.h"
#include "utils/CommonUtil.h"
#include "rocksdb/db.h"
#include "rocksdb/slice.h"
#include "rocksdb/options.h"
#include <exception>
namespace zilliz {
namespace vecwise {
namespace server {
static const std::string ROCKSDB_DEFAULT_GROUP = "default";
RocksIdMapper::RocksIdMapper()
: db_(nullptr) {
OpenDb();
}
RocksIdMapper::~RocksIdMapper() {
CloseDb();
}
void RocksIdMapper::OpenDb() {
std::lock_guard<std::mutex> lck(db_mutex_);
if(db_) {
return;
}
ConfigNode& config = ServerConfig::GetInstance().GetConfig(CONFIG_DB);
std::string db_path = config.GetValue(CONFIG_DB_PATH);
db_path += "/id_mapping";
CommonUtil::CreateDirectory(db_path);
rocksdb::Options options;
// Optimize RocksDB. This is the easiest way to get RocksDB to perform well
options.IncreaseParallelism();
options.OptimizeLevelStyleCompaction();
// create the DB if it's not already present
options.create_if_missing = true;
options.max_open_files = config.GetInt32Value(CONFIG_DB_IDMAPPER_MAX_FILE, 512);
//load column families
std::vector<std::string> column_names;
rocksdb::Status s = rocksdb::DB::ListColumnFamilies(options, db_path, &column_names);
if (!s.ok()) {
SERVER_LOG_ERROR << "ID mapper failed to initialize:" << s.ToString();
}
if(column_names.empty()) {
column_names.push_back("default");
}
SERVER_LOG_INFO << "ID mapper has " << std::to_string(column_names.size()) << " groups";
std::vector<rocksdb::ColumnFamilyDescriptor> column_families;
for(auto& column_name : column_names) {
rocksdb::ColumnFamilyDescriptor desc;
desc.name = column_name;
column_families.emplace_back(desc);
}
// open DB
std::vector<rocksdb::ColumnFamilyHandle*> column_handles;
s = rocksdb::DB::Open(options, db_path, column_families, &column_handles, &db_);
if(!s.ok()) {
SERVER_LOG_ERROR << "ID mapper failed to initialize:" << s.ToString();
db_ = nullptr;
}
column_handles_.clear();
for(auto handler : column_handles) {
column_handles_.insert(std::make_pair(handler->GetName(), handler));
}
}
void RocksIdMapper::CloseDb() {
std::lock_guard<std::mutex> lck(db_mutex_);
for(auto& iter : column_handles_) {
delete iter.second;
}
column_handles_.clear();
if(db_) {
db_->Close();
delete db_;
}
}
ServerError RocksIdMapper::AddGroup(const std::string& group) {
std::lock_guard<std::mutex> lck(db_mutex_);
return AddGroupInternal(group);
}
bool RocksIdMapper::IsGroupExist(const std::string& group) const {
std::lock_guard<std::mutex> lck(db_mutex_);
return IsGroupExistInternal(group);
}
ServerError RocksIdMapper::AllGroups(std::vector<std::string>& groups) const {
groups.clear();
std::lock_guard<std::mutex> lck(db_mutex_);
for(auto& pair : column_handles_) {
if(pair.first == ROCKSDB_DEFAULT_GROUP) {
continue;
}
groups.push_back(pair.first);
}
return SERVER_SUCCESS;
}
ServerError RocksIdMapper::Put(const std::string& nid, const std::string& sid, const std::string& group) {
std::lock_guard<std::mutex> lck(db_mutex_);
return PutInternal(nid, sid, group);
}
ServerError RocksIdMapper::Put(const std::vector<std::string>& nid, const std::vector<std::string>& sid, const std::string& group) {
if(nid.size() != sid.size()) {
return SERVER_INVALID_ARGUMENT;
}
std::lock_guard<std::mutex> lck(db_mutex_);
ServerError err = SERVER_SUCCESS;
for(size_t i = 0; i < nid.size(); i++) {
err = PutInternal(nid[i], sid[i], group);
if(err != SERVER_SUCCESS) {
return err;
}
}
return err;
}
ServerError RocksIdMapper::Get(const std::string& nid, std::string& sid, const std::string& group) const {
std::lock_guard<std::mutex> lck(db_mutex_);
return GetInternal(nid, sid, group);
}
ServerError RocksIdMapper::Get(const std::vector<std::string>& nid, std::vector<std::string>& sid, const std::string& group) const {
sid.clear();
std::lock_guard<std::mutex> lck(db_mutex_);
ServerError err = SERVER_SUCCESS;
for(size_t i = 0; i < nid.size(); i++) {
std::string str_id;
ServerError temp_err = GetInternal(nid[i], str_id, group);
if(temp_err != SERVER_SUCCESS) {
sid.push_back("");
SERVER_LOG_ERROR << "ID mapper failed to get id: " << nid[i];
err = temp_err;
continue;
}
sid.push_back(str_id);
}
return err;
}
ServerError RocksIdMapper::Delete(const std::string& nid, const std::string& group) {
std::lock_guard<std::mutex> lck(db_mutex_);
return DeleteInternal(nid, group);
}
ServerError RocksIdMapper::DeleteGroup(const std::string& group) {
std::lock_guard<std::mutex> lck(db_mutex_);
return DeleteGroupInternal(group);
}
//internal methods(whitout lock)
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
ServerError RocksIdMapper::AddGroupInternal(const std::string& group) {
if(!IsGroupExistInternal(group)) {
if(db_ == nullptr) {
return SERVER_NULL_POINTER;
}
try {//add group
rocksdb::ColumnFamilyHandle *cfh = nullptr;
rocksdb::Status s = db_->CreateColumnFamily(rocksdb::ColumnFamilyOptions(), group, &cfh);
if (!s.ok()) {
SERVER_LOG_ERROR << "ID mapper failed to create group:" << s.ToString();
return SERVER_UNEXPECTED_ERROR;
} else {
column_handles_.insert(std::make_pair(group, cfh));
}
} catch(std::exception& ex) {
SERVER_LOG_ERROR << "ID mapper failed to create group: " << ex.what();
return SERVER_UNEXPECTED_ERROR;
}
}
return SERVER_SUCCESS;
}
bool RocksIdMapper::IsGroupExistInternal(const std::string& group) const {
std::string group_name = group;
if(group_name.empty()){
group_name = ROCKSDB_DEFAULT_GROUP;
}
return (column_handles_.count(group_name) > 0 && column_handles_[group_name] != nullptr);
}
ServerError RocksIdMapper::PutInternal(const std::string& nid, const std::string& sid, const std::string& group) {
if(db_ == nullptr) {
return SERVER_NULL_POINTER;
}
rocksdb::Slice key(nid);
rocksdb::Slice value(sid);
if(group.empty()) {//to default group
rocksdb::Status s = db_->Put(rocksdb::WriteOptions(), key, value);
if (!s.ok()) {
SERVER_LOG_ERROR << "ID mapper failed to put:" << s.ToString();
return SERVER_UNEXPECTED_ERROR;
}
} else {
//try create group
if(AddGroupInternal(group) != SERVER_SUCCESS){
return SERVER_UNEXPECTED_ERROR;
}
rocksdb::ColumnFamilyHandle *cfh = column_handles_[group];
rocksdb::Status s = db_->Put(rocksdb::WriteOptions(), cfh, key, value);
if (!s.ok()) {
SERVER_LOG_ERROR << "ID mapper failed to put:" << s.ToString();
return SERVER_UNEXPECTED_ERROR;
}
}
return SERVER_SUCCESS;
}
ServerError RocksIdMapper::GetInternal(const std::string& nid, std::string& sid, const std::string& group) const {
sid = "";
if(db_ == nullptr) {
return SERVER_NULL_POINTER;
}
rocksdb::ColumnFamilyHandle *cfh = nullptr;
if(column_handles_.count(group) != 0) {
cfh = column_handles_.at(group);
}
rocksdb::Slice key(nid);
rocksdb::Status s;
if(cfh){
s = db_->Get(rocksdb::ReadOptions(), cfh, key, &sid);
} else {
s = db_->Get(rocksdb::ReadOptions(), key, &sid);
}
if(!s.ok()) {
SERVER_LOG_ERROR << "ID mapper failed to get:" << s.ToString();
return SERVER_UNEXPECTED_ERROR;
}
return SERVER_SUCCESS;
}
ServerError RocksIdMapper::DeleteInternal(const std::string& nid, const std::string& group) {
if(db_ == nullptr) {
return SERVER_NULL_POINTER;
}
rocksdb::ColumnFamilyHandle *cfh = nullptr;
if(column_handles_.count(group) != 0) {
cfh = column_handles_.at(group);
}
rocksdb::Slice key(nid);
rocksdb::Status s;
if(cfh){
s = db_->Delete(rocksdb::WriteOptions(), cfh, key);
} else {
s = db_->Delete(rocksdb::WriteOptions(), key);
}
if(!s.ok()) {
SERVER_LOG_ERROR << "ID mapper failed to delete:" << s.ToString();
return SERVER_UNEXPECTED_ERROR;
}
return SERVER_SUCCESS;
}
ServerError RocksIdMapper::DeleteGroupInternal(const std::string& group) {
if(db_ == nullptr) {
return SERVER_NULL_POINTER;
}
rocksdb::ColumnFamilyHandle *cfh = nullptr;
if(column_handles_.count(group) != 0) {
cfh = column_handles_.at(group);
}
if(cfh) {
db_->DropColumnFamily(cfh);
db_->DestroyColumnFamilyHandle(cfh);
column_handles_.erase(group);
}
return SERVER_SUCCESS;
}
}
}
}

68
cpp/src/server/RocksIdMapper.h
View File

@ -1,68 +0,0 @@
/*******************************************************************************
* Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved
* Unauthorized copying of this file, via any medium is strictly prohibited.
* Proprietary and confidential.
******************************************************************************/
#pragma once
#include "utils/Error.h"
#include "VecIdMapper.h"
#include <string>
#include <vector>
#include <unordered_map>
#include <mutex>
namespace rocksdb {
class DB;
class ColumnFamilyHandle;
}
namespace zilliz {
namespace vecwise {
namespace server {
class RocksIdMapper : public IVecIdMapper{
public:
RocksIdMapper();
~RocksIdMapper();
ServerError AddGroup(const std::string& group) override;
bool IsGroupExist(const std::string& group) const override;
ServerError AllGroups(std::vector<std::string>& groups) const override;
ServerError Put(const std::string& nid, const std::string& sid, const std::string& group = "") override;
ServerError Put(const std::vector<std::string>& nid, const std::vector<std::string>& sid, const std::string& group = "") override;
ServerError Get(const std::string& nid, std::string& sid, const std::string& group = "") const override;
ServerError Get(const std::vector<std::string>& nid, std::vector<std::string>& sid, const std::string& group = "") const override;
ServerError Delete(const std::string& nid, const std::string& group = "") override;
ServerError DeleteGroup(const std::string& group) override;
private:
void OpenDb();
void CloseDb();
ServerError AddGroupInternal(const std::string& group);
bool IsGroupExistInternal(const std::string& group) const;
ServerError PutInternal(const std::string& nid, const std::string& sid, const std::string& group);
ServerError GetInternal(const std::string& nid, std::string& sid, const std::string& group) const;
ServerError DeleteInternal(const std::string& nid, const std::string& group);
ServerError DeleteGroupInternal(const std::string& group);
private:
rocksdb::DB* db_;
mutable std::unordered_map<std::string, rocksdb::ColumnFamilyHandle*> column_handles_;
mutable std::mutex db_mutex_;
};
}
}
}

162
cpp/src/server/VecIdMapper.cpp
View File

@ -1,162 +0,0 @@
/*******************************************************************************
* Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved
* Unauthorized copying of this file, via any medium is strictly prohibited.
* Proprietary and confidential.
******************************************************************************/
#include "VecIdMapper.h"
#include "RocksIdMapper.h"
#include "ServerConfig.h"
#include "utils/Log.h"
#include "utils/CommonUtil.h"
#include "rocksdb/db.h"
#include "rocksdb/slice.h"
#include "rocksdb/options.h"
#include <exception>
#include <unordered_map>
namespace zilliz {
namespace vecwise {
namespace server {
IVecIdMapper* IVecIdMapper::GetInstance() {
#if 0
static SimpleIdMapper s_mapper;
return &s_mapper;
#else
static RocksIdMapper s_mapper;
return &s_mapper;
#endif
}
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
class SimpleIdMapper : public IVecIdMapper{
public:
SimpleIdMapper();
~SimpleIdMapper();
ServerError AddGroup(const std::string& group) override;
bool IsGroupExist(const std::string& group) const override;
ServerError AllGroups(std::vector<std::string>& groups) const override;
ServerError Put(const std::string& nid, const std::string& sid, const std::string& group = "") override;
ServerError Put(const std::vector<std::string>& nid, const std::vector<std::string>& sid, const std::string& group = "") override;
ServerError Get(const std::string& nid, std::string& sid, const std::string& group = "") const override;
ServerError Get(const std::vector<std::string>& nid, std::vector<std::string>& sid, const std::string& group = "") const override;
ServerError Delete(const std::string& nid, const std::string& group = "") override;
ServerError DeleteGroup(const std::string& group) override;
private:
using ID_MAPPING = std::unordered_map<std::string, std::string>;
mutable std::unordered_map<std::string, ID_MAPPING> id_groups_;
};
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
SimpleIdMapper::SimpleIdMapper() {
}
SimpleIdMapper::~SimpleIdMapper() {
}
ServerError
SimpleIdMapper::AddGroup(const std::string& group) {
if(id_groups_.count(group) == 0) {
id_groups_.insert(std::make_pair(group, ID_MAPPING()));
}
}
//not thread-safe
bool
SimpleIdMapper::IsGroupExist(const std::string& group) const {
return id_groups_.count(group) > 0;
}
ServerError SimpleIdMapper::AllGroups(std::vector<std::string>& groups) const {
groups.clear();
for(auto& pair : id_groups_) {
groups.push_back(pair.first);
}
return SERVER_SUCCESS;
}
//not thread-safe
ServerError SimpleIdMapper::Put(const std::string& nid, const std::string& sid, const std::string& group) {
ID_MAPPING& mapping = id_groups_[group];
mapping[nid] = sid;
return SERVER_SUCCESS;
}
//not thread-safe
ServerError SimpleIdMapper::Put(const std::vector<std::string>& nid, const std::vector<std::string>& sid, const std::string& group) {
if(nid.size() != sid.size()) {
return SERVER_INVALID_ARGUMENT;
}
ID_MAPPING& mapping = id_groups_[group];
for(size_t i = 0; i < nid.size(); i++) {
mapping[nid[i]] = sid[i];
}
return SERVER_SUCCESS;
}
//not thread-safe
ServerError SimpleIdMapper::Get(const std::string& nid, std::string& sid, const std::string& group) const {
ID_MAPPING& mapping = id_groups_[group];
auto iter = mapping.find(nid);
if(iter == mapping.end()) {
return SERVER_INVALID_ARGUMENT;
}
sid = iter->second;
return SERVER_SUCCESS;
}
//not thread-safe
ServerError SimpleIdMapper::Get(const std::vector<std::string>& nid, std::vector<std::string>& sid, const std::string& group) const {
sid.clear();
ID_MAPPING& mapping = id_groups_[group];
ServerError err = SERVER_SUCCESS;
for(size_t i = 0; i < nid.size(); i++) {
auto iter = mapping.find(nid[i]);
if(iter == mapping.end()) {
sid.push_back("");
SERVER_LOG_ERROR << "ID mapper failed to find id: " << nid[i];
err = SERVER_INVALID_ARGUMENT;
continue;
}
sid.push_back(iter->second);
}
return err;
}
//not thread-safe
ServerError SimpleIdMapper::Delete(const std::string& nid, const std::string& group) {
ID_MAPPING& mapping = id_groups_[group];
mapping.erase(nid);
return SERVER_SUCCESS;
}
//not thread-safe
ServerError SimpleIdMapper::DeleteGroup(const std::string& group) {
id_groups_.erase(group);
return SERVER_SUCCESS;
}
}
}
}

40
cpp/src/server/VecIdMapper.h
View File

@ -1,40 +0,0 @@
/*******************************************************************************
* Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved
* Unauthorized copying of this file, via any medium is strictly prohibited.
* Proprietary and confidential.
******************************************************************************/
#pragma once
#include "utils/Error.h"
#include <string>
#include <vector>
namespace zilliz {
namespace vecwise {
namespace server {
class IVecIdMapper {
public:
static IVecIdMapper* GetInstance();
virtual ~IVecIdMapper(){}
virtual ServerError AddGroup(const std::string& group) = 0;
virtual bool IsGroupExist(const std::string& group) const = 0;
virtual ServerError AllGroups(std::vector<std::string>& groups) const = 0;
virtual ServerError Put(const std::string& nid, const std::string& sid, const std::string& group = "") = 0;
virtual ServerError Put(const std::vector<std::string>& nid, const std::vector<std::string>& sid, const std::string& group = "") = 0;
virtual ServerError Get(const std::string& nid, std::string& sid, const std::string& group = "") const = 0;
//NOTE: the 'sid' will be cleared at begin of the function
virtual ServerError Get(const std::vector<std::string>& nid, std::vector<std::string>& sid, const std::string& group = "") const = 0;
virtual ServerError Delete(const std::string& nid, const std::string& group = "") = 0;
virtual ServerError DeleteGroup(const std::string& group) = 0;
};
}
}
}

1393
cpp/src/thrift/gen-cpp/MegasearchService.cpp
View File

File diff suppressed because it is too large Load Diff

459
cpp/src/thrift/gen-cpp/MegasearchService.h
View File

@ -46,30 +46,6 @@ class MegasearchServiceIf {
*/
virtual void DeleteTable(const std::string& table_name) = 0;
/**
* @brief Create table partition
*
* This method is used to create table partition.
*
* @param param, use to provide partition information to be created.
*
*
* @param param
*/
virtual void CreateTablePartition(const CreateTablePartitionParam& param) = 0;
/**
* @brief Delete table partition
*
* This method is used to delete table partition.
*
* @param param, use to provide partition information to be deleted.
*
*
* @param param
*/
virtual void DeleteTablePartition(const DeleteTablePartitionParam& param) = 0;
/**
* @brief Add vector array to table
*
@ -92,22 +68,24 @@ class MegasearchServiceIf {
*
* @param table_name, table_name is queried.
* @param query_record_array, all vector are going to be queried.
* @param query_range_array, optional ranges for conditional search. If not specified, search whole table
* @param topk, how many similarity vectors will be searched.
*
* @return query result array.
*
* @param table_name
* @param query_record_array
* @param query_range_array
* @param topk
*/
virtual void SearchVector(std::vector<TopKQueryResult> & _return, const std::string& table_name, const std::vector<QueryRecord> & query_record_array, const int64_t topk) = 0;
virtual void SearchVector(std::vector<TopKQueryResult> & _return, const std::string& table_name, const std::vector<RowRecord> & query_record_array, const std::vector<Range> & query_range_array, const int64_t topk) = 0;
/**
* @brief Show table information
* @brief Get table schema
*
* This method is used to show table information.
* This method is used to get table schema.
*
* @param table_name, which table is show.
* @param table_name, target table name.
*
* @return table schema
*
@ -115,6 +93,19 @@ class MegasearchServiceIf {
*/
virtual void DescribeTable(TableSchema& _return, const std::string& table_name) = 0;
/**
* @brief Get table row count
*
* This method is used to get table row count.
*
* @param table_name, target table name.
*
* @return table row count
*
* @param table_name
*/
virtual int64_t GetTableRowCount(const std::string& table_name) = 0;
/**
* @brief List all tables in database
*
@ -170,21 +161,19 @@ class MegasearchServiceNull : virtual public MegasearchServiceIf {
void DeleteTable(const std::string& /* table_name */) {
return;
}
void CreateTablePartition(const CreateTablePartitionParam& /* param */) {
return;
}
void DeleteTablePartition(const DeleteTablePartitionParam& /* param */) {
return;
}
void AddVector(std::vector<int64_t> & /* _return */, const std::string& /* table_name */, const std::vector<RowRecord> & /* record_array */) {
return;
}
void SearchVector(std::vector<TopKQueryResult> & /* _return */, const std::string& /* table_name */, const std::vector<QueryRecord> & /* query_record_array */, const int64_t /* topk */) {
void SearchVector(std::vector<TopKQueryResult> & /* _return */, const std::string& /* table_name */, const std::vector<RowRecord> & /* query_record_array */, const std::vector<Range> & /* query_range_array */, const int64_t /* topk */) {
return;
}
void DescribeTable(TableSchema& /* _return */, const std::string& /* table_name */) {
return;
}
int64_t GetTableRowCount(const std::string& /* table_name */) {
int64_t _return = 0;
return _return;
}
void ShowTables(std::vector<std::string> & /* _return */) {
return;
}
@ -401,214 +390,6 @@ class MegasearchService_DeleteTable_presult {
};
typedef struct _MegasearchService_CreateTablePartition_args__isset {
_MegasearchService_CreateTablePartition_args__isset() : param(false) {}
bool param :1;
} _MegasearchService_CreateTablePartition_args__isset;
class MegasearchService_CreateTablePartition_args {
public:
MegasearchService_CreateTablePartition_args(const MegasearchService_CreateTablePartition_args&);
MegasearchService_CreateTablePartition_args& operator=(const MegasearchService_CreateTablePartition_args&);
MegasearchService_CreateTablePartition_args() {
}
virtual ~MegasearchService_CreateTablePartition_args() throw();
CreateTablePartitionParam param;
_MegasearchService_CreateTablePartition_args__isset __isset;
void __set_param(const CreateTablePartitionParam& val);
bool operator == (const MegasearchService_CreateTablePartition_args & rhs) const
{
if (!(param == rhs.param))
return false;
return true;
}
bool operator != (const MegasearchService_CreateTablePartition_args &rhs) const {
return !(*this == rhs);
}
bool operator < (const MegasearchService_CreateTablePartition_args & ) const;
uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
};
class MegasearchService_CreateTablePartition_pargs {
public:
virtual ~MegasearchService_CreateTablePartition_pargs() throw();
const CreateTablePartitionParam* param;
uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
};
typedef struct _MegasearchService_CreateTablePartition_result__isset {
_MegasearchService_CreateTablePartition_result__isset() : e(false) {}
bool e :1;
} _MegasearchService_CreateTablePartition_result__isset;
class MegasearchService_CreateTablePartition_result {
public:
MegasearchService_CreateTablePartition_result(const MegasearchService_CreateTablePartition_result&);
MegasearchService_CreateTablePartition_result& operator=(const MegasearchService_CreateTablePartition_result&);
MegasearchService_CreateTablePartition_result() {
}
virtual ~MegasearchService_CreateTablePartition_result() throw();
Exception e;
_MegasearchService_CreateTablePartition_result__isset __isset;
void __set_e(const Exception& val);
bool operator == (const MegasearchService_CreateTablePartition_result & rhs) const
{
if (!(e == rhs.e))
return false;
return true;
}
bool operator != (const MegasearchService_CreateTablePartition_result &rhs) const {
return !(*this == rhs);
}
bool operator < (const MegasearchService_CreateTablePartition_result & ) const;
uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
};
typedef struct _MegasearchService_CreateTablePartition_presult__isset {
_MegasearchService_CreateTablePartition_presult__isset() : e(false) {}
bool e :1;
} _MegasearchService_CreateTablePartition_presult__isset;
class MegasearchService_CreateTablePartition_presult {
public:
virtual ~MegasearchService_CreateTablePartition_presult() throw();
Exception e;
_MegasearchService_CreateTablePartition_presult__isset __isset;
uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
};
typedef struct _MegasearchService_DeleteTablePartition_args__isset {
_MegasearchService_DeleteTablePartition_args__isset() : param(false) {}
bool param :1;
} _MegasearchService_DeleteTablePartition_args__isset;
class MegasearchService_DeleteTablePartition_args {
public:
MegasearchService_DeleteTablePartition_args(const MegasearchService_DeleteTablePartition_args&);
MegasearchService_DeleteTablePartition_args& operator=(const MegasearchService_DeleteTablePartition_args&);
MegasearchService_DeleteTablePartition_args() {
}
virtual ~MegasearchService_DeleteTablePartition_args() throw();
DeleteTablePartitionParam param;
_MegasearchService_DeleteTablePartition_args__isset __isset;
void __set_param(const DeleteTablePartitionParam& val);
bool operator == (const MegasearchService_DeleteTablePartition_args & rhs) const
{
if (!(param == rhs.param))
return false;
return true;
}
bool operator != (const MegasearchService_DeleteTablePartition_args &rhs) const {
return !(*this == rhs);
}
bool operator < (const MegasearchService_DeleteTablePartition_args & ) const;
uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
};
class MegasearchService_DeleteTablePartition_pargs {
public:
virtual ~MegasearchService_DeleteTablePartition_pargs() throw();
const DeleteTablePartitionParam* param;
uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
};
typedef struct _MegasearchService_DeleteTablePartition_result__isset {
_MegasearchService_DeleteTablePartition_result__isset() : e(false) {}
bool e :1;
} _MegasearchService_DeleteTablePartition_result__isset;
class MegasearchService_DeleteTablePartition_result {
public:
MegasearchService_DeleteTablePartition_result(const MegasearchService_DeleteTablePartition_result&);
MegasearchService_DeleteTablePartition_result& operator=(const MegasearchService_DeleteTablePartition_result&);
MegasearchService_DeleteTablePartition_result() {
}
virtual ~MegasearchService_DeleteTablePartition_result() throw();
Exception e;
_MegasearchService_DeleteTablePartition_result__isset __isset;
void __set_e(const Exception& val);
bool operator == (const MegasearchService_DeleteTablePartition_result & rhs) const
{
if (!(e == rhs.e))
return false;
return true;
}
bool operator != (const MegasearchService_DeleteTablePartition_result &rhs) const {
return !(*this == rhs);
}
bool operator < (const MegasearchService_DeleteTablePartition_result & ) const;
uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
};
typedef struct _MegasearchService_DeleteTablePartition_presult__isset {
_MegasearchService_DeleteTablePartition_presult__isset() : e(false) {}
bool e :1;
} _MegasearchService_DeleteTablePartition_presult__isset;
class MegasearchService_DeleteTablePartition_presult {
public:
virtual ~MegasearchService_DeleteTablePartition_presult() throw();
Exception e;
_MegasearchService_DeleteTablePartition_presult__isset __isset;
uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
};
typedef struct _MegasearchService_AddVector_args__isset {
_MegasearchService_AddVector_args__isset() : table_name(false), record_array(false) {}
bool table_name :1;
@ -729,9 +510,10 @@ class MegasearchService_AddVector_presult {
};
typedef struct _MegasearchService_SearchVector_args__isset {
_MegasearchService_SearchVector_args__isset() : table_name(false), query_record_array(false), topk(false) {}
_MegasearchService_SearchVector_args__isset() : table_name(false), query_record_array(false), query_range_array(false), topk(false) {}
bool table_name :1;
bool query_record_array :1;
bool query_range_array :1;
bool topk :1;
} _MegasearchService_SearchVector_args__isset;
@ -745,14 +527,17 @@ class MegasearchService_SearchVector_args {
virtual ~MegasearchService_SearchVector_args() throw();
std::string table_name;
std::vector<QueryRecord> query_record_array;
std::vector<RowRecord> query_record_array;
std::vector<Range> query_range_array;
int64_t topk;
_MegasearchService_SearchVector_args__isset __isset;
void __set_table_name(const std::string& val);
void __set_query_record_array(const std::vector<QueryRecord> & val);
void __set_query_record_array(const std::vector<RowRecord> & val);
void __set_query_range_array(const std::vector<Range> & val);
void __set_topk(const int64_t val);
@ -762,6 +547,8 @@ class MegasearchService_SearchVector_args {
return false;
if (!(query_record_array == rhs.query_record_array))
return false;
if (!(query_range_array == rhs.query_range_array))
return false;
if (!(topk == rhs.topk))
return false;
return true;
@ -784,7 +571,8 @@ class MegasearchService_SearchVector_pargs {
virtual ~MegasearchService_SearchVector_pargs() throw();
const std::string* table_name;
const std::vector<QueryRecord> * query_record_array;
const std::vector<RowRecord> * query_record_array;
const std::vector<Range> * query_range_array;
const int64_t* topk;
uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
@ -966,6 +754,118 @@ class MegasearchService_DescribeTable_presult {
};
typedef struct _MegasearchService_GetTableRowCount_args__isset {
_MegasearchService_GetTableRowCount_args__isset() : table_name(false) {}
bool table_name :1;
} _MegasearchService_GetTableRowCount_args__isset;
class MegasearchService_GetTableRowCount_args {
public:
MegasearchService_GetTableRowCount_args(const MegasearchService_GetTableRowCount_args&);
MegasearchService_GetTableRowCount_args& operator=(const MegasearchService_GetTableRowCount_args&);
MegasearchService_GetTableRowCount_args() : table_name() {
}
virtual ~MegasearchService_GetTableRowCount_args() throw();
std::string table_name;
_MegasearchService_GetTableRowCount_args__isset __isset;
void __set_table_name(const std::string& val);
bool operator == (const MegasearchService_GetTableRowCount_args & rhs) const
{
if (!(table_name == rhs.table_name))
return false;
return true;
}
bool operator != (const MegasearchService_GetTableRowCount_args &rhs) const {
return !(*this == rhs);
}
bool operator < (const MegasearchService_GetTableRowCount_args & ) const;
uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
};
class MegasearchService_GetTableRowCount_pargs {
public:
virtual ~MegasearchService_GetTableRowCount_pargs() throw();
const std::string* table_name;
uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
};
typedef struct _MegasearchService_GetTableRowCount_result__isset {
_MegasearchService_GetTableRowCount_result__isset() : success(false), e(false) {}
bool success :1;
bool e :1;
} _MegasearchService_GetTableRowCount_result__isset;
class MegasearchService_GetTableRowCount_result {
public:
MegasearchService_GetTableRowCount_result(const MegasearchService_GetTableRowCount_result&);
MegasearchService_GetTableRowCount_result& operator=(const MegasearchService_GetTableRowCount_result&);
MegasearchService_GetTableRowCount_result() : success(0) {
}
virtual ~MegasearchService_GetTableRowCount_result() throw();
int64_t success;
Exception e;
_MegasearchService_GetTableRowCount_result__isset __isset;
void __set_success(const int64_t val);
void __set_e(const Exception& val);
bool operator == (const MegasearchService_GetTableRowCount_result & rhs) const
{
if (!(success == rhs.success))
return false;
if (!(e == rhs.e))
return false;
return true;
}
bool operator != (const MegasearchService_GetTableRowCount_result &rhs) const {
return !(*this == rhs);
}
bool operator < (const MegasearchService_GetTableRowCount_result & ) const;
uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
};
typedef struct _MegasearchService_GetTableRowCount_presult__isset {
_MegasearchService_GetTableRowCount_presult__isset() : success(false), e(false) {}
bool success :1;
bool e :1;
} _MegasearchService_GetTableRowCount_presult__isset;
class MegasearchService_GetTableRowCount_presult {
public:
virtual ~MegasearchService_GetTableRowCount_presult() throw();
int64_t* success;
Exception e;
_MegasearchService_GetTableRowCount_presult__isset __isset;
uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
};
class MegasearchService_ShowTables_args {
public:
@ -1209,21 +1109,18 @@ class MegasearchServiceClient : virtual public MegasearchServiceIf {
void DeleteTable(const std::string& table_name);
void send_DeleteTable(const std::string& table_name);
void recv_DeleteTable();
void CreateTablePartition(const CreateTablePartitionParam& param);
void send_CreateTablePartition(const CreateTablePartitionParam& param);
void recv_CreateTablePartition();
void DeleteTablePartition(const DeleteTablePartitionParam& param);
void send_DeleteTablePartition(const DeleteTablePartitionParam& param);
void recv_DeleteTablePartition();
void AddVector(std::vector<int64_t> & _return, const std::string& table_name, const std::vector<RowRecord> & record_array);
void send_AddVector(const std::string& table_name, const std::vector<RowRecord> & record_array);
void recv_AddVector(std::vector<int64_t> & _return);
void SearchVector(std::vector<TopKQueryResult> & _return, const std::string& table_name, const std::vector<QueryRecord> & query_record_array, const int64_t topk);
void send_SearchVector(const std::string& table_name, const std::vector<QueryRecord> & query_record_array, const int64_t topk);
void SearchVector(std::vector<TopKQueryResult> & _return, const std::string& table_name, const std::vector<RowRecord> & query_record_array, const std::vector<Range> & query_range_array, const int64_t topk);
void send_SearchVector(const std::string& table_name, const std::vector<RowRecord> & query_record_array, const std::vector<Range> & query_range_array, const int64_t topk);
void recv_SearchVector(std::vector<TopKQueryResult> & _return);
void DescribeTable(TableSchema& _return, const std::string& table_name);
void send_DescribeTable(const std::string& table_name);
void recv_DescribeTable(TableSchema& _return);
int64_t GetTableRowCount(const std::string& table_name);
void send_GetTableRowCount(const std::string& table_name);
int64_t recv_GetTableRowCount();
void ShowTables(std::vector<std::string> & _return);
void send_ShowTables();
void recv_ShowTables(std::vector<std::string> & _return);
@ -1247,11 +1144,10 @@ class MegasearchServiceProcessor : public ::apache::thrift::TDispatchProcessor {
ProcessMap processMap_;
void process_CreateTable(int32_t seqid, ::apache::thrift::protocol::TProtocol* iprot, ::apache::thrift::protocol::TProtocol* oprot, void* callContext);
void process_DeleteTable(int32_t seqid, ::apache::thrift::protocol::TProtocol* iprot, ::apache::thrift::protocol::TProtocol* oprot, void* callContext);
void process_CreateTablePartition(int32_t seqid, ::apache::thrift::protocol::TProtocol* iprot, ::apache::thrift::protocol::TProtocol* oprot, void* callContext);
void process_DeleteTablePartition(int32_t seqid, ::apache::thrift::protocol::TProtocol* iprot, ::apache::thrift::protocol::TProtocol* oprot, void* callContext);
void process_AddVector(int32_t seqid, ::apache::thrift::protocol::TProtocol* iprot, ::apache::thrift::protocol::TProtocol* oprot, void* callContext);
void process_SearchVector(int32_t seqid, ::apache::thrift::protocol::TProtocol* iprot, ::apache::thrift::protocol::TProtocol* oprot, void* callContext);
void process_DescribeTable(int32_t seqid, ::apache::thrift::protocol::TProtocol* iprot, ::apache::thrift::protocol::TProtocol* oprot, void* callContext);
void process_GetTableRowCount(int32_t seqid, ::apache::thrift::protocol::TProtocol* iprot, ::apache::thrift::protocol::TProtocol* oprot, void* callContext);
void process_ShowTables(int32_t seqid, ::apache::thrift::protocol::TProtocol* iprot, ::apache::thrift::protocol::TProtocol* oprot, void* callContext);
void process_Ping(int32_t seqid, ::apache::thrift::protocol::TProtocol* iprot, ::apache::thrift::protocol::TProtocol* oprot, void* callContext);
public:
@ -1259,11 +1155,10 @@ class MegasearchServiceProcessor : public ::apache::thrift::TDispatchProcessor {
iface_(iface) {
processMap_["CreateTable"] = &MegasearchServiceProcessor::process_CreateTable;
processMap_["DeleteTable"] = &MegasearchServiceProcessor::process_DeleteTable;
processMap_["CreateTablePartition"] = &MegasearchServiceProcessor::process_CreateTablePartition;
processMap_["DeleteTablePartition"] = &MegasearchServiceProcessor::process_DeleteTablePartition;
processMap_["AddVector"] = &MegasearchServiceProcessor::process_AddVector;
processMap_["SearchVector"] = &MegasearchServiceProcessor::process_SearchVector;
processMap_["DescribeTable"] = &MegasearchServiceProcessor::process_DescribeTable;
processMap_["GetTableRowCount"] = &MegasearchServiceProcessor::process_GetTableRowCount;
processMap_["ShowTables"] = &MegasearchServiceProcessor::process_ShowTables;
processMap_["Ping"] = &MegasearchServiceProcessor::process_Ping;
}
@ -1312,24 +1207,6 @@ class MegasearchServiceMultiface : virtual public MegasearchServiceIf {
ifaces_[i]->DeleteTable(table_name);
}
void CreateTablePartition(const CreateTablePartitionParam& param) {
size_t sz = ifaces_.size();
size_t i = 0;
for (; i < (sz - 1); ++i) {
ifaces_[i]->CreateTablePartition(param);
}
ifaces_[i]->CreateTablePartition(param);
}
void DeleteTablePartition(const DeleteTablePartitionParam& param) {
size_t sz = ifaces_.size();
size_t i = 0;
for (; i < (sz - 1); ++i) {
ifaces_[i]->DeleteTablePartition(param);
}
ifaces_[i]->DeleteTablePartition(param);
}
void AddVector(std::vector<int64_t> & _return, const std::string& table_name, const std::vector<RowRecord> & record_array) {
size_t sz = ifaces_.size();
size_t i = 0;
@ -1340,13 +1217,13 @@ class MegasearchServiceMultiface : virtual public MegasearchServiceIf {
return;
}
void SearchVector(std::vector<TopKQueryResult> & _return, const std::string& table_name, const std::vector<QueryRecord> & query_record_array, const int64_t topk) {
void SearchVector(std::vector<TopKQueryResult> & _return, const std::string& table_name, const std::vector<RowRecord> & query_record_array, const std::vector<Range> & query_range_array, const int64_t topk) {
size_t sz = ifaces_.size();
size_t i = 0;
for (; i < (sz - 1); ++i) {
ifaces_[i]->SearchVector(_return, table_name, query_record_array, topk);
ifaces_[i]->SearchVector(_return, table_name, query_record_array, query_range_array, topk);
}
ifaces_[i]->SearchVector(_return, table_name, query_record_array, topk);
ifaces_[i]->SearchVector(_return, table_name, query_record_array, query_range_array, topk);
return;
}
@ -1360,6 +1237,15 @@ class MegasearchServiceMultiface : virtual public MegasearchServiceIf {
return;
}
int64_t GetTableRowCount(const std::string& table_name) {
size_t sz = ifaces_.size();
size_t i = 0;
for (; i < (sz - 1); ++i) {
ifaces_[i]->GetTableRowCount(table_name);
}
return ifaces_[i]->GetTableRowCount(table_name);
}
void ShowTables(std::vector<std::string> & _return) {
size_t sz = ifaces_.size();
size_t i = 0;
@ -1416,21 +1302,18 @@ class MegasearchServiceConcurrentClient : virtual public MegasearchServiceIf {
void DeleteTable(const std::string& table_name);
int32_t send_DeleteTable(const std::string& table_name);
void recv_DeleteTable(const int32_t seqid);
void CreateTablePartition(const CreateTablePartitionParam& param);
int32_t send_CreateTablePartition(const CreateTablePartitionParam& param);
void recv_CreateTablePartition(const int32_t seqid);
void DeleteTablePartition(const DeleteTablePartitionParam& param);
int32_t send_DeleteTablePartition(const DeleteTablePartitionParam& param);
void recv_DeleteTablePartition(const int32_t seqid);
void AddVector(std::vector<int64_t> & _return, const std::string& table_name, const std::vector<RowRecord> & record_array);
int32_t send_AddVector(const std::string& table_name, const std::vector<RowRecord> & record_array);
void recv_AddVector(std::vector<int64_t> & _return, const int32_t seqid);
void SearchVector(std::vector<TopKQueryResult> & _return, const std::string& table_name, const std::vector<QueryRecord> & query_record_array, const int64_t topk);
int32_t send_SearchVector(const std::string& table_name, const std::vector<QueryRecord> & query_record_array, const int64_t topk);
void SearchVector(std::vector<TopKQueryResult> & _return, const std::string& table_name, const std::vector<RowRecord> & query_record_array, const std::vector<Range> & query_range_array, const int64_t topk);
int32_t send_SearchVector(const std::string& table_name, const std::vector<RowRecord> & query_record_array, const std::vector<Range> & query_range_array, const int64_t topk);
void recv_SearchVector(std::vector<TopKQueryResult> & _return, const int32_t seqid);
void DescribeTable(TableSchema& _return, const std::string& table_name);
int32_t send_DescribeTable(const std::string& table_name);
void recv_DescribeTable(TableSchema& _return, const int32_t seqid);
int64_t GetTableRowCount(const std::string& table_name);
int32_t send_GetTableRowCount(const std::string& table_name);
int64_t recv_GetTableRowCount(const int32_t seqid);
void ShowTables(std::vector<std::string> & _return);
int32_t send_ShowTables();
void recv_ShowTables(std::vector<std::string> & _return, const int32_t seqid);

56
cpp/src/thrift/gen-cpp/MegasearchService_server.skeleton.cpp
View File

@ -50,36 +50,6 @@ class MegasearchServiceHandler : virtual public MegasearchServiceIf {
printf("DeleteTable\n");
}
/**
* @brief Create table partition
*
* This method is used to create table partition.
*
* @param param, use to provide partition information to be created.
*
*
* @param param
*/
void CreateTablePartition(const CreateTablePartitionParam& param) {
// Your implementation goes here
printf("CreateTablePartition\n");
}
/**
* @brief Delete table partition
*
* This method is used to delete table partition.
*
* @param param, use to provide partition information to be deleted.
*
*
* @param param
*/
void DeleteTablePartition(const DeleteTablePartitionParam& param) {
// Your implementation goes here
printf("DeleteTablePartition\n");
}
/**
* @brief Add vector array to table
*
@ -105,25 +75,27 @@ class MegasearchServiceHandler : virtual public MegasearchServiceIf {
*
* @param table_name, table_name is queried.
* @param query_record_array, all vector are going to be queried.
* @param query_range_array, optional ranges for conditional search. If not specified, search whole table
* @param topk, how many similarity vectors will be searched.
*
* @return query result array.
*
* @param table_name
* @param query_record_array
* @param query_range_array
* @param topk
*/
void SearchVector(std::vector<TopKQueryResult> & _return, const std::string& table_name, const std::vector<QueryRecord> & query_record_array, const int64_t topk) {
void SearchVector(std::vector<TopKQueryResult> & _return, const std::string& table_name, const std::vector<RowRecord> & query_record_array, const std::vector<Range> & query_range_array, const int64_t topk) {
// Your implementation goes here
printf("SearchVector\n");
}
/**
* @brief Show table information
* @brief Get table schema
*
* This method is used to show table information.
* This method is used to get table schema.
*
* @param table_name, which table is show.
* @param table_name, target table name.
*
* @return table schema
*
@ -134,6 +106,22 @@ class MegasearchServiceHandler : virtual public MegasearchServiceIf {
printf("DescribeTable\n");
}
/**
* @brief Get table row count
*
* This method is used to get table row count.
*
* @param table_name, target table name.
*
* @return table row count
*
* @param table_name
*/
int64_t GetTableRowCount(const std::string& table_name) {
// Your implementation goes here
printf("GetTableRowCount\n");
}
/**
* @brief List all tables in database
*

1175
cpp/src/thrift/gen-cpp/megasearch_types.cpp
View File

File diff suppressed because it is too large Load Diff

324
cpp/src/thrift/gen-cpp/megasearch_types.h
View File

@ -26,10 +26,9 @@ struct ErrorCode {
CONNECT_FAILED = 1,
PERMISSION_DENIED = 2,
TABLE_NOT_EXISTS = 3,
PARTITION_NOT_EXIST = 4,
ILLEGAL_ARGUMENT = 5,
ILLEGAL_RANGE = 6,
ILLEGAL_DIMENSION = 7
ILLEGAL_ARGUMENT = 4,
ILLEGAL_RANGE = 5,
ILLEGAL_DIMENSION = 6
};
};
@ -39,22 +38,12 @@ std::ostream& operator<<(std::ostream& out, const ErrorCode::type& val);
class Exception;
class Column;
class VectorColumn;
class TableSchema;
class Range;
class CreateTablePartitionParam;
class DeleteTablePartitionParam;
class RowRecord;
class QueryRecord;
class QueryResult;
class TopKQueryResult;
@ -109,108 +98,11 @@ void swap(Exception &a, Exception &b);
std::ostream& operator<<(std::ostream& out, const Exception& obj);
class Column : public virtual ::apache::thrift::TBase {
public:
Column(const Column&);
Column& operator=(const Column&);
Column() : type(0), name() {
}
virtual ~Column() throw();
int32_t type;
std::string name;
void __set_type(const int32_t val);
void __set_name(const std::string& val);
bool operator == (const Column & rhs) const
{
if (!(type == rhs.type))
return false;
if (!(name == rhs.name))
return false;
return true;
}
bool operator != (const Column &rhs) const {
return !(*this == rhs);
}
bool operator < (const Column & ) const;
uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
virtual void printTo(std::ostream& out) const;
};
void swap(Column &a, Column &b);
std::ostream& operator<<(std::ostream& out, const Column& obj);
typedef struct _VectorColumn__isset {
_VectorColumn__isset() : store_raw_vector(true) {}
bool store_raw_vector :1;
} _VectorColumn__isset;
class VectorColumn : public virtual ::apache::thrift::TBase {
public:
VectorColumn(const VectorColumn&);
VectorColumn& operator=(const VectorColumn&);
VectorColumn() : dimension(0), index_type(), store_raw_vector(false) {
}
virtual ~VectorColumn() throw();
Column base;
int64_t dimension;
std::string index_type;
bool store_raw_vector;
_VectorColumn__isset __isset;
void __set_base(const Column& val);
void __set_dimension(const int64_t val);
void __set_index_type(const std::string& val);
void __set_store_raw_vector(const bool val);
bool operator == (const VectorColumn & rhs) const
{
if (!(base == rhs.base))
return false;
if (!(dimension == rhs.dimension))
return false;
if (!(index_type == rhs.index_type))
return false;
if (!(store_raw_vector == rhs.store_raw_vector))
return false;
return true;
}
bool operator != (const VectorColumn &rhs) const {
return !(*this == rhs);
}
bool operator < (const VectorColumn & ) const;
uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
virtual void printTo(std::ostream& out) const;
};
void swap(VectorColumn &a, VectorColumn &b);
std::ostream& operator<<(std::ostream& out, const VectorColumn& obj);
typedef struct _TableSchema__isset {
_TableSchema__isset() : attribute_column_array(false), partition_column_name_array(false) {}
bool attribute_column_array :1;
bool partition_column_name_array :1;
_TableSchema__isset() : index_type(true), dimension(true), store_raw_vector(true) {}
bool index_type :1;
bool dimension :1;
bool store_raw_vector :1;
} _TableSchema__isset;
class TableSchema : public virtual ::apache::thrift::TBase {
@ -218,38 +110,34 @@ class TableSchema : public virtual ::apache::thrift::TBase {
TableSchema(const TableSchema&);
TableSchema& operator=(const TableSchema&);
TableSchema() : table_name() {
TableSchema() : table_name(), index_type(0), dimension(0LL), store_raw_vector(false) {
}
virtual ~TableSchema() throw();
std::string table_name;
std::vector<VectorColumn> vector_column_array;
std::vector<Column> attribute_column_array;
std::vector<std::string> partition_column_name_array;
int32_t index_type;
int64_t dimension;
bool store_raw_vector;
_TableSchema__isset __isset;
void __set_table_name(const std::string& val);
void __set_vector_column_array(const std::vector<VectorColumn> & val);
void __set_index_type(const int32_t val);
void __set_attribute_column_array(const std::vector<Column> & val);
void __set_dimension(const int64_t val);
void __set_partition_column_name_array(const std::vector<std::string> & val);
void __set_store_raw_vector(const bool val);
bool operator == (const TableSchema & rhs) const
{
if (!(table_name == rhs.table_name))
return false;
if (!(vector_column_array == rhs.vector_column_array))
if (!(index_type == rhs.index_type))
return false;
if (__isset.attribute_column_array != rhs.__isset.attribute_column_array)
if (!(dimension == rhs.dimension))
return false;
else if (__isset.attribute_column_array && !(attribute_column_array == rhs.attribute_column_array))
return false;
if (__isset.partition_column_name_array != rhs.__isset.partition_column_name_array)
return false;
else if (__isset.partition_column_name_array && !(partition_column_name_array == rhs.partition_column_name_array))
if (!(store_raw_vector == rhs.store_raw_vector))
return false;
return true;
}
@ -269,6 +157,11 @@ void swap(TableSchema &a, TableSchema &b);
std::ostream& operator<<(std::ostream& out, const TableSchema& obj);
typedef struct _Range__isset {
_Range__isset() : start_value(false), end_value(false) {}
bool start_value :1;
bool end_value :1;
} _Range__isset;
class Range : public virtual ::apache::thrift::TBase {
public:
@ -282,6 +175,8 @@ class Range : public virtual ::apache::thrift::TBase {
std::string start_value;
std::string end_value;
_Range__isset __isset;
void __set_start_value(const std::string& val);
void __set_end_value(const std::string& val);
@ -311,120 +206,22 @@ void swap(Range &a, Range &b);
std::ostream& operator<<(std::ostream& out, const Range& obj);
class CreateTablePartitionParam : public virtual ::apache::thrift::TBase {
public:
CreateTablePartitionParam(const CreateTablePartitionParam&);
CreateTablePartitionParam& operator=(const CreateTablePartitionParam&);
CreateTablePartitionParam() : table_name(), partition_name() {
}
virtual ~CreateTablePartitionParam() throw();
std::string table_name;
std::string partition_name;
std::map<std::string, Range> range_map;
void __set_table_name(const std::string& val);
void __set_partition_name(const std::string& val);
void __set_range_map(const std::map<std::string, Range> & val);
bool operator == (const CreateTablePartitionParam & rhs) const
{
if (!(table_name == rhs.table_name))
return false;
if (!(partition_name == rhs.partition_name))
return false;
if (!(range_map == rhs.range_map))
return false;
return true;
}
bool operator != (const CreateTablePartitionParam &rhs) const {
return !(*this == rhs);
}
bool operator < (const CreateTablePartitionParam & ) const;
uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
virtual void printTo(std::ostream& out) const;
};
void swap(CreateTablePartitionParam &a, CreateTablePartitionParam &b);
std::ostream& operator<<(std::ostream& out, const CreateTablePartitionParam& obj);
class DeleteTablePartitionParam : public virtual ::apache::thrift::TBase {
public:
DeleteTablePartitionParam(const DeleteTablePartitionParam&);
DeleteTablePartitionParam& operator=(const DeleteTablePartitionParam&);
DeleteTablePartitionParam() : table_name() {
}
virtual ~DeleteTablePartitionParam() throw();
std::string table_name;
std::vector<std::string> partition_name_array;
void __set_table_name(const std::string& val);
void __set_partition_name_array(const std::vector<std::string> & val);
bool operator == (const DeleteTablePartitionParam & rhs) const
{
if (!(table_name == rhs.table_name))
return false;
if (!(partition_name_array == rhs.partition_name_array))
return false;
return true;
}
bool operator != (const DeleteTablePartitionParam &rhs) const {
return !(*this == rhs);
}
bool operator < (const DeleteTablePartitionParam & ) const;
uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
virtual void printTo(std::ostream& out) const;
};
void swap(DeleteTablePartitionParam &a, DeleteTablePartitionParam &b);
std::ostream& operator<<(std::ostream& out, const DeleteTablePartitionParam& obj);
typedef struct _RowRecord__isset {
_RowRecord__isset() : attribute_map(false) {}
bool attribute_map :1;
} _RowRecord__isset;
class RowRecord : public virtual ::apache::thrift::TBase {
public:
RowRecord(const RowRecord&);
RowRecord& operator=(const RowRecord&);
RowRecord() {
RowRecord() : vector_data() {
}
virtual ~RowRecord() throw();
std::map<std::string, std::string> vector_map;
std::map<std::string, std::string> attribute_map;
std::string vector_data;
_RowRecord__isset __isset;
void __set_vector_map(const std::map<std::string, std::string> & val);
void __set_attribute_map(const std::map<std::string, std::string> & val);
void __set_vector_data(const std::string& val);
bool operator == (const RowRecord & rhs) const
{
if (!(vector_map == rhs.vector_map))
return false;
if (!(attribute_map == rhs.attribute_map))
if (!(vector_data == rhs.vector_data))
return false;
return true;
}
@ -444,68 +241,10 @@ void swap(RowRecord &a, RowRecord &b);
std::ostream& operator<<(std::ostream& out, const RowRecord& obj);
typedef struct _QueryRecord__isset {
_QueryRecord__isset() : selected_column_array(false), partition_filter_column_map(false) {}
bool selected_column_array :1;
bool partition_filter_column_map :1;
} _QueryRecord__isset;
class QueryRecord : public virtual ::apache::thrift::TBase {
public:
QueryRecord(const QueryRecord&);
QueryRecord& operator=(const QueryRecord&);
QueryRecord() {
}
virtual ~QueryRecord() throw();
std::map<std::string, std::string> vector_map;
std::vector<std::string> selected_column_array;
std::map<std::string, std::vector<Range> > partition_filter_column_map;
_QueryRecord__isset __isset;
void __set_vector_map(const std::map<std::string, std::string> & val);
void __set_selected_column_array(const std::vector<std::string> & val);
void __set_partition_filter_column_map(const std::map<std::string, std::vector<Range> > & val);
bool operator == (const QueryRecord & rhs) const
{
if (!(vector_map == rhs.vector_map))
return false;
if (__isset.selected_column_array != rhs.__isset.selected_column_array)
return false;
else if (__isset.selected_column_array && !(selected_column_array == rhs.selected_column_array))
return false;
if (__isset.partition_filter_column_map != rhs.__isset.partition_filter_column_map)
return false;
else if (__isset.partition_filter_column_map && !(partition_filter_column_map == rhs.partition_filter_column_map))
return false;
return true;
}
bool operator != (const QueryRecord &rhs) const {
return !(*this == rhs);
}
bool operator < (const QueryRecord & ) const;
uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
virtual void printTo(std::ostream& out) const;
};
void swap(QueryRecord &a, QueryRecord &b);
std::ostream& operator<<(std::ostream& out, const QueryRecord& obj);
typedef struct _QueryResult__isset {
_QueryResult__isset() : id(false), score(false), column_map(false) {}
_QueryResult__isset() : id(false), score(false) {}
bool id :1;
bool score :1;
bool column_map :1;
} _QueryResult__isset;
class QueryResult : public virtual ::apache::thrift::TBase {
@ -519,7 +258,6 @@ class QueryResult : public virtual ::apache::thrift::TBase {
virtual ~QueryResult() throw();
int64_t id;
double score;
std::map<std::string, std::string> column_map;
_QueryResult__isset __isset;
@ -527,16 +265,12 @@ class QueryResult : public virtual ::apache::thrift::TBase {
void __set_score(const double val);
void __set_column_map(const std::map<std::string, std::string> & val);
bool operator == (const QueryResult & rhs) const
{
if (!(id == rhs.id))
return false;
if (!(score == rhs.score))
return false;
if (!(column_map == rhs.column_map))
return false;
return true;
}
bool operator != (const QueryResult &rhs) const {

128
cpp/src/thrift/megasearch.thrift
View File

@ -17,104 +17,56 @@ enum ErrorCode {
SUCCESS = 0,
CONNECT_FAILED,
PERMISSION_DENIED,
TABLE_NOT_EXISTS,
PARTITION_NOT_EXIST,
ILLEGAL_ARGUMENT,
ILLEGAL_RANGE,
ILLEGAL_DIMENSION,
TABLE_NOT_EXISTS,
ILLEGAL_ARGUMENT,
ILLEGAL_RANGE,
ILLEGAL_DIMENSION,
}
exception Exception {
1: ErrorCode code;
2: string reason;
1: ErrorCode code;
2: string reason;
}
/**
* @brief Table column description
*/
struct Column {
1: required i32 type; ///< Column Type: 0:invealid/1:int8/2:int16/3:int32/4:int64/5:float32/6:float64/7:date/8:vector
2: required string name; ///< Column name
}
/**
* @brief Table vector column description
*/
struct VectorColumn {
1: required Column base; ///< Base column schema
2: required i64 dimension; ///< Vector dimension
3: required string index_type; ///< Index type, optional: raw, ivf
4: bool store_raw_vector = false; ///< Is vector self stored in the table
}
/**
* @brief Table Schema
*/
struct TableSchema {
1: required string table_name; ///< Table name
2: required list<VectorColumn> vector_column_array; ///< Vector column description
3: optional list<Column> attribute_column_array; ///< Columns description
4: optional list<string> partition_column_name_array; ///< Partition column name
1: required string table_name; ///< Table name
2: i32 index_type = 0; ///< Index type, optional: 0-invalid, 1-idmap, 2-ivflat
3: i64 dimension = 0; ///< Vector dimension
4: bool store_raw_vector = false; ///< Store raw data
}
/**
* @brief Range Schema
*/
struct Range {
1: required string start_value; ///< Range start
2: required string end_value; ///< Range stop
}
/**
* @brief Create table partition parameters
*/
struct CreateTablePartitionParam {
1: required string table_name; ///< Table name, vector/float32/float64 type column is not allowed for partition
2: required string partition_name; ///< Partition name, created partition name
3: required map<string, Range> range_map; ///< Column name to Range map
}
/**
* @brief Delete table partition parameters
*/
struct DeleteTablePartitionParam {
1: required string table_name; ///< Table name
2: required list<string> partition_name_array; ///< Partition name array
1: string start_value; ///< Range start
2: string end_value; ///< Range stop
}
/**
* @brief Record inserted
*/
struct RowRecord {
1: required map<string, binary> vector_map; ///< Vector columns
2: map<string, string> attribute_map; ///< Other attribute columns
}
/**
* @brief Query record
*/
struct QueryRecord {
1: required map<string, binary> vector_map; ///< Query vectors
2: optional list<string> selected_column_array; ///< Output column array
3: optional map<string, list<Range>> partition_filter_column_map; ///< Range used to select partitions
1: required binary vector_data; ///< Vector data, double array
}
/**
* @brief Query result
*/
struct QueryResult {
1: i64 id; ///< Output result
2: double score; ///< Vector similarity score: 0 ~ 100
3: map<string, string> column_map; ///< Other column
1: i64 id; ///< Output result
2: double score; ///< Vector similarity score: 0 ~ 100
}
/**
* @brief TopK query result
*/
struct TopKQueryResult {
1: list<QueryResult> query_result_arrays; ///< TopK query result
1: list<QueryResult> query_result_arrays; ///< TopK query result
}
service MegasearchService {
@ -140,28 +92,6 @@ service MegasearchService {
void DeleteTable(2: string table_name) throws(1: Exception e);
/**
* @brief Create table partition
*
* This method is used to create table partition.
*
* @param param, use to provide partition information to be created.
*
*/
void CreateTablePartition(2: CreateTablePartitionParam param) throws(1: Exception e);
/**
* @brief Delete table partition
*
* This method is used to delete table partition.
*
* @param param, use to provide partition information to be deleted.
*
*/
void DeleteTablePartition(2: DeleteTablePartitionParam param) throws(1: Exception e);
/**
* @brief Add vector array to table
*
@ -183,25 +113,40 @@ service MegasearchService {
*
* @param table_name, table_name is queried.
* @param query_record_array, all vector are going to be queried.
* @param query_range_array, optional ranges for conditional search. If not specified, search whole table
* @param topk, how many similarity vectors will be searched.
*
* @return query result array.
*/
list<TopKQueryResult> SearchVector(2: string table_name,
3: list<QueryRecord> query_record_array,
4: i64 topk) throws(1: Exception e);
3: list<RowRecord> query_record_array,
4: list<Range> query_range_array,
5: i64 topk) throws(1: Exception e);
/**
* @brief Show table information
* @brief Get table schema
*
* This method is used to show table information.
* This method is used to get table schema.
*
* @param table_name, which table is show.
* @param table_name, target table name.
*
* @return table schema
*/
TableSchema DescribeTable(2: string table_name) throws(1: Exception e);
/**
* @brief Get table row count
*
* This method is used to get table row count.
*
* @param table_name, target table name.
*
* @return table row count
*/
i64 GetTableRowCount(2: string table_name) throws(1: Exception e);
/**
* @brief List all tables in database
*
@ -212,6 +157,7 @@ service MegasearchService {
*/
list<string> ShowTables() throws(1: Exception e);
/**
* @brief Give the server status
*

2
cpp/src/utils/Error.h
View File

@ -31,7 +31,7 @@ constexpr ServerError SERVER_INVALID_ARGUMENT = ToGlobalServerErrorCode(0x004);
constexpr ServerError SERVER_FILE_NOT_FOUND = ToGlobalServerErrorCode(0x005);
constexpr ServerError SERVER_NOT_IMPLEMENT = ToGlobalServerErrorCode(0x006);
constexpr ServerError SERVER_BLOCKING_QUEUE_EMPTY = ToGlobalServerErrorCode(0x007);
constexpr ServerError SERVER_GROUP_NOT_EXIST = ToGlobalServerErrorCode(0x008);
constexpr ServerError SERVER_TABLE_NOT_EXIST = ToGlobalServerErrorCode(0x008);
constexpr ServerError SERVER_INVALID_TIME_RANGE = ToGlobalServerErrorCode(0x009);
constexpr ServerError SERVER_INVALID_VECTOR_DIMENSION = ToGlobalServerErrorCode(0x00a);
constexpr ServerError SERVER_LICENSE_VALIDATION_FAIL = ToGlobalServerErrorCode(0x00b);

24
cpp/unittest/db/db_tests.cpp
View File

@ -67,15 +67,15 @@ TEST_F(DBTest2, ARHIVE_DISK_CHECK) {
long size;
engine::meta::TableSchema group_info;
group_info.dimension = group_dim;
group_info.table_id = group_name;
group_info.dimension_ = group_dim;
group_info.table_id_ = group_name;
engine::Status stat = db_->CreateTable(group_info);
engine::meta::TableSchema group_info_get;
group_info_get.table_id = group_name;
group_info_get.table_id_ = group_name;
stat = db_->DescribeTable(group_info_get);
ASSERT_STATS(stat);
ASSERT_EQ(group_info_get.dimension, group_dim);
ASSERT_EQ(group_info_get.dimension_, group_dim);
engine::IDNumbers vector_ids;
engine::IDNumbers target_ids;
@ -114,15 +114,15 @@ TEST_F(DBTest, DB_TEST) {
static const int group_dim = 256;
engine::meta::TableSchema group_info;
group_info.dimension = group_dim;
group_info.table_id = group_name;
group_info.dimension_ = group_dim;
group_info.table_id_ = group_name;
engine::Status stat = db_->CreateTable(group_info);
engine::meta::TableSchema group_info_get;
group_info_get.table_id = group_name;
group_info_get.table_id_ = group_name;
stat = db_->DescribeTable(group_info_get);
ASSERT_STATS(stat);
ASSERT_EQ(group_info_get.dimension, group_dim);
ASSERT_EQ(group_info_get.dimension_, group_dim);
engine::IDNumbers vector_ids;
engine::IDNumbers target_ids;
@ -200,15 +200,15 @@ TEST_F(DBTest, SEARCH_TEST) {
static const int group_dim = 256;
engine::meta::TableSchema group_info;
group_info.dimension = group_dim;
group_info.table_id = group_name;
group_info.dimension_ = group_dim;
group_info.table_id_ = group_name;
engine::Status stat = db_->CreateTable(group_info);
engine::meta::TableSchema group_info_get;
group_info_get.table_id = group_name;
group_info_get.table_id_ = group_name;
stat = db_->DescribeTable(group_info_get);
ASSERT_STATS(stat);
ASSERT_EQ(group_info_get.dimension, group_dim);
ASSERT_EQ(group_info_get.dimension_, group_dim);
// prepare raw data
size_t nb = 250000;

86
cpp/unittest/db/meta_tests.cpp
View File

@ -21,22 +21,22 @@ TEST_F(MetaTest, GROUP_TEST) {
auto table_id = "meta_test_group";
meta::TableSchema group;
group.table_id = table_id;
group.table_id_ = table_id;
auto status = impl_->CreateTable(group);
ASSERT_TRUE(status.ok());
auto gid = group.id;
group.id = -1;
auto gid = group.id_;
group.id_ = -1;
status = impl_->DescribeTable(group);
ASSERT_TRUE(status.ok());
ASSERT_EQ(group.id, gid);
ASSERT_EQ(group.table_id, table_id);
ASSERT_EQ(group.id_, gid);
ASSERT_EQ(group.table_id_, table_id);
group.table_id = "not_found";
group.table_id_ = "not_found";
status = impl_->DescribeTable(group);
ASSERT_TRUE(!status.ok());
group.table_id = table_id;
group.table_id_ = table_id;
status = impl_->CreateTable(group);
ASSERT_TRUE(!status.ok());
}
@ -45,49 +45,49 @@ TEST_F(MetaTest, table_file_TEST) {
auto table_id = "meta_test_group";
meta::TableSchema group;
group.table_id = table_id;
group.table_id_ = table_id;
auto status = impl_->CreateTable(group);
meta::TableFileSchema table_file;
table_file.table_id = group.table_id;
table_file.table_id_ = group.table_id_;
status = impl_->CreateTableFile(table_file);
ASSERT_TRUE(status.ok());
ASSERT_EQ(table_file.file_type, meta::TableFileSchema::NEW);
ASSERT_EQ(table_file.file_type_, meta::TableFileSchema::NEW);
auto file_id = table_file.file_id;
auto file_id = table_file.file_id_;
auto new_file_type = meta::TableFileSchema::INDEX;
table_file.file_type = new_file_type;
table_file.file_type_ = new_file_type;
status = impl_->UpdateTableFile(table_file);
ASSERT_TRUE(status.ok());
ASSERT_EQ(table_file.file_type, new_file_type);
ASSERT_EQ(table_file.file_type_, new_file_type);
meta::DatesT dates;
dates.push_back(meta::Meta::GetDate());
status = impl_->DropPartitionsByDates(table_file.table_id, dates);
status = impl_->DropPartitionsByDates(table_file.table_id_, dates);
ASSERT_FALSE(status.ok());
dates.clear();
for (auto i=2; i < 10; ++i) {
dates.push_back(meta::Meta::GetDateWithDelta(-1*i));
}
status = impl_->DropPartitionsByDates(table_file.table_id, dates);
status = impl_->DropPartitionsByDates(table_file.table_id_, dates);
ASSERT_TRUE(status.ok());
table_file.date = meta::Meta::GetDateWithDelta(-2);
table_file.date_ = meta::Meta::GetDateWithDelta(-2);
status = impl_->UpdateTableFile(table_file);
ASSERT_TRUE(status.ok());
ASSERT_EQ(table_file.date, meta::Meta::GetDateWithDelta(-2));
ASSERT_FALSE(table_file.file_type == meta::TableFileSchema::TO_DELETE);
ASSERT_EQ(table_file.date_, meta::Meta::GetDateWithDelta(-2));
ASSERT_FALSE(table_file.file_type_ == meta::TableFileSchema::TO_DELETE);
dates.clear();
dates.push_back(table_file.date);
status = impl_->DropPartitionsByDates(table_file.table_id, dates);
dates.push_back(table_file.date_);
status = impl_->DropPartitionsByDates(table_file.table_id_, dates);
ASSERT_TRUE(status.ok());
status = impl_->GetTableFile(table_file);
ASSERT_TRUE(status.ok());
ASSERT_TRUE(table_file.file_type == meta::TableFileSchema::TO_DELETE);
ASSERT_TRUE(table_file.file_type_ == meta::TableFileSchema::TO_DELETE);
}
TEST_F(MetaTest, ARCHIVE_TEST_DAYS) {
@ -103,21 +103,21 @@ TEST_F(MetaTest, ARCHIVE_TEST_DAYS) {
auto table_id = "meta_test_group";
meta::TableSchema group;
group.table_id = table_id;
group.table_id_ = table_id;
auto status = impl.CreateTable(group);
meta::TableFilesSchema files;
meta::TableFileSchema table_file;
table_file.table_id = group.table_id;
table_file.table_id_ = group.table_id_;
auto cnt = 100;
long ts = utils::GetMicroSecTimeStamp();
std::vector<int> days;
for (auto i=0; i<cnt; ++i) {
status = impl.CreateTableFile(table_file);
table_file.file_type = meta::TableFileSchema::NEW;
table_file.file_type_ = meta::TableFileSchema::NEW;
int day = rand() % (days_num*2);
table_file.created_on = ts - day*meta::D_SEC*meta::US_PS - 10000;
table_file.created_on_ = ts - day*meta::D_SEC*meta::US_PS - 10000;
status = impl.UpdateTableFile(table_file);
files.push_back(table_file);
days.push_back(day);
@ -130,9 +130,9 @@ TEST_F(MetaTest, ARCHIVE_TEST_DAYS) {
status = impl.GetTableFile(file);
ASSERT_TRUE(status.ok());
if (days[i] < days_num) {
ASSERT_EQ(file.file_type, meta::TableFileSchema::NEW);
ASSERT_EQ(file.file_type_, meta::TableFileSchema::NEW);
} else {
ASSERT_EQ(file.file_type, meta::TableFileSchema::TO_DELETE);
ASSERT_EQ(file.file_type_, meta::TableFileSchema::TO_DELETE);
}
i++;
}
@ -149,19 +149,19 @@ TEST_F(MetaTest, ARCHIVE_TEST_DISK) {
auto table_id = "meta_test_group";
meta::TableSchema group;
group.table_id = table_id;
group.table_id_ = table_id;
auto status = impl.CreateTable(group);
meta::TableFilesSchema files;
meta::TableFileSchema table_file;
table_file.table_id = group.table_id;
table_file.table_id_ = group.table_id_;
auto cnt = 10;
auto each_size = 2UL;
for (auto i=0; i<cnt; ++i) {
status = impl.CreateTableFile(table_file);
table_file.file_type = meta::TableFileSchema::NEW;
table_file.size = each_size * meta::G;
table_file.file_type_ = meta::TableFileSchema::NEW;
table_file.size_ = each_size * meta::G;
status = impl.UpdateTableFile(table_file);
files.push_back(table_file);
}
@ -173,9 +173,9 @@ TEST_F(MetaTest, ARCHIVE_TEST_DISK) {
status = impl.GetTableFile(file);
ASSERT_TRUE(status.ok());
if (i < 5) {
ASSERT_TRUE(file.file_type == meta::TableFileSchema::TO_DELETE);
ASSERT_TRUE(file.file_type_ == meta::TableFileSchema::TO_DELETE);
} else {
ASSERT_EQ(file.file_type, meta::TableFileSchema::NEW);
ASSERT_EQ(file.file_type_, meta::TableFileSchema::NEW);
}
++i;
}
@ -187,7 +187,7 @@ TEST_F(MetaTest, TABLE_FILES_TEST) {
auto table_id = "meta_test_group";
meta::TableSchema group;
group.table_id = table_id;
group.table_id_ = table_id;
auto status = impl_->CreateTable(group);
int new_files_cnt = 4;
@ -196,29 +196,29 @@ TEST_F(MetaTest, TABLE_FILES_TEST) {
int index_files_cnt = 7;
meta::TableFileSchema table_file;
table_file.table_id = group.table_id;
table_file.table_id_ = group.table_id_;
for (auto i=0; i<new_files_cnt; ++i) {
status = impl_->CreateTableFile(table_file);
table_file.file_type = meta::TableFileSchema::NEW;
table_file.file_type_ = meta::TableFileSchema::NEW;
status = impl_->UpdateTableFile(table_file);
}
for (auto i=0; i<raw_files_cnt; ++i) {
status = impl_->CreateTableFile(table_file);
table_file.file_type = meta::TableFileSchema::RAW;
table_file.file_type_ = meta::TableFileSchema::RAW;
status = impl_->UpdateTableFile(table_file);
}
for (auto i=0; i<to_index_files_cnt; ++i) {
status = impl_->CreateTableFile(table_file);
table_file.file_type = meta::TableFileSchema::TO_INDEX;
table_file.file_type_ = meta::TableFileSchema::TO_INDEX;
status = impl_->UpdateTableFile(table_file);
}
for (auto i=0; i<index_files_cnt; ++i) {
status = impl_->CreateTableFile(table_file);
table_file.file_type = meta::TableFileSchema::INDEX;
table_file.file_type_ = meta::TableFileSchema::INDEX;
status = impl_->UpdateTableFile(table_file);
}
@ -229,17 +229,17 @@ TEST_F(MetaTest, TABLE_FILES_TEST) {
ASSERT_EQ(files.size(), to_index_files_cnt);
meta::DatePartionedTableFilesSchema dated_files;
status = impl_->FilesToMerge(group.table_id, dated_files);
status = impl_->FilesToMerge(group.table_id_, dated_files);
ASSERT_TRUE(status.ok());
ASSERT_EQ(dated_files[table_file.date].size(), raw_files_cnt);
ASSERT_EQ(dated_files[table_file.date_].size(), raw_files_cnt);
status = impl_->FilesToIndex(files);
ASSERT_TRUE(status.ok());
ASSERT_EQ(files.size(), to_index_files_cnt);
meta::DatesT dates = {table_file.date};
meta::DatesT dates = {table_file.date_};
status = impl_->FilesToSearch(table_id, dates, dated_files);
ASSERT_TRUE(status.ok());
ASSERT_EQ(dated_files[table_file.date].size(),
ASSERT_EQ(dated_files[table_file.date_].size(),
to_index_files_cnt+raw_files_cnt+index_files_cnt);
}

3
cpp/unittest/faiss_wrapper/CMakeLists.txt
View File

@ -13,8 +13,6 @@ include_directories(/usr/local/cuda/include)
link_directories("/usr/local/cuda/lib64")
set(require_files
../../src/server/VecIdMapper.cpp
../../src/server/RocksIdMapper.cpp
../../src/server/ServerConfig.cpp
../../src/utils/CommonUtil.cpp
../../src/utils/TimeRecorder.cpp
@ -38,7 +36,6 @@ set(wrapper_libs
cudart
cublas
sqlite3
rocksdb
snappy
bz2
z

6
cpp/unittest/metrics/metrics_test.cpp
View File

@ -38,12 +38,12 @@ TEST_F(DBTest, Metric_Tes) {
static const int group_dim = 256;
engine::meta::TableSchema group_info;
group_info.dimension = group_dim;
group_info.table_id = group_name;
group_info.dimension_ = group_dim;
group_info.table_id_ = group_name;
engine::Status stat = db_->CreateTable(group_info);
engine::meta::TableSchema group_info_get;
group_info_get.table_id = group_name;
group_info_get.table_id_ = group_name;
stat = db_->DescribeTable(group_info_get);

3
cpp/unittest/server/CMakeLists.txt
View File

@ -15,8 +15,6 @@ aux_source_directory(../../src/wrapper wrapper_src)
aux_source_directory(./ test_srcs)
set(require_files
../../src/server/VecIdMapper.cpp
../../src/server/RocksIdMapper.cpp
../../src/server/ServerConfig.cpp
../../src/utils/CommonUtil.cpp
../../src/utils/TimeRecorder.cpp
@ -35,7 +33,6 @@ cuda_add_executable(server_test
set(require_libs
stdc++
rocksdb
faiss
cudart
cublas

118
cpp/unittest/server/idmapper_test.cpp
View File

@ -1,118 +0,0 @@
////////////////////////////////////////////////////////////////////////////////
// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved
// Unauthorized copying of this file, via any medium is strictly prohibited.
// Proprietary and confidential.
////////////////////////////////////////////////////////////////////////////////
#include <gtest/gtest.h>
#include "server/ServerConfig.h"
#include "server/VecIdMapper.h"
#include "utils/TimeRecorder.h"
#include "utils/CommonUtil.h"
#include <time.h>
using namespace zilliz::vecwise;
namespace {
std::string CurrentTime() {
time_t tt;
time(&tt);
tt = tt + 8 * 3600;
tm *t = gmtime(&tt);
std::string str = std::to_string(t->tm_year + 1900) + "_" + std::to_string(t->tm_mon + 1)
+ "_" + std::to_string(t->tm_mday) + "_" + std::to_string(t->tm_hour)
+ "_" + std::to_string(t->tm_min) + "_" + std::to_string(t->tm_sec);
return str;
}
std::string GetGroupID() {
static std::string s_id(CurrentTime());
return s_id;
}
}
TEST(IdMapperTest, IDMAPPER_TEST) {
server::IVecIdMapper* mapper = server::IVecIdMapper::GetInstance();
std::string group_id = GetGroupID();
std::vector<std::string> nid = {"1", "50", "900", "10000"};
std::vector<std::string> sid = {"one", "fifty", "nine zero zero", "many"};
server::ServerError err = mapper->Put(nid, sid, group_id);
ASSERT_EQ(err, server::SERVER_SUCCESS);
err = mapper->Put(nid, std::vector<std::string>(), group_id);
ASSERT_NE(err, server::SERVER_SUCCESS);
std::vector<std::string> res;
err = mapper->Get(nid, res, group_id);
ASSERT_EQ(res.size(), nid.size());
for(size_t i = 0; i < res.size(); i++) {
ASSERT_EQ(res[i], sid[i]);
}
std::string str_id;
err = mapper->Get(nid[1], str_id, group_id);
ASSERT_EQ(str_id, "fifty");
err = mapper->Get(nid[1], str_id);
ASSERT_EQ(str_id, "");
err = mapper->Get(nid[2], str_id, group_id);
ASSERT_EQ(str_id, "nine zero zero");
err = mapper->Delete(nid[2], group_id);
ASSERT_EQ(err, server::SERVER_SUCCESS);
err = mapper->Get(nid[2], str_id, group_id);
ASSERT_EQ(str_id, "");
err = mapper->Get(nid[3], str_id, group_id);
ASSERT_EQ(str_id, "many");
err = mapper->DeleteGroup(group_id);
ASSERT_EQ(err, server::SERVER_SUCCESS);
err = mapper->Get(nid[3], str_id, group_id);
ASSERT_EQ(str_id, "");
std::string ct = CurrentTime();
err = mapper->Put("current_time", ct, "time");
ASSERT_EQ(err, server::SERVER_SUCCESS);
err = mapper->Get("current_time", str_id, "time");
ASSERT_EQ(str_id, ct);
//test performance
nid.clear();
sid.clear();
const int64_t count = 1000000;
{
server::TimeRecorder rc("prepare id data");
for (int64_t i = 0; i < count; i++) {
nid.push_back(std::to_string(i + 100000));
sid.push_back("val_" + std::to_string(i));
}
rc.Record("done!");
}
{
std::string str_info = "Insert " + std::to_string(count) + " k/v into mapper";
server::TimeRecorder rc(str_info);
err = mapper->Put(nid, sid);
ASSERT_EQ(err, server::SERVER_SUCCESS);
rc.Record("done!");
}
{
std::string str_info = "Get " + std::to_string(count) + " k/v from mapper";
server::TimeRecorder rc(str_info);
std::vector<std::string> res;
err = mapper->Get(nid, res);
ASSERT_EQ(res.size(), nid.size());
rc.Record("done!");
}
}