diff --git a/cpp/src/db/DB.h b/cpp/src/db/DB.h index 282e762717..b143f0c233 100644 --- a/cpp/src/db/DB.h +++ b/cpp/src/db/DB.h @@ -46,6 +46,9 @@ public: virtual Status Size(uint64_t& result) = 0; virtual Status BuildIndex(const std::string& table_id) = 0; + virtual Status CreateIndex(const std::string& table_id, const TableIndex& index) = 0; + virtual Status DescribeIndex(const std::string& table_id, TableIndex& index) = 0; + virtual Status DropIndex(const std::string& table_id) = 0; virtual Status DropAll() = 0; diff --git a/cpp/src/db/DBImpl.cpp b/cpp/src/db/DBImpl.cpp index ddee622669..a649257a0b 100644 --- a/cpp/src/db/DBImpl.cpp +++ b/cpp/src/db/DBImpl.cpp @@ -6,6 +6,7 @@ #include "DBImpl.h" #include "src/db/meta/SqliteMetaImpl.h" #include "Log.h" +#include "Utils.h" #include "engine/EngineFactory.h" #include "Factories.h" #include "metrics/Metrics.h" @@ -470,7 +471,8 @@ Status DBImpl::MergeFiles(const std::string& table_id, const meta::DateT& date, } else { table_file.file_type_ = meta::TableFileSchema::RAW; } - table_file.size_ = index_size; + table_file.file_size_ = index->PhysicalSize(); + table_file.row_count_ = index->Count(); updated.push_back(table_file); status = meta_ptr_->UpdateTableFiles(updated); ENGINE_LOG_DEBUG << "New merged file " << table_file.file_id_ << @@ -574,7 +576,58 @@ Status DBImpl::BuildIndex(const std::string& table_id) { times++; } return Status::OK(); - /* return BuildIndexByTable(table_id); */ +} + +Status DBImpl::CreateIndex(const std::string& table_id, const TableIndex& index) { + { + std::unique_lock lock(build_index_mutex_); + + //step 1: check index difference + TableIndex old_index; + auto status = DescribeIndex(table_id, old_index); + if(!status.ok()) { + ENGINE_LOG_ERROR << "Failed to get table index info"; + return status; + } + + if(utils::IsSameIndex(old_index, index)) { + ENGINE_LOG_DEBUG << "Same index setting, no need to create index again"; + return Status::OK(); + } + + //step 2: drop old index files + DropIndex(table_id); + + //step 3: update index info + + status = meta_ptr_->UpdateTableIndexParam(table_id, index); + if (!status.ok()) { + ENGINE_LOG_ERROR << "Failed to update table index info"; + return status; + } + } + + bool has = false; + auto status = meta_ptr_->HasNonIndexFiles(table_id, has); + int times = 1; + + while (has) { + ENGINE_LOG_DEBUG << "Non index files detected! Will build index " << times; + status = meta_ptr_->UpdateTableFilesToIndex(table_id); + /* StartBuildIndexTask(true); */ + std::this_thread::sleep_for(std::chrono::milliseconds(std::min(10*1000, times*100))); + status = meta_ptr_->HasNonIndexFiles(table_id, has); + times++; + } + return Status::OK(); +} + +Status DBImpl::DescribeIndex(const std::string& table_id, TableIndex& index) { + return meta_ptr_->DescribeTableIndex(table_id, index); +} + +Status DBImpl::DropIndex(const std::string& table_id) { + return meta_ptr_->DropTableIndex(table_id); } Status DBImpl::BuildIndex(const meta::TableFileSchema& file) { @@ -650,26 +703,27 @@ Status DBImpl::BuildIndex(const meta::TableFileSchema& file) { //step 6: update meta table_file.file_type_ = meta::TableFileSchema::INDEX; - table_file.size_ = index->Size(); + table_file.file_size_ = index->PhysicalSize(); + table_file.row_count_ = index->Count(); - auto to_remove = file; - to_remove.file_type_ = meta::TableFileSchema::TO_DELETE; + auto origin_file = file; + origin_file.file_type_ = meta::TableFileSchema::BACKUP; - meta::TableFilesSchema update_files = {table_file, to_remove}; + meta::TableFilesSchema update_files = {table_file, origin_file}; status = meta_ptr_->UpdateTableFiles(update_files); if(status.ok()) { ENGINE_LOG_DEBUG << "New index file " << table_file.file_id_ << " of size " << index->PhysicalSize() << " bytes" - << " from file " << to_remove.file_id_; + << " from file " << origin_file.file_id_; if(options_.insert_cache_immediately_) { index->Cache(); } } else { //failed to update meta, mark the new file as to_delete, don't delete old file - to_remove.file_type_ = meta::TableFileSchema::TO_INDEX; - status = meta_ptr_->UpdateTableFile(to_remove); - ENGINE_LOG_DEBUG << "Failed to update file to index, mark file: " << to_remove.file_id_ << " to to_index"; + origin_file.file_type_ = meta::TableFileSchema::TO_INDEX; + status = meta_ptr_->UpdateTableFile(origin_file); + ENGINE_LOG_DEBUG << "Failed to update file to index, mark file: " << origin_file.file_id_ << " to to_index"; table_file.file_type_ = meta::TableFileSchema::TO_DELETE; status = meta_ptr_->UpdateTableFile(table_file); @@ -685,30 +739,6 @@ Status DBImpl::BuildIndex(const meta::TableFileSchema& file) { return Status::OK(); } -Status DBImpl::BuildIndexByTable(const std::string& table_id) { - std::unique_lock lock(build_index_mutex_); - meta::TableFilesSchema to_index_files; - meta_ptr_->FilesToIndex(to_index_files); - - Status status; - - for (auto& file : to_index_files) { - status = BuildIndex(file); - if (!status.ok()) { - ENGINE_LOG_ERROR << "Building index for " << file.id_ << " failed: " << status.ToString(); - return status; - } - ENGINE_LOG_DEBUG << "Sync building index for " << file.id_ << " passed"; - - if (shutting_down_.load(std::memory_order_acquire)){ - ENGINE_LOG_DEBUG << "Server will shutdown, skip build index action for table " << table_id; - break; - } - } - - return status; -} - void DBImpl::BackgroundBuildIndex() { ENGINE_LOG_TRACE << " Background build index thread start"; diff --git a/cpp/src/db/DBImpl.h b/cpp/src/db/DBImpl.h index 4aa7ac07bf..97c36fadbc 100644 --- a/cpp/src/db/DBImpl.h +++ b/cpp/src/db/DBImpl.h @@ -93,6 +93,12 @@ class DBImpl : public DB { Status BuildIndex(const std::string& table_id) override; + Status CreateIndex(const std::string& table_id, const TableIndex& index) override; + + Status DescribeIndex(const std::string& table_id, TableIndex& index) override; + + Status DropIndex(const std::string& table_id) override; + ~DBImpl() override; private: @@ -122,8 +128,6 @@ class DBImpl : public DB { void StartBuildIndexTask(bool force=false); void BackgroundBuildIndex(); - Status - BuildIndexByTable(const std::string& table_id); Status BuildIndex(const meta::TableFileSchema &); diff --git a/cpp/src/db/Types.h b/cpp/src/db/Types.h index acf7feea40..a2ffa606e5 100644 --- a/cpp/src/db/Types.h +++ b/cpp/src/db/Types.h @@ -5,7 +5,10 @@ ******************************************************************************/ #pragma once +#include "db/engine/ExecutionEngine.h" + #include +#include namespace zilliz { namespace milvus { @@ -18,6 +21,12 @@ typedef std::vector IDNumbers; typedef std::vector> QueryResult; typedef std::vector QueryResults; +struct TableIndex { + int32_t engine_type_ = (int)EngineType::FAISS_IDMAP; + int32_t nlist = 16384; + int32_t index_file_size = 1024; //MB + int32_t metric_type = (int)MetricType::L2; +}; } // namespace engine } // namespace milvus diff --git a/cpp/src/db/Utils.cpp b/cpp/src/db/Utils.cpp index 0fe1f76d29..5a0d3cafa2 100644 --- a/cpp/src/db/Utils.cpp +++ b/cpp/src/db/Utils.cpp @@ -142,6 +142,13 @@ Status DeleteTableFilePath(const DBMetaOptions& options, meta::TableFileSchema& return Status::OK(); } +bool IsSameIndex(const TableIndex& index1, const TableIndex& index2) { + return index1.engine_type_ == index2.engine_type_ + && index1.nlist == index2.nlist + && index1.index_file_size == index2.index_file_size + && index1.metric_type == index2.metric_type; +} + } // namespace utils } // namespace engine } // namespace milvus diff --git a/cpp/src/db/Utils.h b/cpp/src/db/Utils.h index 8329f5a1fc..47a8fca9b7 100644 --- a/cpp/src/db/Utils.h +++ b/cpp/src/db/Utils.h @@ -7,6 +7,7 @@ #include "Options.h" #include "db/meta/MetaTypes.h" +#include "db/Types.h" #include @@ -24,6 +25,8 @@ Status CreateTableFilePath(const DBMetaOptions& options, meta::TableFileSchema& Status GetTableFilePath(const DBMetaOptions& options, meta::TableFileSchema& table_file); Status DeleteTableFilePath(const DBMetaOptions& options, meta::TableFileSchema& table_file); +bool IsSameIndex(const TableIndex& index1, const TableIndex& index2); + } // namespace utils } // namespace engine } // namespace milvus diff --git a/cpp/src/db/engine/ExecutionEngine.h b/cpp/src/db/engine/ExecutionEngine.h index 88be75aeb9..0f2cf42b22 100644 --- a/cpp/src/db/engine/ExecutionEngine.h +++ b/cpp/src/db/engine/ExecutionEngine.h @@ -23,6 +23,11 @@ enum class EngineType { MAX_VALUE = NSG_MIX, }; +enum class MetricType { + L2 = 1, + IP = 2, +}; + class ExecutionEngine { public: diff --git a/cpp/src/db/insert/MemTableFile.cpp b/cpp/src/db/insert/MemTableFile.cpp index 326658df5f..3cbb862389 100644 --- a/cpp/src/db/insert/MemTableFile.cpp +++ b/cpp/src/db/insert/MemTableFile.cpp @@ -86,7 +86,7 @@ Status MemTableFile::Serialize() { execution_engine_->Serialize(); auto end_time = METRICS_NOW_TIME; auto total_time = METRICS_MICROSECONDS(start_time, end_time); - table_file_schema_.size_ = size; + table_file_schema_.row_count_ = execution_engine_->Count(); server::Metrics::GetInstance().DiskStoreIOSpeedGaugeSet((double) size / total_time); diff --git a/cpp/src/db/meta/Meta.h b/cpp/src/db/meta/Meta.h index e88761b446..80ae0fb22e 100644 --- a/cpp/src/db/meta/Meta.h +++ b/cpp/src/db/meta/Meta.h @@ -8,6 +8,7 @@ #include "MetaTypes.h" #include "db/Options.h" #include "db/Status.h" +#include "db/Types.h" #include #include @@ -38,6 +39,9 @@ class Meta { virtual Status AllTables(std::vector &table_schema_array) = 0; + virtual Status + UpdateTableIndexParam(const std::string &table_id, const TableIndex& index) = 0; + virtual Status DeleteTable(const std::string &table_id) = 0; @@ -83,6 +87,12 @@ class Meta { virtual Status HasNonIndexFiles(const std::string &table_id, bool &has) = 0; + virtual Status + DescribeTableIndex(const std::string &table_id, TableIndex& index) = 0; + + virtual Status + DropTableIndex(const std::string &table_id) = 0; + virtual Status CleanUp() = 0; diff --git a/cpp/src/db/meta/MetaTypes.h b/cpp/src/db/meta/MetaTypes.h index 9f58734f39..0e554b2330 100644 --- a/cpp/src/db/meta/MetaTypes.h +++ b/cpp/src/db/meta/MetaTypes.h @@ -28,12 +28,13 @@ struct TableSchema { size_t id_ = 0; std::string table_id_; - int state_ = (int)NORMAL; - size_t files_cnt_ = 0; + int32_t state_ = (int)NORMAL; uint16_t dimension_ = 0; - long created_on_ = 0; - int engine_type_ = (int)EngineType::FAISS_IDMAP; - bool store_raw_data_ = false; + int64_t created_on_ = 0; + int32_t engine_type_ = (int)EngineType::FAISS_IDMAP; + int32_t nlist_ = 16384; + int32_t index_file_size_ = 1024; //MB + int32_t metric_type_ = (int)MetricType::L2; }; // TableSchema struct TableFileSchema { @@ -45,19 +46,21 @@ struct TableFileSchema { TO_DELETE, NEW_MERGE, NEW_INDEX, + BACKUP, } FILE_TYPE; size_t id_ = 0; std::string table_id_; - int engine_type_ = (int)EngineType::FAISS_IDMAP; + int32_t engine_type_ = (int)EngineType::FAISS_IDMAP; std::string file_id_; - int file_type_ = NEW; - size_t size_ = 0; + int32_t file_type_ = NEW; + size_t file_size_ = 0; + size_t row_count_ = 0; DateT date_ = EmptyDate; uint16_t dimension_ = 0; std::string location_; - long updated_time_ = 0; - long created_on_ = 0; + int64_t updated_time_ = 0; + int64_t created_on_ = 0; }; // TableFileSchema typedef std::vector TableFilesSchema; diff --git a/cpp/src/db/meta/MySQLMetaImpl.cpp b/cpp/src/db/meta/MySQLMetaImpl.cpp index bf4589351e..fa2697ec32 100644 --- a/cpp/src/db/meta/MySQLMetaImpl.cpp +++ b/cpp/src/db/meta/MySQLMetaImpl.cpp @@ -167,9 +167,10 @@ Status MySQLMetaImpl::Initialize() { "state INT NOT NULL, " << "dimension SMALLINT NOT NULL, " << "created_on BIGINT NOT NULL, " << - "files_cnt BIGINT DEFAULT 0 NOT NULL, " << "engine_type INT DEFAULT 1 NOT NULL, " << - "store_raw_data BOOL DEFAULT false NOT NULL);"; + "nlist INT DEFAULT 16384 NOT NULL, " << + "index_file_size INT DEFAULT 1024 NOT NULL, " << + "metric_type INT DEFAULT 1 NOT NULL);"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::Initialize: " << InitializeQuery.str(); @@ -183,7 +184,8 @@ Status MySQLMetaImpl::Initialize() { "engine_type INT DEFAULT 1 NOT NULL, " << "file_id VARCHAR(255) NOT NULL, " << "file_type INT DEFAULT 0 NOT NULL, " << - "size BIGINT DEFAULT 0 NOT NULL, " << + "file_size BIGINT DEFAULT 0 NOT NULL, " << + "row_count BIGINT DEFAULT 0 NOT NULL, " << "updated_time BIGINT NOT NULL, " << "created_on BIGINT NOT NULL, " << "date INT DEFAULT -1 NOT NULL);"; @@ -325,8 +327,6 @@ Status MySQLMetaImpl::CreateTable(TableSchema &table_schema) { } } - - table_schema.files_cnt_ = 0; table_schema.id_ = -1; table_schema.created_on_ = utils::GetMicroSecTimeStamp(); @@ -336,13 +336,11 @@ Status MySQLMetaImpl::CreateTable(TableSchema &table_schema) { std::string state = std::to_string(table_schema.state_); std::string dimension = std::to_string(table_schema.dimension_); std::string created_on = std::to_string(table_schema.created_on_); - std::string files_cnt = "0"; std::string engine_type = std::to_string(table_schema.engine_type_); - std::string store_raw_data = table_schema.store_raw_data_ ? "true" : "false"; createTableQuery << "INSERT INTO Tables VALUES" << "(" << id << ", " << quote << table_id << ", " << state << ", " << dimension << ", " << - created_on << ", " << files_cnt << ", " << engine_type << ", " << store_raw_data << ");"; + created_on << ", " << engine_type << ");"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::CreateTable: " << createTableQuery.str(); @@ -430,6 +428,18 @@ Status MySQLMetaImpl::HasNonIndexFiles(const std::string &table_id, bool &has) { return Status::OK(); } +Status MySQLMetaImpl::UpdateTableIndexParam(const std::string &table_id, const TableIndex& index) { + return Status::OK(); +} + +Status MySQLMetaImpl::DescribeTableIndex(const std::string &table_id, TableIndex& index) { + return Status::OK(); +} + +Status MySQLMetaImpl::DropTableIndex(const std::string &table_id) { + return Status::OK(); +} + Status MySQLMetaImpl::DeleteTable(const std::string &table_id) { @@ -561,12 +571,7 @@ Status MySQLMetaImpl::DescribeTable(TableSchema &table_schema) { table_schema.dimension_ = resRow["dimension"]; - table_schema.files_cnt_ = resRow["files_cnt"]; - table_schema.engine_type_ = resRow["engine_type"]; - - int store_raw_data = resRow["store_raw_data"]; - table_schema.store_raw_data_ = (store_raw_data == 1); } else { return Status::NotFound("Table " + table_schema.table_id_ + " not found"); } @@ -668,13 +673,8 @@ Status MySQLMetaImpl::AllTables(std::vector &table_schema_array) { table_schema.dimension_ = resRow["dimension"]; - table_schema.files_cnt_ = resRow["files_cnt"]; - table_schema.engine_type_ = resRow["engine_type"]; - int store_raw_data = resRow["store_raw_data"]; - table_schema.store_raw_data_ = (store_raw_data == 1); - table_schema_array.emplace_back(table_schema); } } catch (const BadQuery &er) { @@ -709,7 +709,8 @@ Status MySQLMetaImpl::CreateTableFile(TableFileSchema &file_schema) { NextFileId(file_schema.file_id_); file_schema.dimension_ = table_schema.dimension_; - file_schema.size_ = 0; + file_schema.file_size_ = 0; + file_schema.row_count_ = 0; file_schema.created_on_ = utils::GetMicroSecTimeStamp(); file_schema.updated_time_ = file_schema.created_on_; file_schema.engine_type_ = table_schema.engine_type_; @@ -720,7 +721,7 @@ Status MySQLMetaImpl::CreateTableFile(TableFileSchema &file_schema) { std::string engine_type = std::to_string(file_schema.engine_type_); std::string file_id = file_schema.file_id_; std::string file_type = std::to_string(file_schema.file_type_); - std::string size = std::to_string(file_schema.size_); + std::string row_count = std::to_string(file_schema.row_count_); std::string updated_time = std::to_string(file_schema.updated_time_); std::string created_on = std::to_string(file_schema.created_on_); std::string date = std::to_string(file_schema.date_); @@ -737,7 +738,7 @@ Status MySQLMetaImpl::CreateTableFile(TableFileSchema &file_schema) { createTableFileQuery << "INSERT INTO TableFiles VALUES" << "(" << id << ", " << quote << table_id << ", " << engine_type << ", " << - quote << file_id << ", " << file_type << ", " << size << ", " << + quote << file_id << ", " << file_type << ", " << row_count << ", " << updated_time << ", " << created_on << ", " << date << ");"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::CreateTableFile: " << createTableFileQuery.str(); @@ -792,7 +793,7 @@ Status MySQLMetaImpl::FilesToIndex(TableFilesSchema &files) { Query filesToIndexQuery = connectionPtr->query(); - filesToIndexQuery << "SELECT id, table_id, engine_type, file_id, file_type, size, date " << + filesToIndexQuery << "SELECT id, table_id, engine_type, file_id, file_type, row_count, date " << "FROM TableFiles " << "WHERE file_type = " << std::to_string(TableFileSchema::TO_INDEX) << ";"; @@ -819,7 +820,7 @@ Status MySQLMetaImpl::FilesToIndex(TableFilesSchema &files) { table_file.file_type_ = resRow["file_type"]; - table_file.size_ = resRow["size"]; + table_file.row_count_ = resRow["row_count"]; table_file.date_ = resRow["date"]; @@ -877,7 +878,7 @@ Status MySQLMetaImpl::FilesToSearch(const std::string &table_id, if (partition.empty()) { Query filesToSearchQuery = connectionPtr->query(); - filesToSearchQuery << "SELECT id, table_id, engine_type, file_id, file_type, size, date " << + filesToSearchQuery << "SELECT id, table_id, engine_type, file_id, file_type, row_count, date " << "FROM TableFiles " << "WHERE table_id = " << quote << table_id << " AND " << "(file_type = " << std::to_string(TableFileSchema::RAW) << " OR " << @@ -899,7 +900,7 @@ Status MySQLMetaImpl::FilesToSearch(const std::string &table_id, std::string partitionListStr = partitionListSS.str(); partitionListStr = partitionListStr.substr(0, partitionListStr.size() - 2); //remove the last ", " - filesToSearchQuery << "SELECT id, table_id, engine_type, file_id, file_type, size, date " << + filesToSearchQuery << "SELECT id, table_id, engine_type, file_id, file_type, row_count, date " << "FROM TableFiles " << "WHERE table_id = " << quote << table_id << " AND " << "date IN (" << partitionListStr << ") AND " << @@ -938,7 +939,7 @@ Status MySQLMetaImpl::FilesToSearch(const std::string &table_id, table_file.file_type_ = resRow["file_type"]; - table_file.size_ = resRow["size"]; + table_file.row_count_ = resRow["row_count"]; table_file.date_ = resRow["date"]; @@ -988,7 +989,7 @@ Status MySQLMetaImpl::FilesToSearch(const std::string &table_id, } Query filesToSearchQuery = connectionPtr->query(); - filesToSearchQuery << "SELECT id, table_id, engine_type, file_id, file_type, size, date " << + filesToSearchQuery << "SELECT id, table_id, engine_type, file_id, file_type, row_count, date " << "FROM TableFiles " << "WHERE table_id = " << quote << table_id; @@ -1049,7 +1050,7 @@ Status MySQLMetaImpl::FilesToSearch(const std::string &table_id, table_file.file_type_ = resRow["file_type"]; - table_file.size_ = resRow["size"]; + table_file.row_count_ = resRow["row_count"]; table_file.date_ = resRow["date"]; @@ -1097,11 +1098,11 @@ Status MySQLMetaImpl::FilesToMerge(const std::string &table_id, Query filesToMergeQuery = connectionPtr->query(); - filesToMergeQuery << "SELECT id, table_id, file_id, file_type, size, date " << + filesToMergeQuery << "SELECT id, table_id, file_id, file_type, file_size, date " << "FROM TableFiles " << "WHERE table_id = " << quote << table_id << " AND " << "file_type = " << std::to_string(TableFileSchema::RAW) << " " << - "ORDER BY size DESC" << ";"; + "ORDER BY row_count DESC" << ";"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::FilesToMerge: " << filesToMergeQuery.str(); @@ -1131,7 +1132,7 @@ Status MySQLMetaImpl::FilesToMerge(const std::string &table_id, table_file.file_type_ = resRow["file_type"]; - table_file.size_ = resRow["size"]; + table_file.file_size_ = resRow["file_size"]; table_file.date_ = resRow["date"]; @@ -1189,7 +1190,7 @@ Status MySQLMetaImpl::GetTableFiles(const std::string &table_id, Query getTableFileQuery = connectionPtr->query(); - getTableFileQuery << "SELECT id, engine_type, file_id, file_type, size, date " << + getTableFileQuery << "SELECT id, engine_type, file_id, file_type, file_size, row_count, date " << "FROM TableFiles " << "WHERE table_id = " << quote << table_id << " AND " << "(" << idStr << ");"; @@ -1222,7 +1223,9 @@ Status MySQLMetaImpl::GetTableFiles(const std::string &table_id, file_schema.file_type_ = resRow["file_type"]; - file_schema.size_ = resRow["size"]; + file_schema.file_size_ = resRow["file_size"]; + + file_schema.row_count_ = resRow["row_count"]; file_schema.date_ = resRow["date"]; @@ -1321,7 +1324,7 @@ Status MySQLMetaImpl::Size(uint64_t &result) { Query getSizeQuery = connectionPtr->query(); - getSizeQuery << "SELECT IFNULL(SUM(size),0) AS sum " << + getSizeQuery << "SELECT IFNULL(SUM(file_size),0) AS sum " << "FROM TableFiles " << "WHERE file_type <> " << std::to_string(TableFileSchema::TO_DELETE) << ";"; @@ -1379,7 +1382,7 @@ Status MySQLMetaImpl::DiscardFiles(long long to_discard_size) { Query discardFilesQuery = connectionPtr->query(); - discardFilesQuery << "SELECT id, size " << + discardFilesQuery << "SELECT id, file_size " << "FROM TableFiles " << "WHERE file_type <> " << std::to_string(TableFileSchema::TO_DELETE) << " " << "ORDER BY id ASC " << @@ -1401,11 +1404,11 @@ Status MySQLMetaImpl::DiscardFiles(long long to_discard_size) { break; } table_file.id_ = resRow["id"]; - table_file.size_ = resRow["size"]; + table_file.file_size_ = resRow["file_size"]; idsToDiscardSS << "id = " << std::to_string(table_file.id_) << " OR "; ENGINE_LOG_DEBUG << "Discard table_file.id=" << table_file.file_id_ - << " table_file.size=" << table_file.size_; - to_discard_size -= table_file.size_; + << " table_file.size=" << table_file.file_size_; + to_discard_size -= table_file.file_size_; } std::string idsToDiscardStr = idsToDiscardSS.str(); @@ -1480,7 +1483,8 @@ Status MySQLMetaImpl::UpdateTableFile(TableFileSchema &file_schema) { std::string engine_type = std::to_string(file_schema.engine_type_); std::string file_id = file_schema.file_id_; std::string file_type = std::to_string(file_schema.file_type_); - std::string size = std::to_string(file_schema.size_); + std::string file_size = std::to_string(file_schema.file_size_); + std::string row_count = std::to_string(file_schema.row_count_); std::string updated_time = std::to_string(file_schema.updated_time_); std::string created_on = std::to_string(file_schema.created_on_); std::string date = std::to_string(file_schema.date_); @@ -1490,7 +1494,8 @@ Status MySQLMetaImpl::UpdateTableFile(TableFileSchema &file_schema) { "engine_type = " << engine_type << ", " << "file_id = " << quote << file_id << ", " << "file_type = " << file_type << ", " << - "size = " << size << ", " << + "file_size = " << file_size << ", " << + "row_count = " << row_count << ", " << "updated_time = " << updated_time << ", " << "created_on = " << created_on << ", " << "date = " << date << " " << @@ -1606,7 +1611,8 @@ Status MySQLMetaImpl::UpdateTableFiles(TableFilesSchema &files) { std::string engine_type = std::to_string(file_schema.engine_type_); std::string file_id = file_schema.file_id_; std::string file_type = std::to_string(file_schema.file_type_); - std::string size = std::to_string(file_schema.size_); + std::string file_size = std::to_string(file_schema.file_size_); + std::string row_count = std::to_string(file_schema.row_count_); std::string updated_time = std::to_string(file_schema.updated_time_); std::string created_on = std::to_string(file_schema.created_on_); std::string date = std::to_string(file_schema.date_); @@ -1616,7 +1622,8 @@ Status MySQLMetaImpl::UpdateTableFiles(TableFilesSchema &files) { "engine_type = " << engine_type << ", " << "file_id = " << quote << file_id << ", " << "file_type = " << file_type << ", " << - "size = " << size << ", " << + "file_size = " << file_size << ", " << + "row_count = " << row_count << ", " << "updated_time = " << updated_time << ", " << "created_on = " << created_on << ", " << "date = " << date << " " << diff --git a/cpp/src/db/meta/MySQLMetaImpl.h b/cpp/src/db/meta/MySQLMetaImpl.h index 30695423dd..3fdd80beed 100644 --- a/cpp/src/db/meta/MySQLMetaImpl.h +++ b/cpp/src/db/meta/MySQLMetaImpl.h @@ -43,6 +43,12 @@ class MySQLMetaImpl : public Meta { Status HasNonIndexFiles(const std::string &table_id, bool &has) override; + Status UpdateTableIndexParam(const std::string &table_id, const TableIndex& index) override; + + Status DescribeTableIndex(const std::string &table_id, TableIndex& index) override; + + Status DropTableIndex(const std::string &table_id) override; + Status UpdateTableFile(TableFileSchema &file_schema) override; Status UpdateTableFilesToIndex(const std::string &table_id) override; diff --git a/cpp/src/db/meta/SqliteMetaImpl.cpp b/cpp/src/db/meta/SqliteMetaImpl.cpp index 9053139e0b..9118eadd17 100644 --- a/cpp/src/db/meta/SqliteMetaImpl.cpp +++ b/cpp/src/db/meta/SqliteMetaImpl.cpp @@ -62,16 +62,18 @@ inline auto StoragePrototype(const std::string &path) { make_column("state", &TableSchema::state_), make_column("dimension", &TableSchema::dimension_), make_column("created_on", &TableSchema::created_on_), - make_column("files_cnt", &TableSchema::files_cnt_, default_value(0)), make_column("engine_type", &TableSchema::engine_type_), - make_column("store_raw_data", &TableSchema::store_raw_data_)), + make_column("nlist", &TableSchema::nlist_), + make_column("index_file_size", &TableSchema::index_file_size_), + make_column("metric_type", &TableSchema::metric_type_)), make_table("TableFiles", make_column("id", &TableFileSchema::id_, primary_key()), make_column("table_id", &TableFileSchema::table_id_), make_column("engine_type", &TableFileSchema::engine_type_), make_column("file_id", &TableFileSchema::file_id_), make_column("file_type", &TableFileSchema::file_type_), - make_column("size", &TableFileSchema::size_, default_value(0)), + make_column("file_size", &TableFileSchema::file_size_, default_value(0)), + make_column("row_count", &TableFileSchema::row_count_, default_value(0)), make_column("updated_time", &TableFileSchema::updated_time_), make_column("created_on", &TableFileSchema::created_on_), make_column("date", &TableFileSchema::date_)) @@ -188,7 +190,6 @@ Status SqliteMetaImpl::CreateTable(TableSchema &table_schema) { } } - table_schema.files_cnt_ = 0; table_schema.id_ = -1; table_schema.created_on_ = utils::GetMicroSecTimeStamp(); @@ -218,10 +219,8 @@ Status SqliteMetaImpl::DeleteTable(const std::string& table_id) { //soft delete table auto tables = ConnectorPtr->select(columns(&TableSchema::id_, - &TableSchema::files_cnt_, &TableSchema::dimension_, &TableSchema::engine_type_, - &TableSchema::store_raw_data_, &TableSchema::created_on_), where(c(&TableSchema::table_id_) == table_id)); for (auto &table : tables) { @@ -229,11 +228,9 @@ Status SqliteMetaImpl::DeleteTable(const std::string& table_id) { table_schema.table_id_ = table_id; table_schema.state_ = (int)TableSchema::TO_DELETE; table_schema.id_ = std::get<0>(table); - table_schema.files_cnt_ = std::get<1>(table); - table_schema.dimension_ = std::get<2>(table); - table_schema.engine_type_ = std::get<3>(table); - table_schema.store_raw_data_ = std::get<4>(table); - table_schema.created_on_ = std::get<5>(table); + table_schema.dimension_ = std::get<1>(table); + table_schema.engine_type_ = std::get<2>(table); + table_schema.created_on_ = std::get<3>(table); ConnectorPtr->update(table_schema); } @@ -274,20 +271,15 @@ Status SqliteMetaImpl::DescribeTable(TableSchema &table_schema) { MetricCollector metric; auto groups = ConnectorPtr->select(columns(&TableSchema::id_, - &TableSchema::table_id_, - &TableSchema::files_cnt_, &TableSchema::dimension_, - &TableSchema::engine_type_, - &TableSchema::store_raw_data_), + &TableSchema::engine_type_), where(c(&TableSchema::table_id_) == table_schema.table_id_ and c(&TableSchema::state_) != (int)TableSchema::TO_DELETE)); if (groups.size() == 1) { table_schema.id_ = std::get<0>(groups[0]); - table_schema.files_cnt_ = std::get<2>(groups[0]); - table_schema.dimension_ = std::get<3>(groups[0]); - table_schema.engine_type_ = std::get<4>(groups[0]); - table_schema.store_raw_data_ = std::get<5>(groups[0]); + table_schema.dimension_ = std::get<1>(groups[0]); + table_schema.engine_type_ = std::get<2>(groups[0]); } else { return Status::NotFound("Table " + table_schema.table_id_ + " not found"); } @@ -302,17 +294,16 @@ Status SqliteMetaImpl::DescribeTable(TableSchema &table_schema) { Status SqliteMetaImpl::HasNonIndexFiles(const std::string& table_id, bool& has) { has = false; try { + std::vector file_types = { + (int) TableFileSchema::RAW, + (int) TableFileSchema::NEW, + (int) TableFileSchema::NEW_MERGE, + (int) TableFileSchema::NEW_INDEX, + (int) TableFileSchema::TO_INDEX, + }; auto selected = ConnectorPtr->select(columns(&TableFileSchema::id_, &TableFileSchema::file_type_), - where((c(&TableFileSchema::file_type_) == (int) TableFileSchema::RAW - or - c(&TableFileSchema::file_type_) == (int) TableFileSchema::NEW - or - c(&TableFileSchema::file_type_) == (int) TableFileSchema::NEW_MERGE - or - c(&TableFileSchema::file_type_) == (int) TableFileSchema::NEW_INDEX - or - c(&TableFileSchema::file_type_) == (int) TableFileSchema::TO_INDEX) + where(in(&TableFileSchema::file_type_, file_types) and c(&TableFileSchema::table_id_) == table_id )); @@ -353,6 +344,118 @@ Status SqliteMetaImpl::HasNonIndexFiles(const std::string& table_id, bool& has) return Status::OK(); } +Status SqliteMetaImpl::UpdateTableIndexParam(const std::string &table_id, const TableIndex& index) { + try { + MetricCollector metric; + + //multi-threads call sqlite update may get exception('bad logic', etc), so we add a lock here + std::lock_guard meta_lock(meta_mutex_); + + auto tables = ConnectorPtr->select(columns(&TableSchema::id_, + &TableSchema::state_, + &TableSchema::dimension_, + &TableSchema::created_on_), + where(c(&TableSchema::table_id_) == table_id + and c(&TableSchema::state_) != (int) TableSchema::TO_DELETE)); + + if(tables.size() > 0) { + meta::TableSchema table_schema; + table_schema.id_ = std::get<0>(tables[0]); + table_schema.table_id_ = table_id; + table_schema.state_ = std::get<1>(tables[0]); + table_schema.dimension_ = std::get<2>(tables[0]); + table_schema.created_on_ = std::get<3>(tables[0]); + table_schema.engine_type_ = index.engine_type_; + table_schema.nlist_ = index.nlist; + table_schema.index_file_size_ = index.index_file_size; + table_schema.metric_type_ = index.metric_type; + + ConnectorPtr->update(table_schema); + } else { + return Status::NotFound("Table " + table_id + " not found"); + } + + //set all backup file to raw + ConnectorPtr->update_all( + set( + c(&TableFileSchema::file_type_) = (int) TableFileSchema::RAW, + c(&TableFileSchema::updated_time_) = utils::GetMicroSecTimeStamp() + ), + where( + c(&TableFileSchema::table_id_) == table_id and + c(&TableFileSchema::file_type_) == (int) TableFileSchema::BACKUP + )); + + } catch (std::exception &e) { + std::string msg = "Encounter exception when update table index: table_id = " + table_id; + return HandleException(msg, e); + } + return Status::OK(); +} + +Status SqliteMetaImpl::DescribeTableIndex(const std::string &table_id, TableIndex& index) { + try { + MetricCollector metric; + + auto groups = ConnectorPtr->select(columns(&TableSchema::engine_type_, + &TableSchema::nlist_, + &TableSchema::index_file_size_, + &TableSchema::metric_type_), + where(c(&TableSchema::table_id_) == table_id + and c(&TableSchema::state_) != (int)TableSchema::TO_DELETE)); + + if (groups.size() == 1) { + index.engine_type_ = std::get<0>(groups[0]); + index.nlist = std::get<1>(groups[0]); + index.index_file_size = std::get<2>(groups[0]); + index.metric_type = std::get<3>(groups[0]); + } else { + return Status::NotFound("Table " + table_id + " not found"); + } + + } catch (std::exception &e) { + return HandleException("Encounter exception when describe index", e); + } + + return Status::OK(); +} + +Status SqliteMetaImpl::DropTableIndex(const std::string &table_id) { + try { + MetricCollector metric; + + //multi-threads call sqlite update may get exception('bad logic', etc), so we add a lock here + std::lock_guard meta_lock(meta_mutex_); + + //soft delete index files + ConnectorPtr->update_all( + set( + c(&TableFileSchema::file_type_) = (int) TableFileSchema::TO_DELETE, + c(&TableFileSchema::updated_time_) = utils::GetMicroSecTimeStamp() + ), + where( + c(&TableFileSchema::table_id_) == table_id and + c(&TableFileSchema::file_type_) == (int) TableFileSchema::INDEX + )); + + //set all backup file to raw + ConnectorPtr->update_all( + set( + c(&TableFileSchema::file_type_) = (int) TableFileSchema::RAW, + c(&TableFileSchema::updated_time_) = utils::GetMicroSecTimeStamp() + ), + where( + c(&TableFileSchema::table_id_) == table_id and + c(&TableFileSchema::file_type_) == (int) TableFileSchema::BACKUP + )); + + } catch (std::exception &e) { + return HandleException("Encounter exception when delete table index files", e); + } + + return Status::OK(); +} + Status SqliteMetaImpl::HasTable(const std::string &table_id, bool &has_or_not) { has_or_not = false; @@ -380,19 +483,15 @@ Status SqliteMetaImpl::AllTables(std::vector& table_schema_array) { auto selected = ConnectorPtr->select(columns(&TableSchema::id_, &TableSchema::table_id_, - &TableSchema::files_cnt_, &TableSchema::dimension_, - &TableSchema::engine_type_, - &TableSchema::store_raw_data_), + &TableSchema::engine_type_), where(c(&TableSchema::state_) != (int)TableSchema::TO_DELETE)); for (auto &table : selected) { TableSchema schema; schema.id_ = std::get<0>(table); schema.table_id_ = std::get<1>(table); - schema.files_cnt_ = std::get<2>(table); - schema.dimension_ = std::get<3>(table); - schema.engine_type_ = std::get<4>(table); - schema.store_raw_data_ = std::get<5>(table); + schema.dimension_ = std::get<2>(table); + schema.engine_type_ = std::get<3>(table); table_schema_array.emplace_back(schema); } @@ -420,7 +519,8 @@ Status SqliteMetaImpl::CreateTableFile(TableFileSchema &file_schema) { NextFileId(file_schema.file_id_); file_schema.dimension_ = table_schema.dimension_; - file_schema.size_ = 0; + file_schema.file_size_ = 0; + file_schema.row_count_ = 0; file_schema.created_on_ = utils::GetMicroSecTimeStamp(); file_schema.updated_time_ = file_schema.created_on_; file_schema.engine_type_ = table_schema.engine_type_; @@ -450,7 +550,7 @@ Status SqliteMetaImpl::FilesToIndex(TableFilesSchema &files) { &TableFileSchema::table_id_, &TableFileSchema::file_id_, &TableFileSchema::file_type_, - &TableFileSchema::size_, + &TableFileSchema::row_count_, &TableFileSchema::date_, &TableFileSchema::engine_type_), where(c(&TableFileSchema::file_type_) @@ -464,7 +564,7 @@ Status SqliteMetaImpl::FilesToIndex(TableFilesSchema &files) { table_file.table_id_ = std::get<1>(file); table_file.file_id_ = std::get<2>(file); table_file.file_type_ = std::get<3>(file); - table_file.size_ = std::get<4>(file); + table_file.row_count_ = std::get<4>(file); table_file.date_ = std::get<5>(file); table_file.engine_type_ = std::get<6>(file); @@ -499,19 +599,16 @@ Status SqliteMetaImpl::FilesToSearch(const std::string &table_id, MetricCollector metric; if (partition.empty()) { + std::vector file_type = {(int) TableFileSchema::RAW, (int) TableFileSchema::TO_INDEX, (int) TableFileSchema::INDEX}; auto selected = ConnectorPtr->select(columns(&TableFileSchema::id_, &TableFileSchema::table_id_, &TableFileSchema::file_id_, &TableFileSchema::file_type_, - &TableFileSchema::size_, + &TableFileSchema::row_count_, &TableFileSchema::date_, &TableFileSchema::engine_type_), where(c(&TableFileSchema::table_id_) == table_id and - (c(&TableFileSchema::file_type_) == (int) TableFileSchema::RAW or - c(&TableFileSchema::file_type_) - == (int) TableFileSchema::TO_INDEX or - c(&TableFileSchema::file_type_) - == (int) TableFileSchema::INDEX))); + in(&TableFileSchema::file_type_, file_type))); TableSchema table_schema; table_schema.table_id_ = table_id; @@ -527,7 +624,7 @@ Status SqliteMetaImpl::FilesToSearch(const std::string &table_id, table_file.table_id_ = std::get<1>(file); table_file.file_id_ = std::get<2>(file); table_file.file_type_ = std::get<3>(file); - table_file.size_ = std::get<4>(file); + table_file.row_count_ = std::get<4>(file); table_file.date_ = std::get<5>(file); table_file.engine_type_ = std::get<6>(file); table_file.dimension_ = table_schema.dimension_; @@ -540,20 +637,17 @@ Status SqliteMetaImpl::FilesToSearch(const std::string &table_id, } } else { + std::vector file_type = {(int) TableFileSchema::RAW, (int) TableFileSchema::TO_INDEX, (int) TableFileSchema::INDEX}; auto selected = ConnectorPtr->select(columns(&TableFileSchema::id_, &TableFileSchema::table_id_, &TableFileSchema::file_id_, &TableFileSchema::file_type_, - &TableFileSchema::size_, + &TableFileSchema::row_count_, &TableFileSchema::date_, &TableFileSchema::engine_type_), where(c(&TableFileSchema::table_id_) == table_id and - in(&TableFileSchema::date_, partition) and - (c(&TableFileSchema::file_type_) == (int) TableFileSchema::RAW or - c(&TableFileSchema::file_type_) - == (int) TableFileSchema::TO_INDEX or - c(&TableFileSchema::file_type_) - == (int) TableFileSchema::INDEX))); + in(&TableFileSchema::date_, partition) and + in(&TableFileSchema::file_type_, file_type))); TableSchema table_schema; table_schema.table_id_ = table_id; @@ -569,7 +663,7 @@ Status SqliteMetaImpl::FilesToSearch(const std::string &table_id, table_file.table_id_ = std::get<1>(file); table_file.file_id_ = std::get<2>(file); table_file.file_type_ = std::get<3>(file); - table_file.size_ = std::get<4>(file); + table_file.row_count_ = std::get<4>(file); table_file.date_ = std::get<5>(file); table_file.engine_type_ = std::get<6>(file); table_file.dimension_ = table_schema.dimension_; @@ -601,7 +695,7 @@ Status SqliteMetaImpl::FilesToSearch(const std::string &table_id, &TableFileSchema::table_id_, &TableFileSchema::file_id_, &TableFileSchema::file_type_, - &TableFileSchema::size_, + &TableFileSchema::row_count_, &TableFileSchema::date_, &TableFileSchema::engine_type_); @@ -643,7 +737,7 @@ Status SqliteMetaImpl::FilesToSearch(const std::string &table_id, table_file.table_id_ = std::get<1>(file); table_file.file_id_ = std::get<2>(file); table_file.file_type_ = std::get<3>(file); - table_file.size_ = std::get<4>(file); + table_file.row_count_ = std::get<4>(file); table_file.date_ = std::get<5>(file); table_file.engine_type_ = std::get<6>(file); table_file.dimension_ = table_schema.dimension_; @@ -673,11 +767,11 @@ Status SqliteMetaImpl::FilesToMerge(const std::string &table_id, &TableFileSchema::table_id_, &TableFileSchema::file_id_, &TableFileSchema::file_type_, - &TableFileSchema::size_, + &TableFileSchema::file_size_, &TableFileSchema::date_), where(c(&TableFileSchema::file_type_) == (int) TableFileSchema::RAW and c(&TableFileSchema::table_id_) == table_id), - order_by(&TableFileSchema::size_).desc()); + order_by(&TableFileSchema::file_size_).desc()); TableSchema table_schema; table_schema.table_id_ = table_id; @@ -693,7 +787,7 @@ Status SqliteMetaImpl::FilesToMerge(const std::string &table_id, table_file.table_id_ = std::get<1>(file); table_file.file_id_ = std::get<2>(file); table_file.file_type_ = std::get<3>(file); - table_file.size_ = std::get<4>(file); + table_file.file_size_ = std::get<4>(file); table_file.date_ = std::get<5>(file); table_file.dimension_ = table_schema.dimension_; utils::GetTableFilePath(options_, table_file); @@ -718,7 +812,8 @@ Status SqliteMetaImpl::GetTableFiles(const std::string& table_id, auto files = ConnectorPtr->select(columns(&TableFileSchema::id_, &TableFileSchema::file_id_, &TableFileSchema::file_type_, - &TableFileSchema::size_, + &TableFileSchema::file_size_, + &TableFileSchema::row_count_, &TableFileSchema::date_, &TableFileSchema::engine_type_), where(c(&TableFileSchema::table_id_) == table_id and @@ -738,9 +833,10 @@ Status SqliteMetaImpl::GetTableFiles(const std::string& table_id, file_schema.id_ = std::get<0>(file); file_schema.file_id_ = std::get<1>(file); file_schema.file_type_ = std::get<2>(file); - file_schema.size_ = std::get<3>(file); - file_schema.date_ = std::get<4>(file); - file_schema.engine_type_ = std::get<5>(file); + file_schema.file_size_ = std::get<3>(file); + file_schema.row_count_ = std::get<4>(file); + file_schema.date_ = std::get<5>(file); + file_schema.engine_type_ = std::get<6>(file); file_schema.dimension_ = table_schema.dimension_; utils::GetTableFilePath(options_, file_schema); @@ -797,23 +893,17 @@ Status SqliteMetaImpl::Archive() { Status SqliteMetaImpl::Size(uint64_t &result) { result = 0; try { - auto files = ConnectorPtr->select(columns(&TableFileSchema::size_, - &TableFileSchema::file_type_, - &TableFileSchema::engine_type_), + auto selected = ConnectorPtr->select(columns(sum(&TableFileSchema::file_size_)), where( c(&TableFileSchema::file_type_) != (int) TableFileSchema::TO_DELETE )); - - for (auto &file : files) { - auto file_size = std::get<0>(file); - auto file_type = std::get<1>(file); - auto engine_type = std::get<2>(file); - if(file_type == (int)TableFileSchema::INDEX && engine_type == (int)EngineType::FAISS_IVFSQ8) { - result += (uint64_t)file_size/4;//hardcode for sq8 - } else { - result += (uint64_t)file_size; + for (auto &total_size : selected) { + if (!std::get<0>(total_size)) { + continue; } + result += (uint64_t) (*std::get<0>(total_size)); } + } catch (std::exception &e) { return HandleException("Encounter exception when calculte db size", e); } @@ -836,7 +926,7 @@ Status SqliteMetaImpl::DiscardFiles(long to_discard_size) { auto commited = ConnectorPtr->transaction([&]() mutable { auto selected = ConnectorPtr->select(columns(&TableFileSchema::id_, - &TableFileSchema::size_), + &TableFileSchema::file_size_), where(c(&TableFileSchema::file_type_) != (int) TableFileSchema::TO_DELETE), order_by(&TableFileSchema::id_), @@ -848,11 +938,11 @@ Status SqliteMetaImpl::DiscardFiles(long to_discard_size) { for (auto &file : selected) { if (to_discard_size <= 0) break; table_file.id_ = std::get<0>(file); - table_file.size_ = std::get<1>(file); + table_file.file_size_ = std::get<1>(file); ids.push_back(table_file.id_); ENGINE_LOG_DEBUG << "Discard table_file.id=" << table_file.file_id_ - << " table_file.size=" << table_file.size_; - to_discard_size -= table_file.size_; + << " table_file.size=" << table_file.file_size_; + to_discard_size -= table_file.file_size_; } if (ids.size() == 0) { @@ -1059,12 +1149,8 @@ Status SqliteMetaImpl::CleanUp() { //multi-threads call sqlite update may get exception('bad logic', etc), so we add a lock here std::lock_guard meta_lock(meta_mutex_); - auto files = ConnectorPtr->select(columns(&TableFileSchema::id_), - where(c(&TableFileSchema::file_type_) == (int) TableFileSchema::NEW - or - c(&TableFileSchema::file_type_) == (int) TableFileSchema::NEW_INDEX - or - c(&TableFileSchema::file_type_) == (int) TableFileSchema::NEW_MERGE)); + std::vector file_type = {(int) TableFileSchema::NEW, (int) TableFileSchema::NEW_INDEX, (int) TableFileSchema::NEW_MERGE}; + auto files = ConnectorPtr->select(columns(&TableFileSchema::id_), where(in(&TableFileSchema::file_type_, file_type))); auto commited = ConnectorPtr->transaction([&]() mutable { for (auto &file : files) { @@ -1091,11 +1177,9 @@ Status SqliteMetaImpl::Count(const std::string &table_id, uint64_t &result) { try { MetricCollector metric; - auto selected = ConnectorPtr->select(columns(&TableFileSchema::size_), - where((c(&TableFileSchema::file_type_) == (int) TableFileSchema::RAW - or - c(&TableFileSchema::file_type_) == (int) TableFileSchema::TO_INDEX - or c(&TableFileSchema::file_type_) == (int) TableFileSchema::INDEX) + std::vector file_type = {(int) TableFileSchema::RAW, (int) TableFileSchema::TO_INDEX, (int) TableFileSchema::INDEX}; + auto selected = ConnectorPtr->select(columns(&TableFileSchema::row_count_), + where(in(&TableFileSchema::file_type_, file_type) and c(&TableFileSchema::table_id_) == table_id)); TableSchema table_schema; diff --git a/cpp/src/db/meta/SqliteMetaImpl.h b/cpp/src/db/meta/SqliteMetaImpl.h index 1525f27e4b..34808f202f 100644 --- a/cpp/src/db/meta/SqliteMetaImpl.h +++ b/cpp/src/db/meta/SqliteMetaImpl.h @@ -51,6 +51,15 @@ class SqliteMetaImpl : public Meta { Status HasNonIndexFiles(const std::string &table_id, bool &has) override; + Status + UpdateTableIndexParam(const std::string &table_id, const TableIndex& index) override; + + Status + DescribeTableIndex(const std::string &table_id, TableIndex& index) override; + + Status + DropTableIndex(const std::string &table_id) override; + Status UpdateTableFilesToIndex(const std::string &table_id) override; diff --git a/cpp/src/grpc/cpp_gen.sh b/cpp/src/grpc/cpp_gen.sh index c441783e7b..62b9d95728 100755 --- a/cpp/src/grpc/cpp_gen.sh +++ b/cpp/src/grpc/cpp_gen.sh @@ -1,9 +1,9 @@ #!/bin/bash -/home/yukun/test/milvus/cpp/cmake-build-debug/grpc_ep-prefix/src/grpc_ep/bins/opt/protobuf/protoc -I . --grpc_out=./gen-status --plugin=protoc-gen-grpc="/home/yukun/test/milvus/cpp/cmake-build-debug/grpc_ep-prefix/src/grpc_ep/bins/opt/grpc_cpp_plugin" status.proto +../../cmake-build-debug/grpc_ep-prefix/src/grpc_ep/bins/opt/protobuf/protoc -I . --grpc_out=./gen-status --plugin=protoc-gen-grpc="../../cmake-build-debug/grpc_ep-prefix/src/grpc_ep/bins/opt/grpc_cpp_plugin" status.proto -/home/yukun/test/milvus/cpp/cmake-build-debug/grpc_ep-prefix/src/grpc_ep/bins/opt/protobuf/protoc -I . --cpp_out=./gen-status status.proto +../../cmake-build-debug/grpc_ep-prefix/src/grpc_ep/bins/opt/protobuf/protoc -I . --cpp_out=./gen-status status.proto -/home/yukun/test/milvus/cpp/cmake-build-debug/grpc_ep-prefix/src/grpc_ep/bins/opt/protobuf/protoc -I . --grpc_out=./gen-milvus --plugin=protoc-gen-grpc="/home/yukun/test/milvus/cpp/cmake-build-debug/grpc_ep-prefix/src/grpc_ep/bins/opt/grpc_cpp_plugin" milvus.proto +../../cmake-build-debug/grpc_ep-prefix/src/grpc_ep/bins/opt/protobuf/protoc -I . --grpc_out=./gen-milvus --plugin=protoc-gen-grpc="../../cmake-build-debug/grpc_ep-prefix/src/grpc_ep/bins/opt/grpc_cpp_plugin" milvus.proto -/home/yukun/test/milvus/cpp/cmake-build-debug/grpc_ep-prefix/src/grpc_ep/bins/opt/protobuf/protoc -I . --cpp_out=./gen-milvus milvus.proto \ No newline at end of file +../../cmake-build-debug/grpc_ep-prefix/src/grpc_ep/bins/opt/protobuf/protoc -I . --cpp_out=./gen-milvus milvus.proto \ No newline at end of file diff --git a/cpp/src/grpc/gen-milvus/milvus.pb.cc b/cpp/src/grpc/gen-milvus/milvus.pb.cc index 5ec8fddba0..c8ef38d07b 100644 --- a/cpp/src/grpc/gen-milvus/milvus.pb.cc +++ b/cpp/src/grpc/gen-milvus/milvus.pb.cc @@ -365,9 +365,7 @@ const ::PROTOBUF_NAMESPACE_ID::uint32 TableStruct_milvus_2eproto::offsets[] PROT ~0u, // no _oneof_case_ ~0u, // no _weak_field_map_ PROTOBUF_FIELD_OFFSET(::milvus::grpc::TableSchema, table_name_), - PROTOBUF_FIELD_OFFSET(::milvus::grpc::TableSchema, index_type_), PROTOBUF_FIELD_OFFSET(::milvus::grpc::TableSchema, dimension_), - PROTOBUF_FIELD_OFFSET(::milvus::grpc::TableSchema, store_raw_vector_), ~0u, // no _has_bits_ PROTOBUF_FIELD_OFFSET(::milvus::grpc::Range, _internal_metadata_), ~0u, // no _extensions_ @@ -481,21 +479,21 @@ const ::PROTOBUF_NAMESPACE_ID::uint32 TableStruct_milvus_2eproto::offsets[] PROT static const ::PROTOBUF_NAMESPACE_ID::internal::MigrationSchema schemas[] PROTOBUF_SECTION_VARIABLE(protodesc_cold) = { { 0, -1, sizeof(::milvus::grpc::TableName)}, { 7, -1, sizeof(::milvus::grpc::TableSchema)}, - { 16, -1, sizeof(::milvus::grpc::Range)}, - { 23, -1, sizeof(::milvus::grpc::RowRecord)}, - { 29, -1, sizeof(::milvus::grpc::InsertParam)}, - { 37, -1, sizeof(::milvus::grpc::VectorIds)}, - { 44, -1, sizeof(::milvus::grpc::SearchParam)}, - { 54, -1, sizeof(::milvus::grpc::SearchInFilesParam)}, - { 61, -1, sizeof(::milvus::grpc::QueryResult)}, - { 68, -1, sizeof(::milvus::grpc::TopKQueryResult)}, - { 75, -1, sizeof(::milvus::grpc::StringReply)}, - { 82, -1, sizeof(::milvus::grpc::BoolReply)}, - { 89, -1, sizeof(::milvus::grpc::TableRowCount)}, - { 96, -1, sizeof(::milvus::grpc::Command)}, - { 102, -1, sizeof(::milvus::grpc::Index)}, - { 111, -1, sizeof(::milvus::grpc::IndexParam)}, - { 118, -1, sizeof(::milvus::grpc::DeleteByRangeParam)}, + { 14, -1, sizeof(::milvus::grpc::Range)}, + { 21, -1, sizeof(::milvus::grpc::RowRecord)}, + { 27, -1, sizeof(::milvus::grpc::InsertParam)}, + { 35, -1, sizeof(::milvus::grpc::VectorIds)}, + { 42, -1, sizeof(::milvus::grpc::SearchParam)}, + { 52, -1, sizeof(::milvus::grpc::SearchInFilesParam)}, + { 59, -1, sizeof(::milvus::grpc::QueryResult)}, + { 66, -1, sizeof(::milvus::grpc::TopKQueryResult)}, + { 73, -1, sizeof(::milvus::grpc::StringReply)}, + { 80, -1, sizeof(::milvus::grpc::BoolReply)}, + { 87, -1, sizeof(::milvus::grpc::TableRowCount)}, + { 94, -1, sizeof(::milvus::grpc::Command)}, + { 100, -1, sizeof(::milvus::grpc::Index)}, + { 109, -1, sizeof(::milvus::grpc::IndexParam)}, + { 116, -1, sizeof(::milvus::grpc::DeleteByRangeParam)}, }; static ::PROTOBUF_NAMESPACE_ID::Message const * const file_default_instances[] = { @@ -521,65 +519,64 @@ static ::PROTOBUF_NAMESPACE_ID::Message const * const file_default_instances[] = const char descriptor_table_protodef_milvus_2eproto[] PROTOBUF_SECTION_VARIABLE(protodesc_cold) = "\n\014milvus.proto\022\013milvus.grpc\032\014status.prot" "o\"D\n\tTableName\022#\n\006status\030\001 \001(\0132\023.milvus." - "grpc.Status\022\022\n\ntable_name\030\002 \001(\t\"z\n\013Table" + "grpc.Status\022\022\n\ntable_name\030\002 \001(\t\"L\n\013Table" "Schema\022*\n\ntable_name\030\001 \001(\0132\026.milvus.grpc" - ".TableName\022\022\n\nindex_type\030\002 \001(\005\022\021\n\tdimens" - "ion\030\003 \001(\003\022\030\n\020store_raw_vector\030\004 \001(\010\"/\n\005R" - "ange\022\023\n\013start_value\030\001 \001(\t\022\021\n\tend_value\030\002" - " \001(\t\" \n\tRowRecord\022\023\n\013vector_data\030\001 \003(\002\"i" - "\n\013InsertParam\022\022\n\ntable_name\030\001 \001(\t\0220\n\020row" - "_record_array\030\002 \003(\0132\026.milvus.grpc.RowRec" - "ord\022\024\n\014row_id_array\030\003 \003(\003\"I\n\tVectorIds\022#" - "\n\006status\030\001 \001(\0132\023.milvus.grpc.Status\022\027\n\017v" - "ector_id_array\030\002 \003(\003\"\242\001\n\013SearchParam\022\022\n\n" - "table_name\030\001 \001(\t\0222\n\022query_record_array\030\002" - " \003(\0132\026.milvus.grpc.RowRecord\022-\n\021query_ra" - "nge_array\030\003 \003(\0132\022.milvus.grpc.Range\022\014\n\004t" - "opk\030\004 \001(\003\022\016\n\006nprobe\030\005 \001(\003\"[\n\022SearchInFil" - "esParam\022\025\n\rfile_id_array\030\001 \003(\t\022.\n\014search" - "_param\030\002 \001(\0132\030.milvus.grpc.SearchParam\"+" - "\n\013QueryResult\022\n\n\002id\030\001 \001(\003\022\020\n\010distance\030\002 " - "\001(\001\"m\n\017TopKQueryResult\022#\n\006status\030\001 \001(\0132\023" - ".milvus.grpc.Status\0225\n\023query_result_arra" - "ys\030\002 \003(\0132\030.milvus.grpc.QueryResult\"H\n\013St" - "ringReply\022#\n\006status\030\001 \001(\0132\023.milvus.grpc." - "Status\022\024\n\014string_reply\030\002 \001(\t\"D\n\tBoolRepl" - "y\022#\n\006status\030\001 \001(\0132\023.milvus.grpc.Status\022\022" - "\n\nbool_reply\030\002 \001(\010\"M\n\rTableRowCount\022#\n\006s" - "tatus\030\001 \001(\0132\023.milvus.grpc.Status\022\027\n\017tabl" - "e_row_count\030\002 \001(\003\"\026\n\007Command\022\013\n\003cmd\030\001 \001(" - "\t\"X\n\005Index\022\022\n\nindex_type\030\001 \001(\005\022\r\n\005nlist\030" - "\002 \001(\003\022\027\n\017index_file_size\030\003 \001(\005\022\023\n\013metric" - "_type\030\004 \001(\005\"[\n\nIndexParam\022*\n\ntable_name\030" - "\001 \001(\0132\026.milvus.grpc.TableName\022!\n\005index\030\002" - " \001(\0132\022.milvus.grpc.Index\"K\n\022DeleteByRang" - "eParam\022!\n\005range\030\001 \001(\0132\022.milvus.grpc.Rang" - "e\022\022\n\ntable_name\030\002 \001(\t2\352\007\n\rMilvusService\022" - ">\n\013CreateTable\022\030.milvus.grpc.TableSchema" - "\032\023.milvus.grpc.Status\"\000\022<\n\010HasTable\022\026.mi" - "lvus.grpc.TableName\032\026.milvus.grpc.BoolRe" - "ply\"\000\022:\n\tDropTable\022\026.milvus.grpc.TableNa" - "me\032\023.milvus.grpc.Status\"\000\022=\n\013CreateIndex" - "\022\027.milvus.grpc.IndexParam\032\023.milvus.grpc." - "Status\"\000\022<\n\006Insert\022\030.milvus.grpc.InsertP" - "aram\032\026.milvus.grpc.VectorIds\"\000\022D\n\006Search" - "\022\030.milvus.grpc.SearchParam\032\034.milvus.grpc" - ".TopKQueryResult\"\0000\001\022R\n\rSearchInFiles\022\037." - "milvus.grpc.SearchInFilesParam\032\034.milvus." - "grpc.TopKQueryResult\"\0000\001\022C\n\rDescribeTabl" - "e\022\026.milvus.grpc.TableName\032\030.milvus.grpc." - "TableSchema\"\000\022B\n\nCountTable\022\026.milvus.grp" - "c.TableName\032\032.milvus.grpc.TableRowCount\"" - "\000\022>\n\nShowTables\022\024.milvus.grpc.Command\032\026." - "milvus.grpc.TableName\"\0000\001\0227\n\003Cmd\022\024.milvu" - "s.grpc.Command\032\030.milvus.grpc.StringReply" - "\"\000\022G\n\rDeleteByRange\022\037.milvus.grpc.Delete" - "ByRangeParam\032\023.milvus.grpc.Status\"\000\022=\n\014P" - "reloadTable\022\026.milvus.grpc.TableName\032\023.mi" - "lvus.grpc.Status\"\000\022B\n\rDescribeIndex\022\026.mi" - "lvus.grpc.TableName\032\027.milvus.grpc.IndexP" - "aram\"\000\022:\n\tDropIndex\022\026.milvus.grpc.TableN" - "ame\032\023.milvus.grpc.Status\"\000b\006proto3" + ".TableName\022\021\n\tdimension\030\002 \001(\003\"/\n\005Range\022\023" + "\n\013start_value\030\001 \001(\t\022\021\n\tend_value\030\002 \001(\t\" " + "\n\tRowRecord\022\023\n\013vector_data\030\001 \003(\002\"i\n\013Inse" + "rtParam\022\022\n\ntable_name\030\001 \001(\t\0220\n\020row_recor" + "d_array\030\002 \003(\0132\026.milvus.grpc.RowRecord\022\024\n" + "\014row_id_array\030\003 \003(\003\"I\n\tVectorIds\022#\n\006stat" + "us\030\001 \001(\0132\023.milvus.grpc.Status\022\027\n\017vector_" + "id_array\030\002 \003(\003\"\242\001\n\013SearchParam\022\022\n\ntable_" + "name\030\001 \001(\t\0222\n\022query_record_array\030\002 \003(\0132\026" + ".milvus.grpc.RowRecord\022-\n\021query_range_ar" + "ray\030\003 \003(\0132\022.milvus.grpc.Range\022\014\n\004topk\030\004 " + "\001(\003\022\016\n\006nprobe\030\005 \001(\003\"[\n\022SearchInFilesPara" + "m\022\025\n\rfile_id_array\030\001 \003(\t\022.\n\014search_param" + "\030\002 \001(\0132\030.milvus.grpc.SearchParam\"+\n\013Quer" + "yResult\022\n\n\002id\030\001 \001(\003\022\020\n\010distance\030\002 \001(\001\"m\n" + "\017TopKQueryResult\022#\n\006status\030\001 \001(\0132\023.milvu" + "s.grpc.Status\0225\n\023query_result_arrays\030\002 \003" + "(\0132\030.milvus.grpc.QueryResult\"H\n\013StringRe" + "ply\022#\n\006status\030\001 \001(\0132\023.milvus.grpc.Status" + "\022\024\n\014string_reply\030\002 \001(\t\"D\n\tBoolReply\022#\n\006s" + "tatus\030\001 \001(\0132\023.milvus.grpc.Status\022\022\n\nbool" + "_reply\030\002 \001(\010\"M\n\rTableRowCount\022#\n\006status\030" + "\001 \001(\0132\023.milvus.grpc.Status\022\027\n\017table_row_" + "count\030\002 \001(\003\"\026\n\007Command\022\013\n\003cmd\030\001 \001(\t\"X\n\005I" + "ndex\022\022\n\nindex_type\030\001 \001(\005\022\r\n\005nlist\030\002 \001(\003\022" + "\027\n\017index_file_size\030\003 \001(\005\022\023\n\013metric_type\030" + "\004 \001(\005\"[\n\nIndexParam\022*\n\ntable_name\030\001 \001(\0132" + "\026.milvus.grpc.TableName\022!\n\005index\030\002 \001(\0132\022" + ".milvus.grpc.Index\"K\n\022DeleteByRangeParam" + "\022!\n\005range\030\001 \001(\0132\022.milvus.grpc.Range\022\022\n\nt" + "able_name\030\002 \001(\t2\352\007\n\rMilvusService\022>\n\013Cre" + "ateTable\022\030.milvus.grpc.TableSchema\032\023.mil" + "vus.grpc.Status\"\000\022<\n\010HasTable\022\026.milvus.g" + "rpc.TableName\032\026.milvus.grpc.BoolReply\"\000\022" + ":\n\tDropTable\022\026.milvus.grpc.TableName\032\023.m" + "ilvus.grpc.Status\"\000\022=\n\013CreateIndex\022\027.mil" + "vus.grpc.IndexParam\032\023.milvus.grpc.Status" + "\"\000\022<\n\006Insert\022\030.milvus.grpc.InsertParam\032\026" + ".milvus.grpc.VectorIds\"\000\022D\n\006Search\022\030.mil" + "vus.grpc.SearchParam\032\034.milvus.grpc.TopKQ" + "ueryResult\"\0000\001\022R\n\rSearchInFiles\022\037.milvus" + ".grpc.SearchInFilesParam\032\034.milvus.grpc.T" + "opKQueryResult\"\0000\001\022C\n\rDescribeTable\022\026.mi" + "lvus.grpc.TableName\032\030.milvus.grpc.TableS" + "chema\"\000\022B\n\nCountTable\022\026.milvus.grpc.Tabl" + "eName\032\032.milvus.grpc.TableRowCount\"\000\022>\n\nS" + "howTables\022\024.milvus.grpc.Command\032\026.milvus" + ".grpc.TableName\"\0000\001\0227\n\003Cmd\022\024.milvus.grpc" + ".Command\032\030.milvus.grpc.StringReply\"\000\022G\n\r" + "DeleteByRange\022\037.milvus.grpc.DeleteByRang" + "eParam\032\023.milvus.grpc.Status\"\000\022=\n\014Preload" + "Table\022\026.milvus.grpc.TableName\032\023.milvus.g" + "rpc.Status\"\000\022B\n\rDescribeIndex\022\026.milvus.g" + "rpc.TableName\032\027.milvus.grpc.IndexParam\"\000" + "\022:\n\tDropIndex\022\026.milvus.grpc.TableName\032\023." + "milvus.grpc.Status\"\000b\006proto3" ; static const ::PROTOBUF_NAMESPACE_ID::internal::DescriptorTable*const descriptor_table_milvus_2eproto_deps[1] = { &::descriptor_table_status_2eproto, @@ -606,7 +603,7 @@ static ::PROTOBUF_NAMESPACE_ID::internal::SCCInfoBase*const descriptor_table_mil static ::PROTOBUF_NAMESPACE_ID::internal::once_flag descriptor_table_milvus_2eproto_once; static bool descriptor_table_milvus_2eproto_initialized = false; const ::PROTOBUF_NAMESPACE_ID::internal::DescriptorTable descriptor_table_milvus_2eproto = { - &descriptor_table_milvus_2eproto_initialized, descriptor_table_protodef_milvus_2eproto, "milvus.proto", 2434, + &descriptor_table_milvus_2eproto_initialized, descriptor_table_protodef_milvus_2eproto, "milvus.proto", 2388, &descriptor_table_milvus_2eproto_once, descriptor_table_milvus_2eproto_sccs, descriptor_table_milvus_2eproto_deps, 17, 1, schemas, file_default_instances, TableStruct_milvus_2eproto::offsets, file_level_metadata_milvus_2eproto, 17, file_level_enum_descriptors_milvus_2eproto, file_level_service_descriptors_milvus_2eproto, @@ -981,17 +978,15 @@ TableSchema::TableSchema(const TableSchema& from) } else { table_name_ = nullptr; } - ::memcpy(&dimension_, &from.dimension_, - static_cast(reinterpret_cast(&store_raw_vector_) - - reinterpret_cast(&dimension_)) + sizeof(store_raw_vector_)); + dimension_ = from.dimension_; // @@protoc_insertion_point(copy_constructor:milvus.grpc.TableSchema) } void TableSchema::SharedCtor() { ::PROTOBUF_NAMESPACE_ID::internal::InitSCC(&scc_info_TableSchema_milvus_2eproto.base); ::memset(&table_name_, 0, static_cast( - reinterpret_cast(&store_raw_vector_) - - reinterpret_cast(&table_name_)) + sizeof(store_raw_vector_)); + reinterpret_cast(&dimension_) - + reinterpret_cast(&table_name_)) + sizeof(dimension_)); } TableSchema::~TableSchema() { @@ -1022,9 +1017,7 @@ void TableSchema::Clear() { delete table_name_; } table_name_ = nullptr; - ::memset(&dimension_, 0, static_cast( - reinterpret_cast(&store_raw_vector_) - - reinterpret_cast(&dimension_)) + sizeof(store_raw_vector_)); + dimension_ = PROTOBUF_LONGLONG(0); _internal_metadata_.Clear(); } @@ -1043,27 +1036,13 @@ const char* TableSchema::_InternalParse(const char* ptr, ::PROTOBUF_NAMESPACE_ID CHK_(ptr); } else goto handle_unusual; continue; - // int32 index_type = 2; + // int64 dimension = 2; case 2: if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 16)) { - index_type_ = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint(&ptr); - CHK_(ptr); - } else goto handle_unusual; - continue; - // int64 dimension = 3; - case 3: - if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 24)) { dimension_ = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint(&ptr); CHK_(ptr); } else goto handle_unusual; continue; - // bool store_raw_vector = 4; - case 4: - if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 32)) { - store_raw_vector_ = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint(&ptr); - CHK_(ptr); - } else goto handle_unusual; - continue; default: { handle_unusual: if ((tag & 7) == 4 || tag == 0) { @@ -1105,23 +1084,10 @@ bool TableSchema::MergePartialFromCodedStream( break; } - // int32 index_type = 2; + // int64 dimension = 2; case 2: { if (static_cast< ::PROTOBUF_NAMESPACE_ID::uint8>(tag) == (16 & 0xFF)) { - DO_((::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::ReadPrimitive< - ::PROTOBUF_NAMESPACE_ID::int32, ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::TYPE_INT32>( - input, &index_type_))); - } else { - goto handle_unusual; - } - break; - } - - // int64 dimension = 3; - case 3: { - if (static_cast< ::PROTOBUF_NAMESPACE_ID::uint8>(tag) == (24 & 0xFF)) { - DO_((::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::ReadPrimitive< ::PROTOBUF_NAMESPACE_ID::int64, ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::TYPE_INT64>( input, &dimension_))); @@ -1131,19 +1097,6 @@ bool TableSchema::MergePartialFromCodedStream( break; } - // bool store_raw_vector = 4; - case 4: { - if (static_cast< ::PROTOBUF_NAMESPACE_ID::uint8>(tag) == (32 & 0xFF)) { - - DO_((::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::ReadPrimitive< - bool, ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::TYPE_BOOL>( - input, &store_raw_vector_))); - } else { - goto handle_unusual; - } - break; - } - default: { handle_unusual: if (tag == 0) { @@ -1177,19 +1130,9 @@ void TableSchema::SerializeWithCachedSizes( 1, _Internal::table_name(this), output); } - // int32 index_type = 2; - if (this->index_type() != 0) { - ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt32(2, this->index_type(), output); - } - - // int64 dimension = 3; + // int64 dimension = 2; if (this->dimension() != 0) { - ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt64(3, this->dimension(), output); - } - - // bool store_raw_vector = 4; - if (this->store_raw_vector() != 0) { - ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteBool(4, this->store_raw_vector(), output); + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt64(2, this->dimension(), output); } if (_internal_metadata_.have_unknown_fields()) { @@ -1212,19 +1155,9 @@ void TableSchema::SerializeWithCachedSizes( 1, _Internal::table_name(this), target); } - // int32 index_type = 2; - if (this->index_type() != 0) { - target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt32ToArray(2, this->index_type(), target); - } - - // int64 dimension = 3; + // int64 dimension = 2; if (this->dimension() != 0) { - target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt64ToArray(3, this->dimension(), target); - } - - // bool store_raw_vector = 4; - if (this->store_raw_vector() != 0) { - target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteBoolToArray(4, this->store_raw_vector(), target); + target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt64ToArray(2, this->dimension(), target); } if (_internal_metadata_.have_unknown_fields()) { @@ -1255,25 +1188,13 @@ size_t TableSchema::ByteSizeLong() const { *table_name_); } - // int64 dimension = 3; + // int64 dimension = 2; if (this->dimension() != 0) { total_size += 1 + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::Int64Size( this->dimension()); } - // int32 index_type = 2; - if (this->index_type() != 0) { - total_size += 1 + - ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::Int32Size( - this->index_type()); - } - - // bool store_raw_vector = 4; - if (this->store_raw_vector() != 0) { - total_size += 1 + 1; - } - int cached_size = ::PROTOBUF_NAMESPACE_ID::internal::ToCachedSize(total_size); SetCachedSize(cached_size); return total_size; @@ -1307,12 +1228,6 @@ void TableSchema::MergeFrom(const TableSchema& from) { if (from.dimension() != 0) { set_dimension(from.dimension()); } - if (from.index_type() != 0) { - set_index_type(from.index_type()); - } - if (from.store_raw_vector() != 0) { - set_store_raw_vector(from.store_raw_vector()); - } } void TableSchema::CopyFrom(const ::PROTOBUF_NAMESPACE_ID::Message& from) { @@ -1338,8 +1253,6 @@ void TableSchema::InternalSwap(TableSchema* other) { _internal_metadata_.Swap(&other->_internal_metadata_); swap(table_name_, other->table_name_); swap(dimension_, other->dimension_); - swap(index_type_, other->index_type_); - swap(store_raw_vector_, other->store_raw_vector_); } ::PROTOBUF_NAMESPACE_ID::Metadata TableSchema::GetMetadata() const { diff --git a/cpp/src/grpc/gen-milvus/milvus.pb.h b/cpp/src/grpc/gen-milvus/milvus.pb.h index d4c33b848a..ce34b264fc 100644 --- a/cpp/src/grpc/gen-milvus/milvus.pb.h +++ b/cpp/src/grpc/gen-milvus/milvus.pb.h @@ -395,9 +395,7 @@ class TableSchema : enum : int { kTableNameFieldNumber = 1, - kDimensionFieldNumber = 3, - kIndexTypeFieldNumber = 2, - kStoreRawVectorFieldNumber = 4, + kDimensionFieldNumber = 2, }; // .milvus.grpc.TableName table_name = 1; bool has_table_name() const; @@ -407,21 +405,11 @@ class TableSchema : ::milvus::grpc::TableName* mutable_table_name(); void set_allocated_table_name(::milvus::grpc::TableName* table_name); - // int64 dimension = 3; + // int64 dimension = 2; void clear_dimension(); ::PROTOBUF_NAMESPACE_ID::int64 dimension() const; void set_dimension(::PROTOBUF_NAMESPACE_ID::int64 value); - // int32 index_type = 2; - void clear_index_type(); - ::PROTOBUF_NAMESPACE_ID::int32 index_type() const; - void set_index_type(::PROTOBUF_NAMESPACE_ID::int32 value); - - // bool store_raw_vector = 4; - void clear_store_raw_vector(); - bool store_raw_vector() const; - void set_store_raw_vector(bool value); - // @@protoc_insertion_point(class_scope:milvus.grpc.TableSchema) private: class _Internal; @@ -429,8 +417,6 @@ class TableSchema : ::PROTOBUF_NAMESPACE_ID::internal::InternalMetadataWithArena _internal_metadata_; ::milvus::grpc::TableName* table_name_; ::PROTOBUF_NAMESPACE_ID::int64 dimension_; - ::PROTOBUF_NAMESPACE_ID::int32 index_type_; - bool store_raw_vector_; mutable ::PROTOBUF_NAMESPACE_ID::internal::CachedSize _cached_size_; friend struct ::TableStruct_milvus_2eproto; }; @@ -2820,21 +2806,7 @@ inline void TableSchema::set_allocated_table_name(::milvus::grpc::TableName* tab // @@protoc_insertion_point(field_set_allocated:milvus.grpc.TableSchema.table_name) } -// int32 index_type = 2; -inline void TableSchema::clear_index_type() { - index_type_ = 0; -} -inline ::PROTOBUF_NAMESPACE_ID::int32 TableSchema::index_type() const { - // @@protoc_insertion_point(field_get:milvus.grpc.TableSchema.index_type) - return index_type_; -} -inline void TableSchema::set_index_type(::PROTOBUF_NAMESPACE_ID::int32 value) { - - index_type_ = value; - // @@protoc_insertion_point(field_set:milvus.grpc.TableSchema.index_type) -} - -// int64 dimension = 3; +// int64 dimension = 2; inline void TableSchema::clear_dimension() { dimension_ = PROTOBUF_LONGLONG(0); } @@ -2848,20 +2820,6 @@ inline void TableSchema::set_dimension(::PROTOBUF_NAMESPACE_ID::int64 value) { // @@protoc_insertion_point(field_set:milvus.grpc.TableSchema.dimension) } -// bool store_raw_vector = 4; -inline void TableSchema::clear_store_raw_vector() { - store_raw_vector_ = false; -} -inline bool TableSchema::store_raw_vector() const { - // @@protoc_insertion_point(field_get:milvus.grpc.TableSchema.store_raw_vector) - return store_raw_vector_; -} -inline void TableSchema::set_store_raw_vector(bool value) { - - store_raw_vector_ = value; - // @@protoc_insertion_point(field_set:milvus.grpc.TableSchema.store_raw_vector) -} - // ------------------------------------------------------------------- // Range diff --git a/cpp/src/grpc/milvus.proto b/cpp/src/grpc/milvus.proto index 59d74813d1..47209dc5a1 100644 --- a/cpp/src/grpc/milvus.proto +++ b/cpp/src/grpc/milvus.proto @@ -17,9 +17,7 @@ message TableName { */ message TableSchema { TableName table_name = 1; - int32 index_type = 2; - int64 dimension = 3; - bool store_raw_vector = 4; + int64 dimension = 2; } /** @@ -122,6 +120,8 @@ message Command { /** * @brief Index + * @index_type: 0-invalid, 1-idmap, 2-ivflat, 3-ivfsq8, 4-nsgmix + * @metric_type: 1-L2, 2-IP */ message Index { int32 index_type = 1; diff --git a/cpp/src/grpc/status.proto b/cpp/src/grpc/status.proto index 4e8dc15b74..f5b65c5f77 100644 --- a/cpp/src/grpc/status.proto +++ b/cpp/src/grpc/status.proto @@ -25,6 +25,8 @@ enum ErrorCode { CANNOT_DELETE_FOLDER = 19; CANNOT_DELETE_FILE = 20; BUILD_INDEX_ERROR = 21; + ILLEGAL_NLIST = 22; + ILLEGAL_METRIC_TYPE = 23; } message Status { diff --git a/cpp/src/sdk/examples/grpcsimple/src/ClientTest.cpp b/cpp/src/sdk/examples/grpcsimple/src/ClientTest.cpp index 5225f2a97e..b4f7bf0922 100644 --- a/cpp/src/sdk/examples/grpcsimple/src/ClientTest.cpp +++ b/cpp/src/sdk/examples/grpcsimple/src/ClientTest.cpp @@ -34,9 +34,7 @@ namespace { void PrintTableSchema(const TableSchema& tb_schema) { BLOCK_SPLITER std::cout << "Table name: " << tb_schema.table_name << std::endl; - std::cout << "Table index type: " << (int)tb_schema.index_type << std::endl; std::cout << "Table dimension: " << tb_schema.dimension << std::endl; - std::cout << "Table store raw data: " << (tb_schema.store_raw_vector ? "true" : "false") << std::endl; BLOCK_SPLITER } @@ -88,16 +86,15 @@ namespace { } std::string GetTableName() { - static std::string s_id(CurrentTime()); - return "tbl_" + s_id; +// static std::string s_id(CurrentTime()); +// return "tbl_" + s_id; + return "test"; } TableSchema BuildTableSchema() { TableSchema tb_schema; tb_schema.table_name = TABLE_NAME; - tb_schema.index_type = IndexType::gpu_ivfflat; tb_schema.dimension = TABLE_DIMENSION; - tb_schema.store_raw_vector = true; return tb_schema; } @@ -276,9 +273,19 @@ ClientTest::Test(const std::string& address, const std::string& port) { } {//wait unit build index finish -// std::cout << "Wait until build all index done" << std::endl; -// Status stat = conn->CreateIndex(); -// std::cout << "BuildIndex function call status: " << stat.ToString() << std::endl; + std::cout << "Wait until create all index done" << std::endl; + IndexParam index; + index.table_name = TABLE_NAME; + index.index_type = IndexType::gpu_ivfflat; + index.nlist = 1000; + index.index_file_size = 1024; + index.metric_type = 1; + Status stat = conn->CreateIndex(index); + std::cout << "CreateIndex function call status: " << stat.ToString() << std::endl; + + IndexParam index2; + stat = conn->DescribeIndex(TABLE_NAME, index2); + std::cout << "DescribeIndex function call status: " << stat.ToString() << std::endl; } {//preload table @@ -290,6 +297,11 @@ ClientTest::Test(const std::string& address, const std::string& port) { DoSearch(conn, search_record_array, "Search after build index finish"); } + {//delete index + Status stat = conn->DropIndex(TABLE_NAME); + std::cout << "DropIndex function call status: " << stat.ToString() << std::endl; + } + {//delete table Status stat = conn->DropTable(TABLE_NAME); std::cout << "DeleteTable function call status: " << stat.ToString() << std::endl; diff --git a/cpp/src/sdk/grpc/ClientProxy.cpp b/cpp/src/sdk/grpc/ClientProxy.cpp index 1107da6eff..48c5159bb5 100644 --- a/cpp/src/sdk/grpc/ClientProxy.cpp +++ b/cpp/src/sdk/grpc/ClientProxy.cpp @@ -82,9 +82,7 @@ ClientProxy::CreateTable(const TableSchema ¶m) { try { ::milvus::grpc::TableSchema schema; schema.mutable_table_name()->set_table_name(param.table_name); - schema.set_index_type((int) param.index_type); schema.set_dimension(param.dimension); - schema.set_store_raw_vector(param.store_raw_vector); return client_ptr_->CreateTable(schema); } catch (std::exception &ex) { @@ -119,6 +117,10 @@ ClientProxy::CreateIndex(const IndexParam &index_param) { ::milvus::grpc::IndexParam grpc_index_param; grpc_index_param.mutable_table_name()->set_table_name( index_param.table_name); + grpc_index_param.mutable_index()->set_index_type((int32_t)index_param.index_type); + grpc_index_param.mutable_index()->set_nlist(index_param.nlist); + grpc_index_param.mutable_index()->set_index_file_size(index_param.index_file_size); + grpc_index_param.mutable_index()->set_metric_type(index_param.metric_type); return client_ptr_->CreateIndex(grpc_index_param); } catch (std::exception &ex) { @@ -269,9 +271,7 @@ ClientProxy::DescribeTable(const std::string &table_name, TableSchema &table_sch Status status = client_ptr_->DescribeTable(grpc_schema, table_name); table_schema.table_name = grpc_schema.table_name().table_name(); - table_schema.index_type = (IndexType) grpc_schema.index_type(); table_schema.dimension = grpc_schema.dimension(); - table_schema.store_raw_vector = grpc_schema.store_raw_vector(); return status; } catch (std::exception &ex) { @@ -345,14 +345,35 @@ ClientProxy::PreloadTable(const std::string &table_name) const { } } -IndexParam -ClientProxy::DescribeIndex(const std::string &table_name) const { +Status +ClientProxy::DescribeIndex(const std::string &table_name, IndexParam &index_param) const { + try { + ::milvus::grpc::TableName grpc_table_name; + grpc_table_name.set_table_name(table_name); + ::milvus::grpc::IndexParam grpc_index_param; + Status status = client_ptr_->DescribeIndex(grpc_table_name, grpc_index_param); + index_param.index_type = (IndexType)(grpc_index_param.mutable_index()->index_type()); + index_param.nlist = grpc_index_param.mutable_index()->nlist(); + index_param.index_file_size = grpc_index_param.mutable_index()->index_file_size(); + index_param.metric_type = grpc_index_param.mutable_index()->metric_type(); + return status; + + } catch (std::exception &ex) { + return Status(StatusCode::UnknownError, "fail to describe index: " + std::string(ex.what())); + } } Status ClientProxy::DropIndex(const std::string &table_name) const { - + try { + ::milvus::grpc::TableName grpc_table_name; + grpc_table_name.set_table_name(table_name); + Status status = client_ptr_->DropIndex(grpc_table_name); + return status; + } catch (std::exception &ex) { + return Status(StatusCode::UnknownError, "fail to drop index: " + std::string(ex.what())); + } } } diff --git a/cpp/src/sdk/grpc/ClientProxy.h b/cpp/src/sdk/grpc/ClientProxy.h index f6a39f0445..8a1d34d8e2 100644 --- a/cpp/src/sdk/grpc/ClientProxy.h +++ b/cpp/src/sdk/grpc/ClientProxy.h @@ -75,8 +75,8 @@ public: virtual Status PreloadTable(const std::string &table_name) const override; - virtual IndexParam - DescribeIndex(const std::string &table_name) const override; + virtual Status + DescribeIndex(const std::string &table_name, IndexParam &index_param) const override; virtual Status DropIndex(const std::string &table_name) const override; diff --git a/cpp/src/sdk/grpc/GrpcClient.cpp b/cpp/src/sdk/grpc/GrpcClient.cpp index 00894ea529..77478e5d7e 100644 --- a/cpp/src/sdk/grpc/GrpcClient.cpp +++ b/cpp/src/sdk/grpc/GrpcClient.cpp @@ -270,4 +270,44 @@ GrpcClient::Disconnect() { return Status::OK(); } +Status +GrpcClient::DeleteByRange(grpc::DeleteByRangeParam &delete_by_range_param) { + return Status::OK(); +} + +Status +GrpcClient::DescribeIndex(grpc::TableName &table_name, grpc::IndexParam &index_param) { + ClientContext context; + ::grpc::Status grpc_status = stub_->DescribeIndex(&context, table_name, &index_param); + + if (!grpc_status.ok()) { + std::cerr << "DescribeIndex rpc failed!" << std::endl; + return Status(StatusCode::RPCFailed, grpc_status.error_message()); + } + if (index_param.mutable_table_name()->status().error_code() != grpc::SUCCESS) { + std::cerr << index_param.mutable_table_name()->status().reason() << std::endl; + return Status(StatusCode::ServerFailed, index_param.mutable_table_name()->status().reason()); + } + + return Status::OK(); +} + +Status +GrpcClient::DropIndex(grpc::TableName &table_name) { + ClientContext context; + ::milvus::grpc::Status response; + ::grpc::Status grpc_status = stub_->DropIndex(&context, table_name, &response); + + if (!grpc_status.ok()) { + std::cerr << "DropIndex gRPC failed!" << std::endl; + return Status(StatusCode::RPCFailed, grpc_status.error_message()); + } + + if (response.error_code() != grpc::SUCCESS) { + std::cerr << response.reason() << std::endl; + return Status(StatusCode::ServerFailed, response.reason()); + } + return Status::OK(); +} + } \ No newline at end of file diff --git a/cpp/src/sdk/include/MilvusApi.h b/cpp/src/sdk/include/MilvusApi.h index cb261743e1..766bcbad1d 100644 --- a/cpp/src/sdk/include/MilvusApi.h +++ b/cpp/src/sdk/include/MilvusApi.h @@ -76,9 +76,10 @@ struct TopKQueryResult { */ struct IndexParam { std::string table_name; - int32_t index_type; - int64_t nlist; + IndexType index_type; + int32_t nlist; int32_t index_file_size; + int32_t metric_type; }; /** @@ -354,8 +355,8 @@ class Connection { * * @return index informations and indicate if this operation is successful. */ - virtual IndexParam - DescribeIndex(const std::string &table_name) const = 0; + virtual Status + DescribeIndex(const std::string &table_name, IndexParam &index_param) const = 0; /** * @brief drop index diff --git a/cpp/src/sdk/interface/ConnectionImpl.cpp b/cpp/src/sdk/interface/ConnectionImpl.cpp index b496d1c104..355c06438c 100644 --- a/cpp/src/sdk/interface/ConnectionImpl.cpp +++ b/cpp/src/sdk/interface/ConnectionImpl.cpp @@ -125,14 +125,14 @@ ConnectionImpl::PreloadTable(const std::string &table_name) const { return client_proxy_->PreloadTable(table_name); } -IndexParam -ConnectionImpl::DescribeIndex(const std::string &table_name) const { - +Status +ConnectionImpl::DescribeIndex(const std::string &table_name, IndexParam& index_param) const { + return client_proxy_->DescribeIndex(table_name, index_param); } Status ConnectionImpl::DropIndex(const std::string &table_name) const { - + return client_proxy_->DropIndex(table_name); } } diff --git a/cpp/src/sdk/interface/ConnectionImpl.h b/cpp/src/sdk/interface/ConnectionImpl.h index 24be6060ba..d304736b65 100644 --- a/cpp/src/sdk/interface/ConnectionImpl.h +++ b/cpp/src/sdk/interface/ConnectionImpl.h @@ -81,8 +81,8 @@ public: virtual Status PreloadTable(const std::string &table_name) const override; - virtual IndexParam - DescribeIndex(const std::string &table_name) const override; + virtual Status + DescribeIndex(const std::string &table_name, IndexParam& index_param) const override; virtual Status DropIndex(const std::string &table_name) const override; diff --git a/cpp/src/sdk/thrift/ClientProxy.cpp b/cpp/src/sdk/thrift/ClientProxy.cpp index dff5b98eb6..c258aeefad 100644 --- a/cpp/src/sdk/thrift/ClientProxy.cpp +++ b/cpp/src/sdk/thrift/ClientProxy.cpp @@ -334,8 +334,7 @@ Status ClientProxy::PreloadTable(const std::string &table_name) const { return Status::OK(); } -IndexParam ClientProxy::DescribeIndex(const std::string &table_name) const { - IndexParam index_param; +Status ClientProxy::DescribeIndex(const std::string &table_name, IndexParam &index_param) const { index_param.table_name = table_name; return index_param; } diff --git a/cpp/src/sdk/thrift/ClientProxy.h b/cpp/src/sdk/thrift/ClientProxy.h index 60e070ba8c..27dafc92a6 100644 --- a/cpp/src/sdk/thrift/ClientProxy.h +++ b/cpp/src/sdk/thrift/ClientProxy.h @@ -55,7 +55,7 @@ public: virtual Status PreloadTable(const std::string &table_name) const override; - virtual IndexParam DescribeIndex(const std::string &table_name) const override; + virtual Status DescribeIndex(const std::string &table_name, IndexParam &index_param) const override; virtual Status DropIndex(const std::string &table_name) const override; diff --git a/cpp/src/server/grpc_impl/GrpcRequestHandler.cpp b/cpp/src/server/grpc_impl/GrpcRequestHandler.cpp index 584023e4fc..ad1a0e3d71 100644 --- a/cpp/src/server/grpc_impl/GrpcRequestHandler.cpp +++ b/cpp/src/server/grpc_impl/GrpcRequestHandler.cpp @@ -187,14 +187,24 @@ GrpcRequestHandler::PreloadTable(::grpc::ServerContext *context, GrpcRequestHandler::DescribeIndex(::grpc::ServerContext *context, const ::milvus::grpc::TableName *request, ::milvus::grpc::IndexParam *response) { - + BaseTaskPtr task_ptr = DescribeIndexTask::Create(request->table_name(), *response); + ::milvus::grpc::Status grpc_status; + GrpcRequestScheduler::ExecTask(task_ptr, &grpc_status); + response->mutable_table_name()->mutable_status()->set_reason(grpc_status.reason()); + response->mutable_table_name()->mutable_status()->set_error_code(grpc_status.error_code()); + return ::grpc::Status::OK; } ::grpc::Status GrpcRequestHandler::DropIndex(::grpc::ServerContext *context, const ::milvus::grpc::TableName *request, ::milvus::grpc::Status *response) { - + BaseTaskPtr task_ptr = DropIndexTask::Create(request->table_name()); + ::milvus::grpc::Status grpc_status; + GrpcRequestScheduler::ExecTask(task_ptr, &grpc_status); + response->set_reason(grpc_status.reason()); + response->set_error_code(grpc_status.error_code()); + return ::grpc::Status::OK; } diff --git a/cpp/src/server/grpc_impl/GrpcRequestTask.cpp b/cpp/src/server/grpc_impl/GrpcRequestTask.cpp index 8934045579..10ca2b80ed 100644 --- a/cpp/src/server/grpc_impl/GrpcRequestTask.cpp +++ b/cpp/src/server/grpc_impl/GrpcRequestTask.cpp @@ -130,17 +130,10 @@ CreateTableTask::OnExecute() { return SetError(res, "Invalid table dimension: " + std::to_string(schema_.dimension())); } - res = ValidationUtil::ValidateTableIndexType(schema_.index_type()); - if (res != SERVER_SUCCESS) { - return SetError(res, "Invalid index type: " + std::to_string(schema_.index_type())); - } - //step 2: construct table schema engine::meta::TableSchema table_info; table_info.dimension_ = (uint16_t) schema_.dimension(); table_info.table_id_ = schema_.table_name().table_name(); - table_info.engine_type_ = (int) EngineType(schema_.index_type()); - table_info.store_raw_data_ = schema_.store_raw_vector(); //step 3: create table engine::Status stat = DBWrapper::DB()->CreateTable(table_info); @@ -190,10 +183,7 @@ DescribeTableTask::OnExecute() { } schema_.mutable_table_name()->set_table_name(table_info.table_id_); - - schema_.set_index_type(IndexType((engine::EngineType) table_info.engine_type_)); schema_.set_dimension(table_info.dimension_); - schema_.set_store_raw_vector(table_info.store_raw_data_); } catch (std::exception &ex) { return SetError(SERVER_UNEXPECTED_ERROR, ex.what()); @@ -238,7 +228,12 @@ CreateIndexTask::OnExecute() { } //step 2: check table existence - stat = DBWrapper::DB()->BuildIndex(table_name_); + engine::TableIndex index; + index.engine_type_ = index_param_.mutable_index()->index_type(); + index.nlist = index_param_.mutable_index()->nlist(); + index.index_file_size = index_param_.mutable_index()->index_file_size(); + index.metric_type = index_param_.mutable_index()->metric_type(); + stat = DBWrapper::DB()->CreateIndex(table_name_, index); if (!stat.ok()) { return SetError(SERVER_BUILD_INDEX_ERROR, "Engine failed: " + stat.ToString()); } @@ -758,7 +753,89 @@ PreloadTableTask::OnExecute() { return SERVER_SUCCESS; } +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +DescribeIndexTask::DescribeIndexTask(const std::string &table_name, + ::milvus::grpc::IndexParam &index_param) + : GrpcBaseTask(DDL_DML_TASK_GROUP), + table_name_(table_name), + index_param_(index_param) { +} + +BaseTaskPtr +DescribeIndexTask::Create(const std::string &table_name, + ::milvus::grpc::IndexParam &index_param){ + return std::shared_ptr(new DescribeIndexTask(table_name, index_param)); +} + +ServerError +DescribeIndexTask::OnExecute() { + try { + TimeRecorder rc("DescribeIndexTask"); + + //step 1: check arguments + ServerError res = ValidationUtil::ValidateTableName(table_name_); + if (res != SERVER_SUCCESS) { + return SetError(res, "Invalid table name: " + table_name_); + } + + //step 2: check table existence + engine::TableIndex index; + engine::Status stat = DBWrapper::DB()->DescribeIndex(table_name_, index); + if (!stat.ok()) { + return SetError(DB_META_TRANSACTION_FAILED, "Engine failed: " + stat.ToString()); + } + + index_param_.mutable_table_name()->set_table_name(table_name_); + index_param_.mutable_index()->set_index_type(index.engine_type_); + index_param_.mutable_index()->set_nlist(index.nlist); + index_param_.mutable_index()->set_index_file_size(index.index_file_size); + index_param_.mutable_index()->set_metric_type(index.metric_type); + + rc.ElapseFromBegin("totally cost"); + } catch (std::exception &ex) { + return SetError(SERVER_UNEXPECTED_ERROR, ex.what()); + } + + return SERVER_SUCCESS; +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +DropIndexTask::DropIndexTask(const std::string &table_name) + : GrpcBaseTask(DDL_DML_TASK_GROUP), + table_name_(table_name) { + +} + +BaseTaskPtr +DropIndexTask::Create(const std::string &table_name){ + return std::shared_ptr(new DropIndexTask(table_name)); +} + +ServerError +DropIndexTask::OnExecute() { + try { + TimeRecorder rc("DropIndexTask"); + + //step 1: check arguments + ServerError res = ValidationUtil::ValidateTableName(table_name_); + if (res != SERVER_SUCCESS) { + return SetError(res, "Invalid table name: " + table_name_); + } + + //step 2: check table existence + engine::Status stat = DBWrapper::DB()->DropIndex(table_name_); + if (!stat.ok()) { + return SetError(DB_META_TRANSACTION_FAILED, "Engine failed: " + stat.ToString()); + } + + rc.ElapseFromBegin("totally cost"); + } catch (std::exception &ex) { + return SetError(SERVER_UNEXPECTED_ERROR, ex.what()); + } + + return SERVER_SUCCESS; +} } } diff --git a/cpp/src/server/grpc_impl/GrpcRequestTask.h b/cpp/src/server/grpc_impl/GrpcRequestTask.h index 9a60064e49..e43b9fba60 100644 --- a/cpp/src/server/grpc_impl/GrpcRequestTask.h +++ b/cpp/src/server/grpc_impl/GrpcRequestTask.h @@ -260,6 +260,9 @@ public: protected: DropIndexTask(const std::string &table_name); + ServerError + OnExecute() override; + private: std::string table_name_; diff --git a/cpp/unittest/db/meta_tests.cpp b/cpp/unittest/db/meta_tests.cpp index 0e981f6ae4..4dffeb6db0 100644 --- a/cpp/unittest/db/meta_tests.cpp +++ b/cpp/unittest/db/meta_tests.cpp @@ -180,7 +180,7 @@ TEST_F(MetaTest, ARCHIVE_TEST_DISK) { for (auto i=0; i