diff --git a/CHANGELOG.md b/CHANGELOG.md index 2e585362a7..d88aa18c4c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,16 +12,20 @@ Please mark all change in change log and use the issue from GitHub - \#831 Judge branch error in CommonUtil.cpp - \#977 Server crash when create tables concurrently - \#990 Check gpu resources setting when assign repeated value -- \#995 table count set to 0 if no tables found -- \#1010 improve error message when offset or page_size is equal 0 -- \#1022 check if partition name is legal +- \#995 Table count set to 0 if no tables found +- \#1010 Improve error message when offset or page_size is equal 0 +- \#1022 Check if partition name is valid - \#1028 check if table exists when show partitions - \#1029 check if table exists when try to delete partition - \#1066 optimize http insert and search speed +- \#1022 Check if partition name is legal +- \#1028 Check if table exists when show partitions +- \#1029 Check if table exists when try to delete partition +- \#1066 Optimize http insert and search speed - \#1067 Add binary vectors support in http server - \#1075 Improve error message when page size or offset is illegal - \#1082 Check page_size or offset value to avoid float -- \#1115 http server support load table into memory +- \#1115 Http server support load table into memory - \#1152 Error log output continuously after server start - \#1211 Server down caused by searching with index_type: HNSW - \#1240 Update license declaration @@ -36,6 +40,7 @@ Please mark all change in change log and use the issue from GitHub - \#1507 set_config for insert_buffer_size is wrong - \#1510 Add set interfaces for WAL configurations - \#1511 Fix big integer cannot pass to server correctly +- \#1517 result is not correct when search vectors in multi partition, index type is RNSG - \#1518 Table count did not match after deleting vectors and compact - \#1521 Make cache_insert_data take effect in-service - \#1525 Add setter API for config preload_table @@ -43,6 +48,17 @@ Please mark all change in change log and use the issue from GitHub - \#1530 Set table file with correct engine type in meta - \#1532 Search with ivf_flat failed with open-dataset: sift-256-hamming - \#1535 Degradation searching performance with metric_type: binary_idmap +- \#1549 Fix server/wal config setting bug +- \#1556 Index file not created after table and index created +- \#1560 Search crashed with Super-high dimensional binary vector +- \#1564 Too low recall for glove-200-angular, ivf_pq index +- \#1571 Meta engine type become IDMAP after dropping index for BINARY table +- \#1574 Set all existing bitset in cache when applying deletes +- \#1577 Row count incorrect if delete vectors then create index +- \#1580 Old segment folder not removed after merge/compact if create_index is called before adding data +- \#1590 Server down caused by failure to write file during concurrent mixed operations +- \#1598 Server down during mixed operations +- \#1601 External link bug in HTTP doc ## Feature - \#216 Add CLI to get server info @@ -71,7 +87,7 @@ Please mark all change in change log and use the issue from GitHub - \#738 Use Openblas / lapack from apt install - \#758 Enhance config description - \#791 Remove Arrow -- \#834 add cpu mode for built-in Faiss +- \#834 Add cpu mode for built-in Faiss - \#848 Add ready-to-use config files to the Milvus repo for enhanced user experince - \#860 Remove redundant checks in CacheMgr's constructor - \#908 Move "primary_path" and "secondary_path" to storage config @@ -93,6 +109,7 @@ Please mark all change in change log and use the issue from GitHub - \#1524 Update config "preload_table" description - \#1544 Update resources name in HTTP module - \#1546 Move Config.cpp to config directory +- \#1567 Update yaml config description ## Task - \#1327 Exclude third-party code from codebeat diff --git a/core/conf/demo/server_config.yaml b/core/conf/demo/server_config.yaml index cb15b1487f..52991431e2 100644 --- a/core/conf/demo/server_config.yaml +++ b/core/conf/demo/server_config.yaml @@ -107,10 +107,10 @@ cache_config: # use_blas_threshold | A Milvus performance tuning parameter. This value will be | Integer | 1100 | # | compared with 'nq' to decide if OpenBLAS should be used. | | | # | If nq >= use_blas_threshold, OpenBLAS will be used, search | | | -# | response times will be stable but the search speed will be | | | -# | slower; if nq < use_blas_threshold, SSE will be used, | | | -# | search speed will be faster but search response times will | | | -# | fluctuate. | | | +# | response time will be stable but the search speed will be | | | +# | slower; if nq < use_blas_threshold, AVX or SSE will be | | | +# | used, search speed will be faster but search response time | | | +# | will fluctuate. | | | #----------------------+------------------------------------------------------------+------------+-----------------+ # gpu_search_threshold | A Milvus performance tuning parameter. This value will be | Integer | 1000 | # | compared with 'nq' to decide if the search computation will| | | diff --git a/core/conf/server_cpu_config.template b/core/conf/server_cpu_config.template index e511bc22f7..2a3d52e119 100644 --- a/core/conf/server_cpu_config.template +++ b/core/conf/server_cpu_config.template @@ -107,10 +107,10 @@ cache_config: # use_blas_threshold | A Milvus performance tuning parameter. This value will be | Integer | 1100 | # | compared with 'nq' to decide if OpenBLAS should be used. | | | # | If nq >= use_blas_threshold, OpenBLAS will be used, search | | | -# | response times will be stable but the search speed will be | | | -# | slower; if nq < use_blas_threshold, SSE will be used, | | | -# | search speed will be faster but search response times will | | | -# | fluctuate. | | | +# | response time will be stable but the search speed will be | | | +# | slower; if nq < use_blas_threshold, AVX or SSE will be | | | +# | used, search speed will be faster but search response time | | | +# | will fluctuate. | | | #----------------------+------------------------------------------------------------+------------+-----------------+ # gpu_search_threshold | A Milvus performance tuning parameter. This value will be | Integer | 1000 | # | compared with 'nq' to decide if the search computation will| | | diff --git a/core/conf/server_gpu_config.template b/core/conf/server_gpu_config.template index 2a926f3f6a..593d830c38 100644 --- a/core/conf/server_gpu_config.template +++ b/core/conf/server_gpu_config.template @@ -107,10 +107,10 @@ cache_config: # use_blas_threshold | A Milvus performance tuning parameter. This value will be | Integer | 1100 | # | compared with 'nq' to decide if OpenBLAS should be used. | | | # | If nq >= use_blas_threshold, OpenBLAS will be used, search | | | -# | response times will be stable but the search speed will be | | | -# | slower; if nq < use_blas_threshold, SSE will be used, | | | -# | search speed will be faster but search response times will | | | -# | fluctuate. | | | +# | response time will be stable but the search speed will be | | | +# | slower; if nq < use_blas_threshold, AVX or SSE will be | | | +# | used, search speed will be faster but search response time | | | +# | will fluctuate. | | | #----------------------+------------------------------------------------------------+------------+-----------------+ # gpu_search_threshold | A Milvus performance tuning parameter. This value will be | Integer | 1000 | # | compared with 'nq' to decide if the search computation will| | | diff --git a/core/src/codecs/default/DefaultDeletedDocsFormat.cpp b/core/src/codecs/default/DefaultDeletedDocsFormat.cpp index fb211f6969..0b0039cd4f 100644 --- a/core/src/codecs/default/DefaultDeletedDocsFormat.cpp +++ b/core/src/codecs/default/DefaultDeletedDocsFormat.cpp @@ -43,7 +43,7 @@ DefaultDeletedDocsFormat::read(const store::DirectoryPtr& directory_ptr, segment int del_fd = open(del_file_path.c_str(), O_RDONLY, 00664); if (del_fd == -1) { - std::string err_msg = "Failed to open file: " + del_file_path; + std::string err_msg = "Failed to open file: " + del_file_path + ", error: " + std::strerror(errno); ENGINE_LOG_ERROR << err_msg; throw Exception(SERVER_CANNOT_CREATE_FILE, err_msg); } @@ -91,7 +91,7 @@ DefaultDeletedDocsFormat::write(const store::DirectoryPtr& directory_ptr, const // Write to the temp file, in order to avoid possible race condition with search (concurrent read and write) int del_fd = open(temp_path.c_str(), O_RDWR | O_CREAT, 00664); if (del_fd == -1) { - std::string err_msg = "Failed to open file: " + temp_path; + std::string err_msg = "Failed to open file: " + temp_path + ", error: " + std::strerror(errno); ENGINE_LOG_ERROR << err_msg; throw Exception(SERVER_CANNOT_CREATE_FILE, err_msg); } @@ -130,7 +130,7 @@ DefaultDeletedDocsFormat::write(const store::DirectoryPtr& directory_ptr, const ENGINE_LOG_ERROR << err_msg; throw Exception(SERVER_WRITE_ERROR, err_msg); } - if (::write(del_fd, deleted_docs_list.data(), new_num_bytes) == -1) { + if (::write(del_fd, deleted_docs_list.data(), sizeof(segment::offset_t) * deleted_docs->GetSize()) == -1) { std::string err_msg = "Failed to write to file" + temp_path + ", error: " + std::strerror(errno); ENGINE_LOG_ERROR << err_msg; throw Exception(SERVER_WRITE_ERROR, err_msg); diff --git a/core/src/config/Config.cpp b/core/src/config/Config.cpp index 08eb89592d..dc227d3d71 100644 --- a/core/src/config/Config.cpp +++ b/core/src/config/Config.cpp @@ -667,7 +667,7 @@ Config::CheckConfigVersion(const std::string& value) { std::string msg = "Invalid config version: " + value + ". Expected config version: " + milvus_config_version_map.at(MILVUS_VERSION); SERVER_LOG_ERROR << msg; - // return Status(SERVER_INVALID_ARGUMENT, msg); + return Status(SERVER_INVALID_ARGUMENT, msg); } } return Status::OK(); @@ -1529,8 +1529,7 @@ Status Config::GetStorageConfigS3Enable(bool& value) { std::string str = GetConfigStr(CONFIG_STORAGE, CONFIG_STORAGE_S3_ENABLE, CONFIG_STORAGE_S3_ENABLE_DEFAULT); CONFIG_CHECK(CheckStorageConfigS3Enable(str)); - std::transform(str.begin(), str.end(), str.begin(), ::tolower); - value = (str == "true" || str == "on" || str == "yes" || str == "1"); + CONFIG_CHECK(StringHelpFunctions::ConvertToBoolean(str, value)); return Status::OK(); } @@ -1569,8 +1568,7 @@ Status Config::GetMetricConfigEnableMonitor(bool& value) { std::string str = GetConfigStr(CONFIG_METRIC, CONFIG_METRIC_ENABLE_MONITOR, CONFIG_METRIC_ENABLE_MONITOR_DEFAULT); CONFIG_CHECK(CheckMetricConfigEnableMonitor(str)); - std::transform(str.begin(), str.end(), str.begin(), ::tolower); - value = (str == "true" || str == "on" || str == "yes" || str == "1"); + CONFIG_CHECK(StringHelpFunctions::ConvertToBoolean(str, value)); return Status::OK(); } @@ -1671,8 +1669,7 @@ Status Config::GetGpuResourceConfigEnable(bool& value) { std::string str = GetConfigStr(CONFIG_GPU_RESOURCE, CONFIG_GPU_RESOURCE_ENABLE, CONFIG_GPU_RESOURCE_ENABLE_DEFAULT); CONFIG_CHECK(CheckGpuResourceConfigEnable(str)); - std::transform(str.begin(), str.end(), str.begin(), ::tolower); - value = (str == "true" || str == "on" || str == "yes" || str == "1"); + CONFIG_CHECK(StringHelpFunctions::ConvertToBoolean(str, value)); return Status::OK(); } @@ -1774,8 +1771,7 @@ Status Config::GetWalConfigEnable(bool& wal_enable) { std::string str = GetConfigStr(CONFIG_WAL, CONFIG_WAL_ENABLE, CONFIG_WAL_ENABLE_DEFAULT); CONFIG_CHECK(CheckWalConfigEnable(str)); - std::transform(str.begin(), str.end(), str.begin(), ::tolower); - wal_enable = (str == "true" || str == "on" || str == "yes" || str == "1"); + CONFIG_CHECK(StringHelpFunctions::ConvertToBoolean(str, wal_enable)); return Status::OK(); } @@ -1784,8 +1780,7 @@ Config::GetWalConfigRecoveryErrorIgnore(bool& recovery_error_ignore) { std::string str = GetConfigStr(CONFIG_WAL, CONFIG_WAL_RECOVERY_ERROR_IGNORE, CONFIG_WAL_RECOVERY_ERROR_IGNORE_DEFAULT); CONFIG_CHECK(CheckWalConfigRecoveryErrorIgnore(str)); - std::transform(str.begin(), str.end(), str.begin(), ::tolower); - recovery_error_ignore = (str == "true" || str == "on" || str == "yes" || str == "1"); + CONFIG_CHECK(StringHelpFunctions::ConvertToBoolean(str, recovery_error_ignore)); return Status::OK(); } @@ -2013,7 +2008,7 @@ Config::SetWalConfigEnable(const std::string& value) { Status Config::SetWalConfigRecoveryErrorIgnore(const std::string& value) { CONFIG_CHECK(CheckWalConfigRecoveryErrorIgnore(value)); - return SetConfigValueInMem(CONFIG_WAL, CONFIG_WAL_RECOVERY_ERROR_IGNORE_DEFAULT, value); + return SetConfigValueInMem(CONFIG_WAL, CONFIG_WAL_RECOVERY_ERROR_IGNORE, value); } Status diff --git a/core/src/db/DBImpl.cpp b/core/src/db/DBImpl.cpp index 29ed17308a..f83c0d2b5a 100644 --- a/core/src/db/DBImpl.cpp +++ b/core/src/db/DBImpl.cpp @@ -366,8 +366,8 @@ DBImpl::PreloadTable(const std::string& table_id) { if (file.file_type_ == meta::TableFileSchema::FILE_TYPE::RAW || file.file_type_ == meta::TableFileSchema::FILE_TYPE::TO_INDEX || file.file_type_ == meta::TableFileSchema::FILE_TYPE::BACKUP) { - engine_type = server::ValidationUtil::IsBinaryMetricType(file.metric_type_) ? EngineType::FAISS_BIN_IDMAP - : EngineType::FAISS_IDMAP; + engine_type = + utils::IsBinaryMetricType(file.metric_type_) ? EngineType::FAISS_BIN_IDMAP : EngineType::FAISS_IDMAP; } else { engine_type = (EngineType)file.engine_type_; } @@ -731,7 +731,7 @@ DBImpl::Compact(const std::string& table_id) { break; } } else { - ENGINE_LOG_ERROR << "Segment " << file.segment_id_ << " has no deleted data. No need to compact"; + ENGINE_LOG_DEBUG << "Segment " << file.segment_id_ << " has no deleted data. No need to compact"; } } @@ -739,7 +739,7 @@ DBImpl::Compact(const std::string& table_id) { ENGINE_LOG_DEBUG << "Finished compacting table: " << table_id; } - ENGINE_LOG_ERROR << "Updating meta after compaction..."; + ENGINE_LOG_DEBUG << "Updating meta after compaction..."; /* // Drop index again, in case some files were in the index building process during compacting @@ -1019,7 +1019,7 @@ DBImpl::GetVectorByIdHelper(const std::string& table_id, IDNumber vector_id, Vec auto deleted = std::find(deleted_docs.begin(), deleted_docs.end(), offset); if (deleted == deleted_docs.end()) { // Load raw vector - bool is_binary = server::ValidationUtil::IsBinaryMetricType(file.metric_type_); + bool is_binary = utils::IsBinaryMetricType(file.metric_type_); size_t single_vector_bytes = is_binary ? file.dimension_ / 8 : file.dimension_ * sizeof(float); std::vector raw_vector; status = segment_reader.LoadVectors(offset * single_vector_bytes, single_vector_bytes, raw_vector); diff --git a/core/src/db/Utils.cpp b/core/src/db/Utils.cpp index 9f5f0baae7..89bd523763 100644 --- a/core/src/db/Utils.cpp +++ b/core/src/db/Utils.cpp @@ -220,6 +220,19 @@ IsRawIndexType(int32_t type) { return (type == (int32_t)EngineType::FAISS_IDMAP) || (type == (int32_t)EngineType::FAISS_BIN_IDMAP); } +bool +IsBinaryIndexType(int32_t index_type) { + return (index_type == (int32_t)engine::EngineType::FAISS_BIN_IDMAP) || + (index_type == (int32_t)engine::EngineType::FAISS_BIN_IVFFLAT); +} + +bool +IsBinaryMetricType(int32_t metric_type) { + return (metric_type == (int32_t)engine::MetricType::HAMMING) || + (metric_type == (int32_t)engine::MetricType::JACCARD) || + (metric_type == (int32_t)engine::MetricType::TANIMOTO); +} + meta::DateT GetDate(const std::time_t& t, int day_delta) { struct tm ltm; diff --git a/core/src/db/Utils.h b/core/src/db/Utils.h index 88197d7d5c..c78b4fd717 100644 --- a/core/src/db/Utils.h +++ b/core/src/db/Utils.h @@ -48,6 +48,12 @@ IsSameIndex(const TableIndex& index1, const TableIndex& index2); bool IsRawIndexType(int32_t type); +static bool +IsBinaryIndexType(int32_t index_type); + +bool +IsBinaryMetricType(int32_t metric_type); + meta::DateT GetDate(const std::time_t& t, int day_delta = 0); meta::DateT diff --git a/core/src/db/engine/ExecutionEngineImpl.cpp b/core/src/db/engine/ExecutionEngineImpl.cpp index 7650b6717b..49f3c73b25 100644 --- a/core/src/db/engine/ExecutionEngineImpl.cpp +++ b/core/src/db/engine/ExecutionEngineImpl.cpp @@ -100,9 +100,8 @@ ExecutionEngineImpl::ExecutionEngineImpl(uint16_t dimension, const std::string& index_type_(index_type), metric_type_(metric_type), index_params_(index_params) { - EngineType tmp_index_type = server::ValidationUtil::IsBinaryMetricType((int32_t)metric_type) - ? EngineType::FAISS_BIN_IDMAP - : EngineType::FAISS_IDMAP; + EngineType tmp_index_type = + utils::IsBinaryMetricType((int32_t)metric_type) ? EngineType::FAISS_BIN_IDMAP : EngineType::FAISS_IDMAP; index_ = CreatetVecIndex(tmp_index_type); if (!index_) { throw Exception(DB_ERROR, "Unsupported index type"); diff --git a/core/src/db/insert/MemTable.cpp b/core/src/db/insert/MemTable.cpp index f0bdb7dedf..f91efbabe3 100644 --- a/core/src/db/insert/MemTable.cpp +++ b/core/src/db/insert/MemTable.cpp @@ -236,11 +236,27 @@ MemTable::ApplyDeletes() { utils::GetParentPath(table_file.location_, segment_dir); segment::SegmentReader segment_reader(segment_dir); - auto index = - std::static_pointer_cast(cache::CpuCacheMgr::GetInstance()->GetIndex(table_file.location_)); - faiss::ConcurrentBitsetPtr blacklist = nullptr; - if (index != nullptr) { - status = index->GetBlacklist(blacklist); + auto& segment_id = table_file.segment_id_; + meta::TableFilesSchema segment_files; + status = meta_->GetTableFilesBySegmentId(segment_id, segment_files); + if (!status.ok()) { + break; + } + + // Get all index that contains blacklist in cache + std::vector indexes; + std::vector blacklists; + for (auto& file : segment_files) { + auto index = + std::static_pointer_cast(cache::CpuCacheMgr::GetInstance()->GetIndex(file.location_)); + faiss::ConcurrentBitsetPtr blacklist = nullptr; + if (index != nullptr) { + index->GetBlacklist(blacklist); + if (blacklist != nullptr) { + indexes.emplace_back(index); + blacklists.emplace_back(blacklist); + } + } } std::vector uids; @@ -293,7 +309,7 @@ MemTable::ApplyDeletes() { id_bloom_filter_ptr->Remove(uids[i]); } - if (blacklist != nullptr) { + for (auto& blacklist : blacklists) { if (!blacklist->test(i)) { blacklist->set(i); } @@ -308,8 +324,8 @@ MemTable::ApplyDeletes() { << find_diff.count() << " s in total"; ENGINE_LOG_DEBUG << "Setting deleted docs and bloom filter took " << set_diff.count() << " s in total"; - if (index != nullptr) { - index->SetBlacklist(blacklist); + for (auto i = 0; i < indexes.size(); ++i) { + indexes[i]->SetBlacklist(blacklists[i]); } start = std::chrono::high_resolution_clock::now(); @@ -339,12 +355,6 @@ MemTable::ApplyDeletes() { << " s"; // Update table file row count - auto& segment_id = table_file.segment_id_; - meta::TableFilesSchema segment_files; - status = meta_->GetTableFilesBySegmentId(segment_id, segment_files); - if (!status.ok()) { - break; - } for (auto& file : segment_files) { if (file.file_type_ == meta::TableFileSchema::RAW || file.file_type_ == meta::TableFileSchema::TO_INDEX || file.file_type_ == meta::TableFileSchema::INDEX || file.file_type_ == meta::TableFileSchema::BACKUP) { diff --git a/core/src/db/meta/MetaTypes.h b/core/src/db/meta/MetaTypes.h index d8df8c2622..611df80d98 100644 --- a/core/src/db/meta/MetaTypes.h +++ b/core/src/db/meta/MetaTypes.h @@ -54,7 +54,7 @@ struct TableSchema { int64_t flag_ = 0; int64_t index_file_size_ = DEFAULT_INDEX_FILE_SIZE; int32_t engine_type_ = DEFAULT_ENGINE_TYPE; - std::string index_params_ = "{ \"nlist\": 16384 }"; + std::string index_params_ = "{}"; int32_t metric_type_ = DEFAULT_METRIC_TYPE; std::string owner_table_; std::string partition_tag_; diff --git a/core/src/db/meta/MySQLMetaImpl.cpp b/core/src/db/meta/MySQLMetaImpl.cpp index 6f6e8cdcd3..ca59d82793 100644 --- a/core/src/db/meta/MySQLMetaImpl.cpp +++ b/core/src/db/meta/MySQLMetaImpl.cpp @@ -27,6 +27,7 @@ #include #include #include +#include #include "MetaConsts.h" #include "db/IDGenerator.h" @@ -674,16 +675,7 @@ MySQLMetaImpl::CreateTableFile(TableFileSchema& file_schema) { file_schema.updated_time_ = file_schema.created_on_; file_schema.index_file_size_ = table_schema.index_file_size_; file_schema.index_params_ = table_schema.index_params_; - - if (file_schema.file_type_ == TableFileSchema::FILE_TYPE::NEW || - file_schema.file_type_ == TableFileSchema::FILE_TYPE::NEW_MERGE) { - file_schema.engine_type_ = server::ValidationUtil::IsBinaryMetricType(table_schema.metric_type_) - ? (int32_t)EngineType::FAISS_BIN_IDMAP - : (int32_t)EngineType::FAISS_IDMAP; - } else { - file_schema.engine_type_ = table_schema.engine_type_; - } - + file_schema.engine_type_ = table_schema.engine_type_; file_schema.metric_type_ = table_schema.metric_type_; std::string id = "NULL"; // auto-increment @@ -1335,8 +1327,12 @@ MySQLMetaImpl::DropTableIndex(const std::string& table_id) { } // set table index type to raw - dropTableIndexQuery << "UPDATE " << META_TABLES - << " SET engine_type = " << std::to_string(DEFAULT_ENGINE_TYPE) + dropTableIndexQuery << "UPDATE " << META_TABLES << " SET engine_type = " + << " (CASE" + << " WHEN metric_type in (" << (int32_t)MetricType::HAMMING << " ," + << (int32_t)MetricType::JACCARD << " ," << (int32_t)MetricType::TANIMOTO << ")" + << " THEN " << (int32_t)EngineType::FAISS_BIN_IDMAP << " ELSE " + << (int32_t)EngineType::FAISS_IDMAP << " END)" << " , index_params = '{}'" << " WHERE table_id = " << mysqlpp::quote << table_id << ";"; @@ -2031,6 +2027,7 @@ Status MySQLMetaImpl::CleanUpFilesWithTTL(uint64_t seconds /*, CleanUpFilter* filter*/) { auto now = utils::GetMicroSecTimeStamp(); std::set table_ids; + std::map segment_ids; // remove to_delete files try { @@ -2058,7 +2055,7 @@ MySQLMetaImpl::CleanUpFilesWithTTL(uint64_t seconds /*, CleanUpFilter* filter*/) mysqlpp::StoreQueryResult res = query.store(); TableFileSchema table_file; - std::vector idsToDelete; + std::vector delete_ids; int64_t clean_files = 0; for (auto& resRow : res) { @@ -2083,30 +2080,22 @@ MySQLMetaImpl::CleanUpFilesWithTTL(uint64_t seconds /*, CleanUpFilter* filter*/) server::CommonUtil::EraseFromCache(table_file.location_); if (table_file.file_type_ == (int)TableFileSchema::TO_DELETE) { - // If we are deleting a raw table file, it means it's okay to delete the entire segment directory. - // Else, we can only delete the single file - // TODO(zhiru): We determine whether a table file is raw by its engine type. This is a bit hacky - if (utils::IsRawIndexType(table_file.engine_type_)) { - utils::DeleteSegment(options_, table_file); - std::string segment_dir; - utils::GetParentPath(table_file.location_, segment_dir); - ENGINE_LOG_DEBUG << "Remove segment directory: " << segment_dir; - } else { - utils::DeleteTableFilePath(options_, table_file); - ENGINE_LOG_DEBUG << "Remove table file: " << table_file.location_; - } + // delete file from disk storage + utils::DeleteTableFilePath(options_, table_file); + ENGINE_LOG_DEBUG << "Remove file id:" << table_file.id_ << " location:" << table_file.location_; - idsToDelete.emplace_back(std::to_string(table_file.id_)); + delete_ids.emplace_back(std::to_string(table_file.id_)); table_ids.insert(table_file.table_id_); + segment_ids.insert(std::make_pair(table_file.segment_id_, table_file)); - ++clean_files; + clean_files++; } } // delete file from meta - if (!idsToDelete.empty()) { + if (!delete_ids.empty()) { std::stringstream idsToDeleteSS; - for (auto& id : idsToDelete) { + for (auto& id : delete_ids) { idsToDeleteSS << "id = " << id << " OR "; } @@ -2222,6 +2211,51 @@ MySQLMetaImpl::CleanUpFilesWithTTL(uint64_t seconds /*, CleanUpFilter* filter*/) return HandleException("GENERAL ERROR WHEN CLEANING UP TABLES WITH TTL", e.what()); } + // remove deleted segment folder + // don't remove segment folder until all its tablefiles has been deleted + try { + server::MetricCollector metric; + + { + mysqlpp::ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); + + bool is_null_connection = (connectionPtr == nullptr); + fiu_do_on("MySQLMetaImpl.CleanUpFilesWithTTL.RemoveDeletedSegmentFolder_NUllConnection", + is_null_connection = true); + fiu_do_on("MySQLMetaImpl.CleanUpFilesWithTTL.RemoveDeletedSegmentFolder_ThrowException", + throw std::exception();); + if (is_null_connection) { + return Status(DB_ERROR, "Failed to connect to meta server(mysql)"); + } + + int64_t remove_segments = 0; + for (auto& segment_id : segment_ids) { + mysqlpp::Query query = connectionPtr->query(); + query << "SELECT id" + << " FROM " << META_TABLEFILES << " WHERE segment_id = " << mysqlpp::quote << segment_id.first + << ";"; + + ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUpFilesWithTTL: " << query.str(); + + mysqlpp::StoreQueryResult res = query.store(); + + if (res.empty()) { + utils::DeleteSegment(options_, segment_id.second); + std::string segment_dir; + utils::GetParentPath(segment_id.second.location_, segment_dir); + ENGINE_LOG_DEBUG << "Remove segment directory: " << segment_dir; + ++remove_segments; + } + } + + if (remove_segments > 0) { + ENGINE_LOG_DEBUG << "Remove " << remove_segments << " segments folder"; + } + } + } catch (std::exception& e) { + return HandleException("GENERAL ERROR WHEN CLEANING UP TABLES WITH TTL", e.what()); + } + return Status::OK(); } diff --git a/core/src/db/meta/SqliteMetaImpl.cpp b/core/src/db/meta/SqliteMetaImpl.cpp index 09706ac896..622d36175e 100644 --- a/core/src/db/meta/SqliteMetaImpl.cpp +++ b/core/src/db/meta/SqliteMetaImpl.cpp @@ -69,7 +69,7 @@ StoragePrototype(const std::string& path) { make_column("flag", &TableSchema::flag_, default_value(0)), make_column("index_file_size", &TableSchema::index_file_size_), make_column("engine_type", &TableSchema::engine_type_), - make_column("index_params", &TableSchema::index_params_, default_value("")), + make_column("index_params", &TableSchema::index_params_), make_column("metric_type", &TableSchema::metric_type_), make_column("owner_table", &TableSchema::owner_table_, default_value("")), make_column("partition_tag", &TableSchema::partition_tag_, default_value("")), @@ -375,16 +375,7 @@ SqliteMetaImpl::CreateTableFile(TableFileSchema& file_schema) { file_schema.updated_time_ = file_schema.created_on_; file_schema.index_file_size_ = table_schema.index_file_size_; file_schema.index_params_ = table_schema.index_params_; - - if (file_schema.file_type_ == TableFileSchema::FILE_TYPE::NEW || - file_schema.file_type_ == TableFileSchema::FILE_TYPE::NEW_MERGE) { - file_schema.engine_type_ = server::ValidationUtil::IsBinaryMetricType(table_schema.metric_type_) - ? (int32_t)EngineType::FAISS_BIN_IDMAP - : (int32_t)EngineType::FAISS_IDMAP; - } else { - file_schema.engine_type_ = table_schema.engine_type_; - } - + file_schema.engine_type_ = table_schema.engine_type_; file_schema.metric_type_ = table_schema.metric_type_; // multi-threads call sqlite update may get exception('bad logic', etc), so we add a lock here @@ -813,8 +804,18 @@ SqliteMetaImpl::DropTableIndex(const std::string& table_id) { c(&TableFileSchema::file_type_) == (int)TableFileSchema::BACKUP)); // set table index type to raw + auto groups = ConnectorPtr->select(columns(&TableSchema::metric_type_), + where(c(&TableSchema::table_id_) == table_id)); + + int32_t raw_engine_type = DEFAULT_ENGINE_TYPE; + if (groups.size() == 1) { + int32_t metric_type_ = std::get<0>(groups[0]); + if (engine::utils::IsBinaryMetricType(metric_type_)) { + raw_engine_type = (int32_t)EngineType::FAISS_BIN_IDMAP; + } + } ConnectorPtr->update_all( - set(c(&TableSchema::engine_type_) = DEFAULT_ENGINE_TYPE, c(&TableSchema::index_params_) = "{}"), + set(c(&TableSchema::engine_type_) = raw_engine_type, c(&TableSchema::index_params_) = "{}"), where(c(&TableSchema::table_id_) == table_id)); ENGINE_LOG_DEBUG << "Successfully drop table index, table id = " << table_id; @@ -1198,29 +1199,21 @@ SqliteMetaImpl::FilesByType(const std::string& table_id, const std::vector& file_schema.metric_type_ = table_schema.metric_type_; switch (file_schema.file_type_) { - case (int)TableFileSchema::RAW: - ++raw_count; + case (int)TableFileSchema::RAW:++raw_count; break; - case (int)TableFileSchema::NEW: - ++new_count; + case (int)TableFileSchema::NEW:++new_count; break; - case (int)TableFileSchema::NEW_MERGE: - ++new_merge_count; + case (int)TableFileSchema::NEW_MERGE:++new_merge_count; break; - case (int)TableFileSchema::NEW_INDEX: - ++new_index_count; + case (int)TableFileSchema::NEW_INDEX:++new_index_count; break; - case (int)TableFileSchema::TO_INDEX: - ++to_index_count; + case (int)TableFileSchema::TO_INDEX:++to_index_count; break; - case (int)TableFileSchema::INDEX: - ++index_count; + case (int)TableFileSchema::INDEX:++index_count; break; - case (int)TableFileSchema::BACKUP: - ++backup_count; - break; - default: + case (int)TableFileSchema::BACKUP:++backup_count; break; + default:break; } auto status = utils::GetTableFilePath(options_, file_schema); @@ -1234,29 +1227,25 @@ SqliteMetaImpl::FilesByType(const std::string& table_id, const std::vector& std::string msg = "Get table files by type."; for (int file_type : file_types) { switch (file_type) { - case (int)TableFileSchema::RAW: - msg = msg + " raw files:" + std::to_string(raw_count); + case (int)TableFileSchema::RAW:msg = msg + " raw files:" + std::to_string(raw_count); break; - case (int)TableFileSchema::NEW: - msg = msg + " new files:" + std::to_string(new_count); + case (int)TableFileSchema::NEW:msg = msg + " new files:" + std::to_string(new_count); break; case (int)TableFileSchema::NEW_MERGE: - msg = msg + " new_merge files:" + std::to_string(new_merge_count); + msg = msg + " new_merge files:" + + std::to_string(new_merge_count); break; case (int)TableFileSchema::NEW_INDEX: - msg = msg + " new_index files:" + std::to_string(new_index_count); + msg = msg + " new_index files:" + + std::to_string(new_index_count); break; - case (int)TableFileSchema::TO_INDEX: - msg = msg + " to_index files:" + std::to_string(to_index_count); + case (int)TableFileSchema::TO_INDEX:msg = msg + " to_index files:" + std::to_string(to_index_count); break; - case (int)TableFileSchema::INDEX: - msg = msg + " index files:" + std::to_string(index_count); + case (int)TableFileSchema::INDEX:msg = msg + " index files:" + std::to_string(index_count); break; - case (int)TableFileSchema::BACKUP: - msg = msg + " backup files:" + std::to_string(backup_count); - break; - default: + case (int)TableFileSchema::BACKUP:msg = msg + " backup files:" + std::to_string(backup_count); break; + default:break; } } ENGINE_LOG_DEBUG << msg; @@ -1373,6 +1362,7 @@ Status SqliteMetaImpl::CleanUpFilesWithTTL(uint64_t seconds /*, CleanUpFilter* filter*/) { auto now = utils::GetMicroSecTimeStamp(); std::set table_ids; + std::map segment_ids; // remove to_delete files try { @@ -1422,23 +1412,16 @@ SqliteMetaImpl::CleanUpFilesWithTTL(uint64_t seconds /*, CleanUpFilter* filter*/ server::CommonUtil::EraseFromCache(table_file.location_); if (table_file.file_type_ == (int)TableFileSchema::TO_DELETE) { - // If we are deleting a raw table file, it means it's okay to delete the entire segment directory. - // Else, we can only delete the single file - // TODO(zhiru): We determine whether a table file is raw by its engine type. This is a bit hacky - if (utils::IsRawIndexType(table_file.engine_type_)) { - utils::DeleteSegment(options_, table_file); - std::string segment_dir; - utils::GetParentPath(table_file.location_, segment_dir); - ENGINE_LOG_DEBUG << "Remove segment directory: " << segment_dir; - } else { - utils::DeleteTableFilePath(options_, table_file); - ENGINE_LOG_DEBUG << "Remove table file: " << table_file.location_; - } - // delete file from meta ConnectorPtr->remove(table_file.id_); + // delete file from disk storage + utils::DeleteTableFilePath(options_, table_file); + + ENGINE_LOG_DEBUG << "Remove file id:" << table_file.file_id_ << " location:" + << table_file.location_; table_ids.insert(table_file.table_id_); + segment_ids.insert(std::make_pair(table_file.segment_id_, table_file)); ++clean_files; } @@ -1513,6 +1496,32 @@ SqliteMetaImpl::CleanUpFilesWithTTL(uint64_t seconds /*, CleanUpFilter* filter*/ return HandleException("Encounter exception when delete table folder", e.what()); } + // remove deleted segment folder + // don't remove segment folder until all its tablefiles has been deleted + try { + fiu_do_on("SqliteMetaImpl.CleanUpFilesWithTTL.RemoveSegmentFolder_ThrowException", throw std::exception()); + server::MetricCollector metric; + + int64_t remove_segments = 0; + for (auto& segment_id : segment_ids) { + auto selected = ConnectorPtr->select(columns(&TableFileSchema::id_), + where(c(&TableFileSchema::segment_id_) == segment_id.first)); + if (selected.size() == 0) { + utils::DeleteSegment(options_, segment_id.second); + std::string segment_dir; + utils::GetParentPath(segment_id.second.location_, segment_dir); + ENGINE_LOG_DEBUG << "Remove segment directory: " << segment_dir; + ++remove_segments; + } + } + + if (remove_segments > 0) { + ENGINE_LOG_DEBUG << "Remove " << remove_segments << " segments folder"; + } + } catch (std::exception& e) { + return HandleException("Encounter exception when delete table folder", e.what()); + } + return Status::OK(); } diff --git a/core/src/index/knowhere/knowhere/index/vector_index/IndexIVF.cpp b/core/src/index/knowhere/knowhere/index/vector_index/IndexIVF.cpp index 523cdbaaf3..674638bcca 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/IndexIVF.cpp +++ b/core/src/index/knowhere/knowhere/index/vector_index/IndexIVF.cpp @@ -118,12 +118,12 @@ IVF::Search(const DatasetPtr& dataset, const Config& config) { // std::stringstream ss_res_id, ss_res_dist; // for (int i = 0; i < 10; ++i) { - // printf("%llu", res_ids[i]); + // printf("%llu", p_id[i]); // printf("\n"); - // printf("%.6f", res_dis[i]); + // printf("%.6f", p_dist[i]); // printf("\n"); - // ss_res_id << res_ids[i] << " "; - // ss_res_dist << res_dis[i] << " "; + // ss_res_id << p_id[i] << " "; + // ss_res_dist << p_dist[i] << " "; // } // std::cout << std::endl << "after search: " << std::endl; // std::cout << ss_res_id.str() << std::endl; diff --git a/core/src/index/knowhere/knowhere/index/vector_index/nsg/NSG.cpp b/core/src/index/knowhere/knowhere/index/vector_index/nsg/NSG.cpp index 16d20142b3..e220b42c64 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/nsg/NSG.cpp +++ b/core/src/index/knowhere/knowhere/index/vector_index/nsg/NSG.cpp @@ -698,13 +698,8 @@ NsgIndex::Search(const float* query, const unsigned& nq, const unsigned& dim, co int64_t* ids, SearchParams& params) { std::vector> resset(nq); - if (k >= 45) { - params.search_length = k; - } - TimeRecorder rc("NsgIndex::search", 1); - // TODO(linxj): when to use openmp - if (nq <= 4) { + if (nq == 1) { GetNeighbors(query, resset[0], nsg, ¶ms); } else { #pragma omp parallel for @@ -733,15 +728,6 @@ NsgIndex::Search(const float* query, const unsigned& nq, const unsigned& dim, co } } rc.RecordSection("merge"); - - // ProfilerStart("xx.prof"); - // std::vector resset; - // GetNeighbors(query, resset, nsg, ¶ms); - // for (int i = 0; i < k; ++i) { - // ids[i] = resset[i].id; - // dist[i] = resset[i].distance; - //} - // ProfilerStop(); } void diff --git a/core/src/index/unittest/test_nsg/test_nsg.cpp b/core/src/index/unittest/test_nsg/test_nsg.cpp index 76db0cd8c2..074e3f2736 100644 --- a/core/src/index/unittest/test_nsg/test_nsg.cpp +++ b/core/src/index/unittest/test_nsg/test_nsg.cpp @@ -233,7 +233,7 @@ TEST_F(NSGInterfaceTest, comparetest) { // } // } // } -// printf("R@1 = %.4f\n", n_1 / float(nq)); +// printf("R@1 = %.4f\n", n_1 / float(nq));; // printf("R@10 = %.4f\n", n_10 / float(nq)); // printf("R@100 = %.4f\n", n_100 / float(nq)); //} diff --git a/core/src/scheduler/task/BuildIndexTask.cpp b/core/src/scheduler/task/BuildIndexTask.cpp index 13d3b4a611..feec750e5a 100644 --- a/core/src/scheduler/task/BuildIndexTask.cpp +++ b/core/src/scheduler/task/BuildIndexTask.cpp @@ -12,11 +12,13 @@ #include "scheduler/task/BuildIndexTask.h" #include + #include #include #include #include +#include "db/Utils.h" #include "db/engine/EngineFactory.h" #include "metrics/Metrics.h" #include "scheduler/job/BuildIndexJob.h" @@ -35,8 +37,8 @@ XBuildIndexTask::XBuildIndexTask(TableFileSchemaPtr file, TaskLabelPtr label) if (file->file_type_ == TableFileSchema::FILE_TYPE::RAW || file->file_type_ == TableFileSchema::FILE_TYPE::TO_INDEX || file->file_type_ == TableFileSchema::FILE_TYPE::BACKUP) { - engine_type = server::ValidationUtil::IsBinaryMetricType(file->metric_type_) ? EngineType::FAISS_BIN_IDMAP - : EngineType::FAISS_IDMAP; + engine_type = engine::utils::IsBinaryMetricType(file->metric_type_) ? EngineType::FAISS_BIN_IDMAP + : EngineType::FAISS_IDMAP; } else { engine_type = (EngineType)file->engine_type_; } @@ -206,7 +208,7 @@ XBuildIndexTask::Execute() { // step 6: update meta table_file.file_type_ = engine::meta::TableFileSchema::INDEX; table_file.file_size_ = index->PhysicalSize(); - table_file.row_count_ = index->Count(); + table_file.row_count_ = file_->row_count_; // index->Count(); auto origin_file = *file_; origin_file.file_type_ = engine::meta::TableFileSchema::BACKUP; diff --git a/core/src/scheduler/task/SearchTask.cpp b/core/src/scheduler/task/SearchTask.cpp index 0b32f35ab5..5e71b803e1 100644 --- a/core/src/scheduler/task/SearchTask.cpp +++ b/core/src/scheduler/task/SearchTask.cpp @@ -9,7 +9,10 @@ // is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express // or implied. See the License for the specific language governing permissions and limitations under the License. +#include "scheduler/task/SearchTask.h" + #include + #include #include #include @@ -21,7 +24,6 @@ #include "metrics/Metrics.h" #include "scheduler/SchedInst.h" #include "scheduler/job/SearchJob.h" -#include "scheduler/task/SearchTask.h" #include "segment/SegmentReader.h" #include "utils/Log.h" #include "utils/TimeRecorder.h" @@ -101,7 +103,8 @@ XSearchTask::XSearchTask(const std::shared_ptr& context, TableF if (file_) { // distance -- value 0 means two vectors equal, ascending reduce, L2/HAMMING/JACCARD/TONIMOTO ... // similarity -- infinity value means two vectors equal, descending reduce, IP - if (file_->metric_type_ == static_cast(MetricType::IP)) { + if (file_->metric_type_ == static_cast(MetricType::IP) && + file_->engine_type_ != static_cast(EngineType::FAISS_PQ)) { ascending_reduce = false; } @@ -109,8 +112,8 @@ XSearchTask::XSearchTask(const std::shared_ptr& context, TableF if (file->file_type_ == TableFileSchema::FILE_TYPE::RAW || file->file_type_ == TableFileSchema::FILE_TYPE::TO_INDEX || file->file_type_ == TableFileSchema::FILE_TYPE::BACKUP) { - engine_type = server::ValidationUtil::IsBinaryMetricType(file->metric_type_) ? EngineType::FAISS_BIN_IDMAP - : EngineType::FAISS_IDMAP; + engine_type = engine::utils::IsBinaryMetricType(file->metric_type_) ? EngineType::FAISS_BIN_IDMAP + : EngineType::FAISS_IDMAP; } else { engine_type = (EngineType)file->engine_type_; } @@ -263,13 +266,16 @@ XSearchTask::Execute() { // step 3: pick up topk result auto spec_k = file_->row_count_ < topk ? file_->row_count_ : topk; - if (search_job->GetResultIds().front() == -1 && search_job->GetResultIds().size() > spec_k) { - // initialized results set - search_job->GetResultIds().resize(spec_k); - search_job->GetResultDistances().resize(spec_k); - } + { std::unique_lock lock(search_job->mutex()); + + if (search_job->GetResultIds().front() == -1 && search_job->GetResultIds().size() > spec_k) { + // initialized results set + search_job->GetResultIds().resize(spec_k); + search_job->GetResultDistances().resize(spec_k); + } + XSearchTask::MergeTopkToResultSet(output_ids, output_distance, spec_k, nq, topk, ascending_reduce, search_job->GetResultIds(), search_job->GetResultDistances()); } diff --git a/core/src/server/delivery/request/CreateIndexRequest.cpp b/core/src/server/delivery/request/CreateIndexRequest.cpp index 41edc05631..d1b00c1a2c 100644 --- a/core/src/server/delivery/request/CreateIndexRequest.cpp +++ b/core/src/server/delivery/request/CreateIndexRequest.cpp @@ -11,6 +11,7 @@ #include "server/delivery/request/CreateIndexRequest.h" #include "config/Config.h" +#include "db/Utils.h" #include "server/DBWrapper.h" #include "utils/Log.h" #include "utils/TimeRecorder.h" @@ -83,7 +84,7 @@ CreateIndexRequest::OnExecute() { status = DBWrapper::DB()->DescribeTable(table_info); int32_t adapter_index_type = index_type_; - if (ValidationUtil::IsBinaryMetricType(table_info.metric_type_)) { // binary vector not allow + if (engine::utils::IsBinaryMetricType(table_info.metric_type_)) { // binary vector not allow if (adapter_index_type == static_cast(engine::EngineType::FAISS_IDMAP)) { adapter_index_type = static_cast(engine::EngineType::FAISS_BIN_IDMAP); } else if (adapter_index_type == static_cast(engine::EngineType::FAISS_IVFFLAT)) { diff --git a/core/src/server/delivery/request/CreateTableRequest.cpp b/core/src/server/delivery/request/CreateTableRequest.cpp index 15d07b8cfd..d92db5a94d 100644 --- a/core/src/server/delivery/request/CreateTableRequest.cpp +++ b/core/src/server/delivery/request/CreateTableRequest.cpp @@ -10,6 +10,7 @@ // or implied. See the License for the specific language governing permissions and limitations under the License. #include "server/delivery/request/CreateTableRequest.h" +#include "db/Utils.h" #include "server/DBWrapper.h" #include "server/delivery/request/BaseRequest.h" #include "utils/Log.h" @@ -78,7 +79,7 @@ CreateTableRequest::OnExecute() { table_info.metric_type_ = metric_type_; // some metric type only support binary vector, adapt the index type - if (ValidationUtil::IsBinaryMetricType(metric_type_)) { + if (engine::utils::IsBinaryMetricType(metric_type_)) { if (table_info.engine_type_ == static_cast(engine::EngineType::FAISS_IDMAP)) { table_info.engine_type_ = static_cast(engine::EngineType::FAISS_BIN_IDMAP); } else if (table_info.engine_type_ == static_cast(engine::EngineType::FAISS_IVFFLAT)) { diff --git a/core/src/server/delivery/request/InsertRequest.cpp b/core/src/server/delivery/request/InsertRequest.cpp index 729dc4d292..a739d86b9f 100644 --- a/core/src/server/delivery/request/InsertRequest.cpp +++ b/core/src/server/delivery/request/InsertRequest.cpp @@ -10,6 +10,7 @@ // or implied. See the License for the specific language governing permissions and limitations under the License. #include "server/delivery/request/InsertRequest.h" +#include "db/Utils.h" #include "server/DBWrapper.h" #include "utils/CommonUtil.h" #include "utils/Log.h" @@ -115,7 +116,7 @@ InsertRequest::OnExecute() { #endif // step 4: some metric type doesn't support float vectors if (!vectors_data_.float_data_.empty()) { // insert float vectors - if (ValidationUtil::IsBinaryMetricType(table_schema.metric_type_)) { + if (engine::utils::IsBinaryMetricType(table_schema.metric_type_)) { return Status(SERVER_INVALID_ROWRECORD_ARRAY, "Table metric type doesn't support float vectors."); } @@ -131,7 +132,7 @@ InsertRequest::OnExecute() { "The vector dimension must be equal to the table dimension."); } } else if (!vectors_data_.binary_data_.empty()) { // insert binary vectors - if (!ValidationUtil::IsBinaryMetricType(table_schema.metric_type_)) { + if (!engine::utils::IsBinaryMetricType(table_schema.metric_type_)) { return Status(SERVER_INVALID_ROWRECORD_ARRAY, "Table metric type doesn't support binary vectors."); } diff --git a/core/src/server/delivery/request/SearchRequest.cpp b/core/src/server/delivery/request/SearchRequest.cpp index 0b12f91f5d..e31be87b40 100644 --- a/core/src/server/delivery/request/SearchRequest.cpp +++ b/core/src/server/delivery/request/SearchRequest.cpp @@ -10,6 +10,7 @@ // or implied. See the License for the specific language governing permissions and limitations under the License. #include "server/delivery/request/SearchRequest.h" +#include "db/Utils.h" #include "server/DBWrapper.h" #include "utils/CommonUtil.h" #include "utils/Log.h" @@ -103,7 +104,7 @@ SearchRequest::OnExecute() { rc.RecordSection("check validation"); // step 4: check metric type - if (ValidationUtil::IsBinaryMetricType(table_schema.metric_type_)) { + if (engine::utils::IsBinaryMetricType(table_schema.metric_type_)) { // check prepared binary data if (vectors_data_.binary_data_.size() % vector_count != 0) { return Status(SERVER_INVALID_ROWRECORD_ARRAY, diff --git a/core/src/server/web_impl/README.md b/core/src/server/web_impl/README.md index ac447a529c..3abe5a462c 100644 --- a/core/src/server/web_impl/README.md +++ b/core/src/server/web_impl/README.md @@ -4,34 +4,38 @@ - [Overview](#overview) - [API Reference](#api-reference) - - [`/state`](#state) - - [`/devices`](#devices) - - [`/config/advanced` (GET)](#configadvanced-get) - - [`/config/advanced` (PUT)](#configadvanced-put) - - [`/config/advanced` (OPTIONS)](#configadvanced-options) - - [`/config/gpu_resources` (GET)](#configgpu_resources-get) - - [`/config/gpu_resources` (PUT)](#configgpu_resources-put) - - [`/config/gpu_resources` (OPTIONS)](#configgpu_resources-options) - - [`/tables` (GET)](#tables-get) - - [`/tables` (POST)](#tables-post) - - [`/tables` (OPTIONS)](#tables-options) - - [`/tables/{table_name}` (GET)](#tablestable_name-get) - - [`/tables/{table_name}` (DELETE)](#tablestable_name-delete) - - [`/tables/{table_name}` (OPTIONS)](#tablestable_name-options) - - [`/tables/{table_name}/indexes` (GET)](#tablestable_nameindexes-get) - - [`/tables/{table_name}/indexes` (POST)](#tablestable_nameindexes-post) - - [`/tables/{table_name}/indexes` (DELETE)](#tablestable_nameindexes-delete) - - [`/tables/{table_name}/indexes` (OPTIONS)](#tablestable_nameindexes-options) - - [`/tables/{table_name}/partitions` (GET)](#tablestable_namepartitions-get) - - [`/tables/{table_name}/partitions` (POST)](#tablestable_namepartitions-post) - - [`/tables/{table_name}/partitions` (OPTIONS)](#tablestable_namepartitions-options) - - [`/tables/{table_name}/partitions/{partition_tag}` (DELETE)](#tablestable_namepartitionspartition_tag-delete) - - [`/tables/{table_name}/partitions/{partition_tag}` (OPTIONS)](#tablestable_namepartitionspartition_tag-options) - - [`/tables/{table_name}/vectors` (PUT)](#tablestable_namevectors-put) - - [`/tables/{table_name}/vectors` (POST)](#tablestable_namevectors-post) - - [`/tables/{table_name}/vectors` (OPTIONS)](#tablestable_namevectors-options) - - [`/system/{msg}` (GET)](#systemmsg-get) -- [Error Codes](#error-codes) + - [`/state`](#state) + - [`/devices`](#devices) + - [`/config/advanced` (GET)](#configadvanced-get) + - [`/config/advanced` (PUT)](#configadvanced-put) + - [`/config/advanced` (OPTIONS)](#configadvanced-options) + - [`/config/gpu_resources` (GET)](#configgpu_resources-get) + - [`/config/gpu_resources` (PUT)](#configgpu_resources-put) + - [`/config/gpu_resources` (OPTIONS)](#configgpu_resources-options) + - [`/collections` (GET)](#collections-get) + - [`/collections` (POST)](#collections-post) + - [`/collections` (OPTIONS)](#collections-options) + - [`/collections/{collection_name}` (GET)](#collectionscollection_name-get) + - [`/collections/{collection_name}` (DELETE)](#collectionscollection_name-delete) + - [`/collections/{collection_name}` (OPTIONS)](#collectionscollection_name-options) + - [`/collections/{collection_name}/indexes` (GET)](#collectionscollection_nameindexes-get) + - [`/collections/{collection_name}/indexes` (POST)](#collectionscollection_nameindexes-post) + - [`/collections/{collection_name}/indexes` (DELETE)](#collectionscollection_nameindexes-delete) + - [`/collections/{collection_name}/indexes` (OPTIONS)](#collectionscollection_nameindexes-options) + - [`/collections/{collection_name}/partitions` (GET)](#collectionscollection_namepartitions-get) + - [`/collections/{collection_name}/partitions` (POST)](#collectionscollection_namepartitions-post) + - [`/collections/{collection_name}/partitions` (OPTIONS)](#collectionscollection_namepartitions-options) + - [`/collections/{collection_name}/partitions` (DELETE)](#collectionscollection_namepartitions-delete) + - [`/collections/{collection_name}/segments` (GET)](#collectionscollection_namesegments-get) + - [`/collections/{collection_name}/segments/{segment_name}/vectors` (GET)](#collectionscollection_namesegmentssegment_namevectors-get) + - [`/collections/{collection_name}/segments/{segment_name}/ids` (GET)](#collectionscollection_namesegmentssegment_nameids-get) + - [`/collections/{collection_name}/vectors` (PUT)](#collectionscollection_namevectors-put) + - [`/collections/{collection_name}/vectors` (POST)](#collectionscollection_namevectors-post) + - [`/collections/{collection_name}/vectors` (GET)](#collectionscollection_namevectorsidvector_id-get) + - [`/collections/{collection_name}/vectors` (OPTIONS)](#collectionscollection_namevectors-options) + - [`/system/{msg}` (GET)](#systemmsg-get) + - [`system/{op}` (PUT)](#systemop-put) +- [Error Codes](#error-codes) @@ -47,31 +51,31 @@ Checks whether the web server is running. #### Request -| Request Component | Value | -|-----------------|---| -| Name | `/state` | -| Header | `accept: application/json` | -| Body | N/A | -| Method | GET | +| Request Component | Value | +| ----------------- | -------------------------- | +| Name | `/state` | +| Header | `accept: application/json` | +| Body | N/A | +| Method | GET | #### Response -| Status code | Description | -|-----------------|---| -| 200 | The request is successful.| +| Status code | Description | +| ----------- | -------------------------- | +| 200 | The request is successful. | #### Example ##### Request ```shell -$ curl -X GET "http://192.168.1.65:19121/state" -H "accept: application/json" +$ curl -X GET "http://127.0.0.1:19121/state" -H "accept: application/json" ``` ##### Response ```json -{"message":"Success","code":0} +{ "message": "Success", "code": 0 } ``` ### `/devices` @@ -80,33 +84,32 @@ Gets CPU/GPU information from the host. #### Request -| Request Component | Value | -|-----------------|---| -| Name | `/devices` | -| Header | `accept: application/json` | -| Body | N/A | -| Method | GET | +| Request Component | Value | +| ----------------- | -------------------------- | +| Name | `/devices` | +| Header | `accept: application/json` | +| Body | N/A | +| Method | GET | #### Response -| Status code | Description | -|-----------------|---| -| 200 | The request is successful.| -| 400 | The request is incorrect. Refer to the error message for details. | - +| Status code | Description | +| ----------- | ----------------------------------------------------------------- | +| 200 | The request is successful. | +| 400 | The request is incorrect. Refer to the error message for details. | #### Example ##### Request ```shell -$ curl -X GET "http://192.168.1.65:19121/devices" -H "accept: application/json" +$ curl -X GET "http://127.0.0.1:19121/devices" -H "accept: application/json" ``` ##### Response ```json -{"cpu":{"memory":31},"gpus":{"GPU0":{"memory":5}}} +{ "cpu": { "memory": 31 }, "gpus": { "GPU0": { "memory": 5 } } } ``` ### `/config/advanced` (GET) @@ -115,32 +118,37 @@ Gets the values of parameters in `cache_config` and `engine_config` of the Milvu #### Request -| Request Component | Value | -|-----------------|---| -| Name | `/config/advanced` | -| Header | `accept: application/json` | -| Body | N/A | -| Method | GET | +| Request Component | Value | +| ----------------- | -------------------------- | +| Name | `/config/advanced` | +| Header | `accept: application/json` | +| Body | N/A | +| Method | GET | #### Response -| Status code | Description | -|-----------------|---| -| 200 | The request is successful.| -| 400 | The request is incorrect. Refer to the error message for details. | +| Status code | Description | +| ----------- | ----------------------------------------------------------------- | +| 200 | The request is successful. | +| 400 | The request is incorrect. Refer to the error message for details. | #### Example ##### Request ```shell -$ curl -X GET "http://192.168.1.65:19121/config/advanced" -H "accept: application/json" +$ curl -X GET "http://127.0.0.1:19121/config/advanced" -H "accept: application/json" ``` ##### Response ```json -{"cpu_cache_capacity":4,"cache_insert_data":false,"use_blas_threshold":1100,"gpu_search_threshold":1000} +{ + "cpu_cache_capacity": 4, + "cache_insert_data": false, + "use_blas_threshold": 1100, + "gpu_search_threshold": 1000 +} ``` ### `/config/advanced` (PUT) @@ -169,33 +177,32 @@ Updates the values of parameters in `cache_config` and `engine_config` of the Mi ##### Body Parameters -| Parameter | Description | Required? | -|-----------------|---|------| -| `cpu_cache_capacity` | Value of `cpu_cache_capacity` in the Milvus configuration file. The default is 4.| No | -| `cache_insert_data` | Value of `cache_insert_data` in the Milvus configuration file. The default is false. | No | -| `use_blas_threshold` | Value of `use_blas_threshold` in the Milvus configuration file. The default is 1100. | No | -| `gpu_search_threshold` | Value of `gpu_search_threshold` in the Milvus configuration file. The default is 1000. | No | +| Parameter | Description | Required? | +| ---------------------- | -------------------------------------------------------------------------------------- | --------- | +| `cpu_cache_capacity` | Value of `cpu_cache_capacity` in the Milvus configuration file. The default is 4. | No | +| `cache_insert_data` | Value of `cache_insert_data` in the Milvus configuration file. The default is false. | No | +| `use_blas_threshold` | Value of `use_blas_threshold` in the Milvus configuration file. The default is 1100. | No | +| `gpu_search_threshold` | Value of `gpu_search_threshold` in the Milvus configuration file. The default is 1000. | No | #### Response -| Status code | Description | -|-----------------|---| -| 200 | The request is successful.| -| 400 | The request is incorrect. Refer to the error message for details. | +| Status code | Description | +| ----------- | ----------------------------------------------------------------- | +| 200 | The request is successful. | +| 400 | The request is incorrect. Refer to the error message for details. | #### Example ##### Request ```shell -$ curl -X PUT "http://192.168.1.65:19121/config/advanced" -H "accept: application/json" -H "Content-Type: application/json" -d "{\"cpu_cache_capacity\":4,\"cache_insert_data\":false,\"use_blas_threshold\":1100,\"gpu_search_threshold\":1000}" +$ curl -X PUT "http://127.0.0.1:19121/config/advanced" -H "accept: application/json" -H "Content-Type: application/json" -d "{\"cpu_cache_capacity\":4,\"cache_insert_data\":false,\"use_blas_threshold\":1100,\"gpu_search_threshold\":1000}" ``` ##### Response - ```json -{"message": "OK","code": 0} +{ "message": "OK", "code": 0 } ``` ### `/config/advanced` (OPTIONS) @@ -204,20 +211,19 @@ Use this API for Cross-Origin Resource Sharing (CORS). #### Request -| Request Component | Value | -|-----------------|---| -| Name | `/config/advanced` | -| Header | N/A | -| Body | N/A | -| Method | OPTIONS | - +| Request Component | Value | +| ----------------- | ------------------ | +| Name | `/config/advanced` | +| Header | N/A | +| Body | N/A | +| Method | OPTIONS | #### Example ##### Request ```shell -$ curl -X OPTIONS "http://192.168.1.65:19121/config/advanced" +$ curl -X OPTIONS "http://127.0.0.1:19121/config/advanced" ``` ### `/config/gpu_resources` (GET) @@ -228,33 +234,37 @@ Gets the parameter values in `gpu_resource_config` of the Milvus configuration f #### Request -| Request Component | Value | -|-----------------|---| -| Name | `/config/gpu_resources` | -| Header | `accept: application/json` | -| Body | N/A | -| Method | GET | +| Request Component | Value | +| ----------------- | -------------------------- | +| Name | `/config/gpu_resources` | +| Header | `accept: application/json` | +| Body | N/A | +| Method | GET | #### Response -| Status code | Description | -|-----------------|---| -| 200 | The request is successful.| -| 400 | The request is incorrect. Refer to the error message for details. | +| Status code | Description | +| ----------- | ----------------------------------------------------------------- | +| 200 | The request is successful. | +| 400 | The request is incorrect. Refer to the error message for details. | #### Example ##### Request ```shell -$ curl -X GET "http://192.168.1.65:19121/config/gpu_resources" -H "accept: application/json" +$ curl -X GET "http://127.0.0.1:19121/config/gpu_resources" -H "accept: application/json" ``` ##### Response - ```json -{"enable":true,"cache_capacity":1,"search_resources":["GPU0"],"build_index_resources":["GPU0"]} +{ + "enable": true, + "cache_capacity": 1, + "search_resources": ["GPU0"], + "build_index_resources": ["GPU0"] +} ``` ### `/config/gpu_resources` (PUT) @@ -283,32 +293,32 @@ Updates the parameter values in `gpu_resource_config` of the Milvus configuratio ##### Body Parameters -| Parameter | Description | Required? | -|-----------------|---|------| -| `enable` | Specifies whether to enable GPU resources. | Yes | -| `cache_capacity` | Size of GPU memory per card used for cache in GBs. | Yes | -| `search_resources` | GPU devices used for search computation, must be in format `gpux`. | Yes | -| `build_index_resources` | GPU devices used for index building, must be in format `gpux`. | Yes | +| Parameter | Description | Required? | +| ----------------------- | ------------------------------------------------------------------ | --------- | +| `enable` | Specifies whether to enable GPU resources. | Yes | +| `cache_capacity` | Size of GPU memory per card used for cache in GBs. | Yes | +| `search_resources` | GPU devices used for search computation, must be in format `gpux`. | Yes | +| `build_index_resources` | GPU devices used for index building, must be in format `gpux`. | Yes | #### Response -| Status code | Description | -|-----------------|---| -| 200 | The request is successful.| -| 400 | The request is incorrect. Refer to the error message for details. | +| Status code | Description | +| ----------- | ----------------------------------------------------------------- | +| 200 | The request is successful. | +| 400 | The request is incorrect. Refer to the error message for details. | #### Example ##### Request ```shell -$ curl -X PUT "http://192.168.1.65:19121/config/gpu_resources" -H "accept: application/json" -H "Content-Type: application/json" -d "{\"enable\":true,\"cache_capacity\":1,\"search_resources\":[\"GPU0\"],\"build_index_resources\":[\"GPU0\"]}" +$ curl -X PUT "http://127.0.0.1:19121/config/gpu_resources" -H "accept: application/json" -H "Content-Type: application/json" -d "{\"enable\":true,\"cache_capacity\":1,\"search_resources\":[\"GPU0\"],\"build_index_resources\":[\"GPU0\"]}" ``` ##### Response ```json -{"message": "OK","code": 0} +{ "message": "OK", "code": 0 } ``` ### `/config/gpu_resources` (OPTIONS) @@ -319,67 +329,78 @@ Use this API for Cross-Origin Resource Sharing (CORS). #### Request -| Request Component | Value | -|-----------------|---| -| Name | `/config/gpu_resources` | -| Header | N/A | -| Body | N/A | -| Method | OPTIONS | +| Request Component | Value | +| ----------------- | ----------------------- | +| Name | `/config/gpu_resources` | +| Header | N/A | +| Body | N/A | +| Method | OPTIONS | #### Example ##### Request ```shell -$ curl -X OPTIONS "http://192.168.1.65:19121/config/gpu_resources" +$ curl -X OPTIONS "http://127.0.0.1:19121/config/gpu_resources" ``` -### `/tables` (GET) +### `/collections` (GET) -Gets all tables starting from `offset` and ends with `page_size`. +Gets all collections starting from `offset` and ends with `page_size`. #### Request -| Request Component | Value | -|-----------------|---| -| Name | `/tables` | -| Header | `accept: application/json` | -| Body | N/A | -| Method | GET | +| Request Component | Value | +| ----------------- | -------------------------- | +| Name | `/collections` | +| Header | `accept: application/json` | +| Body | N/A | +| Method | GET | ##### Query Parameters -| Parameter | Description | Required? | -|-----------------|---|------| -| `offset` | Row offset from which the data page starts. The default is 0. | No | -| `page_size` | Size of the data page. The default is 10. | No | - +| Parameter | Description | Required? | +| ----------- | ------------------------------------------------------------- | --------- | +| `offset` | Row offset from which the data page starts. The default is 0. | No | +| `page_size` | Size of the data page. The default is 10. | No | #### Response -| Status code | Description | -|-----------------|---| -| 200 | The request is successful.| -| 400 | The request is incorrect. Refer to the error message for details. | +| Status code | Description | +| ----------- | ----------------------------------------------------------------- | +| 200 | The request is successful. | +| 400 | The request is incorrect. Refer to the error message for details. | #### Example ##### Request ```shell -$ curl -X GET "http://192.168.1.65:19121/tables?offset=0&page_size=1" -H "accept: application/json" +$ curl -X GET "http://127.0.0.1:19121/collections?offset=0&page_size=1" -H "accept: application/json" ``` ##### Response - ```json -{"tables":[{"table_name":"test_table","dimension":1,"index_file_size":10,"metric_type":"L2","count":0,"index":"FLAT","nlist":16384}],"count":58} +{ + "collections": [ + { + "collection_name": "test_collection", + "dimension": 1, + "index_file_size": 10, + "metric_type": "L2", + "count": 0, + "index": "FLAT", + "index_params": {"nlist": 4096} + } + ], + "count": 58 +} ``` -### `/tables` (POST) +### `/collections` (POST) -Creates a table. +Creates a collection. #### Request @@ -389,7 +410,7 @@ Creates a table. Header
accept: application/json
Body

 {
-  "table_name": string,
+  "collection_name": string,
   "dimension": integer($int64),
   "index_file_size": integer($int64),
   "metric_type": string
@@ -401,211 +422,241 @@ Creates a table.
 
 ##### Body Parameters
 
-| Parameter  | Description  |  Required? |
-|-----------------|---|------|
-| `table_name`     |   The name of the table to create, which must be unique within its database.  | Yes   |
-| `dimension`  |  The dimension of the vectors that are to be inserted into the created table. |  Yes  |
-| `index_file_size`    |  Threshold value that triggers index building for raw data files. The default is 1024.   |  No |
-| `metric_type`    |   The method vector distances are compared in Milvus. The default is L2. Currently supported metrics include `L2` (Euclidean distance), `IP` (Inner Product), `HAMMING` (Hamming distance), `JACCARD` (Jaccard distance), and `TANIMOTO` (Tanomoto distance).    |   No  |
+| Parameter         | Description                                                                                                                                                                                                                                                 | Required? |
+| ----------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------- |
+| `collection_name` | The name of the collection to create, which must be unique within its database.                                                                                                                                                                             | Yes       |
+| `dimension`       | The dimension of the vectors that are to be inserted into the created collection.                                                                                                                                                                           | Yes       |
+| `index_file_size` | Threshold value that triggers index building for raw data files. The default is 1024.                                                                                                                                                                       | No        |
+| `metric_type`     | The method vector distances are compared in Milvus. The default is L2. Currently supported metrics include `L2` (Euclidean distance), `IP` (Inner Product), `HAMMING` (Hamming distance), `JACCARD` (Jaccard distance), and `TANIMOTO` (Tanomoto distance). | No        |
 
 #### Response
 
-| Status code    | Description |
-|-----------------|---|
-| 201     | Created |
-| 400     | The request is incorrect. Refer to the error message for details. |
+| Status code | Description                                                       |
+| ----------- | ----------------------------------------------------------------- |
+| 201         | Created                                                           |
+| 400         | The request is incorrect. Refer to the error message for details. |
 
 #### Example
 
 ##### Request
 
 ```shell
-$ curl -X POST "http://192.168.1.65:19121/tables" -H "accept: application/json" -H "Content-Type: application/json" -d "{\"table_name\":\"test_table\",\"dimension\":1,\"index_file_size\":10,\"metric_type\":\"L2\"}"
+$ curl -X POST "http://127.0.0.1:19121/collections" -H "accept: application/json" -H "Content-Type: application/json" -d "{\"collection_name\":\"test_collection\",\"dimension\":1,\"index_file_size\":10,\"metric_type\":\"L2\"}"
 ```
 
 ##### Response
 
-
 ```json
-{"message":"OK","code":0}
+{ "message": "OK", "code": 0 }
 ```
 
-### `/tables` (OPTIONS)
+### `/collections` (OPTIONS)
 
 Use this API for Cross-Origin Resource Sharing (CORS).
 
 #### Request
 
-| Request Component     | Value  |
-|-----------------|---|
-| Name     | `/tables`  |
-| Header  | N/A  |
-| Body    |   N/A |
-| Method    |   OPTIONS |
-
+| Request Component | Value          |
+| ----------------- | -------------- |
+| Name              | `/collections` |
+| Header            | N/A            |
+| Body              | N/A            |
+| Method            | OPTIONS        |
 
 #### Example
 
 ##### Request
 
 ```shell
-$ curl -X OPTIONS "http://192.168.1.65:19121/tables"
+$ curl -X OPTIONS "http://127.0.0.1:19121/collections"
 ```
 
-### `/tables/{table_name}` (GET)
+### `/collections/{collection_name}` (GET)
 
-Gets all information about a table by name.
+Gets all information about a collection by name.
 
 #### Request
 
-| Request Component     | Value  |
-|-----------------|---|
-| Name     | `/tables/{table_name}`  |
-| Header  | `accept: application/json`  |
-| Body    |   N/A |
-| Method    |   GET |
-
+| Request Component | Value                            |
+| ----------------- | -------------------------------- |
+| Name              | `/collections/{collection_name}` |
+| Header            | `accept: application/json`       |
+| Body              | N/A                              |
+| Method            | GET                              |
 
 ##### Query Parameters
 
-| Parameter  | Description  |  Required? |
-|-----------------|---|------|
-| `table_name`     | Name of the table.   | Yes   |
-
+| Parameter         | Description                                                                                                                                                                                                                                                                                                               | Required? |
+| ----------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------- |
+| `collection_name` | Name of the collection.                                                                                                                                                                                                                                                                                                   | Yes       |
+| `info`            | Type of information to acquire. `info` must either be empty or `stat`. When `info` is empty, Milvus returns collection name, dimension, index file size, metric type, offset, index type, and nlist of the collection. When `info` is `stat`, Milvus returns the collection offset, partition status, and segment status. | No        |
 
 #### Response
 
-| Status code    | Description |
-|-----------------|---|
-| 200     | The request is successful.|
-| 400     | The request is incorrect. Refer to the error message for details. |
-| 404     | The required resource does not exist. |
+| Status code | Description                                                       |
+| ----------- | ----------------------------------------------------------------- |
+| 200         | The request is successful.                                        |
+| 400         | The request is incorrect. Refer to the error message for details. |
+| 404         | The required resource does not exist.                             |
 
 #### Example
 
 ##### Request
 
 ```shell
-$ curl -X GET "http://192.168.1.65:19121/tables/test_table" -H "accept: application/json"
+$ curl -X GET "http://127.0.0.1:19121/collections/test_collection" -H "accept: application/json"
 ```
 
 ##### Response
 
 ```json
-{"table_name":"test_table","dimension":1,"index_file_size":10,"metric_type":"L2","count":0,"index":"FLAT","nlist":16384}
+{
+  "collection_name": "test_collection",
+  "dimension": 1,
+  "index_file_size": 10,
+  "metric_type": "L2",
+  "count": 0,
+  "index": "FLAT",
+  "index_params": {"nprobe":  16384}
+}
 ```
 
-### `/tables/{table_name}` (DELETE)
+##### Request
 
-Drops a table by name.
+```shell
+$ curl -X GET "http://127.0.0.1:19121/collections/test_collection?info=stat" -H "accept: application/json"
+```
+
+##### Response
+
+```json
+{
+  "count": 150000,
+  "partitions_stat": [
+    {
+      "count": 1000,
+      "partition_tag": "_default",
+      "segments_stat": [
+        {
+          "count": 1000,
+          "index": "FLAT",
+          "segment_name": "1583727170217439000",
+          "size": 5284922
+        }
+      ]
+    }
+  ]
+}
+```
+
+### `/collections/{collection_name}` (DELETE)
+
+Drops a collection by name.
 
 #### Request
 
-| Request Component     | Value  |
-|-----------------|-----|
-| Name     | `/tables/{table_name}`  |
-| Header  | `accept: application/json`  |
-| Body    |   N/A |
-| Method    |   DELETE |
+| Request Component | Value                            |
+| ----------------- | -------------------------------- |
+| Name              | `/collections/{collection_name}` |
+| Header            | `accept: application/json`       |
+| Body              | N/A                              |
+| Method            | DELETE                           |
 
 ##### Query Parameters
 
-| Parameter  | Description  |  Required? |
-|-----------------|---|------|
-| `table_name`     | Name of the table.   | Yes   |
+| Parameter         | Description             | Required? |
+| ----------------- | ----------------------- | --------- |
+| `collection_name` | Name of the collection. | Yes       |
 
 #### Response
 
-| Status code    | Description |
-|-----------------|---|
-| 204     | Deleted|
-| 400     | The request is incorrect. Refer to the error message for details. |
-| 404     | The required resource does not exist. |
+| Status code | Description                                                       |
+| ----------- | ----------------------------------------------------------------- |
+| 204         | Deleted                                                           |
+| 400         | The request is incorrect. Refer to the error message for details. |
+| 404         | The required resource does not exist.                             |
 
 #### Example
 
 ##### Request
 
-
 ```shell
-$ curl -X DELETE "http://192.168.1.65:19121/tables/test_table" -H "accept: application/json"
+$ curl -X DELETE "http://127.0.0.1:19121/collections/test_collection" -H "accept: application/json"
 ```
 
 If the deletion is successful, no message will be returned.
 
-### `/tables/{table_name}` (OPTIONS)
+### `/collections/{collection_name}` (OPTIONS)
 
 Use this API for Cross-Origin Resource Sharing (CORS).
 
 #### Request
 
-| Request Component     | Value  |
-|-----------------|-----|
-| Name     | `/tables/{table_name}`  |
-| Header  | N/A  |
-| Body    |   N/A |
-| Method    |   OPTIONS |
+| Request Component | Value                            |
+| ----------------- | -------------------------------- |
+| Name              | `/collections/{collection_name}` |
+| Header            | N/A                              |
+| Body              | N/A                              |
+| Method            | OPTIONS                          |
 
 #### Query Parameters
 
-| Parameter  | Description  |  Required? |
-|-----------------|---|------|
-| `table_name`     | Name of the table.   | Yes   |
-
+| Parameter         | Description             | Required? |
+| ----------------- | ----------------------- | --------- |
+| `collection_name` | Name of the collection. | Yes       |
 
 #### Example
 
 ##### Request
 
 ```shell
-$ curl -X OPTIONS "http://192.168.1.65:19121/tables/test_table"
+$ curl -X OPTIONS "http://127.0.0.1:19121/collections/test_collection"
 ```
 
-### `/tables/{table_name}/indexes` (GET)
+### `/collections/{collection_name}/indexes` (GET)
 
-Gets the index type and nlist of a table.
+Gets the index type and nlist of a collection.
 
 #### Request
 
-| Request Component     | Value  |
-|-----------------|-----|
-| Name     | `/tables/{table_name}/indexes`  |
-| Header  | `accept: application/json`  |
-| Body    |   N/A |
-| Method    |   GET |
+| Request Component | Value                                    |
+| ----------------- | ---------------------------------------- |
+| Name              | `/collections/{collection_name}/indexes` |
+| Header            | `accept: application/json`               |
+| Body              | N/A                                      |
+| Method            | GET                                      |
 
 ##### Query Parameters
 
-| Parameter  | Description  |  Required? |
-|-----------------|---|------|
-| `table_name`     | Name of the table.   | Yes   |
+| Parameter         | Description             | Required? |
+| ----------------- | ----------------------- | --------- |
+| `collection_name` | Name of the collection. | Yes       |
 
 #### Response
 
-| Status code    | Description |
-|-----------------|---|
-| 200     | The request is successful.|
-| 400     | The request is incorrect. Refer to the error message for details. |
-| 404     | The required resource does not exist. |
+| Status code | Description                                                       |
+| ----------- | ----------------------------------------------------------------- |
+| 200         | The request is successful.                                        |
+| 400         | The request is incorrect. Refer to the error message for details. |
+| 404         | The required resource does not exist.                             |
 
 #### Example
 
 ##### Request
 
 ```shell
-$ curl -X GET "http://192.168.1.65:19121/tables/test_table/indexes" -H "accept: application/json"
+$ curl -X GET "http://127.0.0.1:19121/collections/test_collection/indexes" -H "accept: application/json"
 ```
 
 ##### Response
 
-
 ```json
-{"index_type":"FLAT","nlist":16384}
+{ "index_type": "FLAT", "params": { "nlist": 4096 } }
 ```
 
-### `/tables/{table_name}/indexes` (POST)
+### `/collections/{collection_name}/indexes` (POST)
 
-Updates the index type and nlist of a table.
+Updates the index type and nlist of a collection.
 
 #### Request
 
@@ -616,7 +667,9 @@ Updates the index type and nlist of a table.
 Body

 {
   "index_type": string,
-  "nlist": integer($int64)
+  "params": {
+      ......
+  }
 }
 
MethodPOST @@ -625,288 +678,419 @@ Updates the index type and nlist of a table. ##### Body Parameters -| Parameter | Description | Required? | -|-----------------|---|------| -| `index_type` | The type of indexing method to query the table. Please refer to [Index Types](https://www.milvus.io/docs/reference/index.md) for detailed introduction of supported indexes. The default is "FLAT". | No | -| `nlist` | Number of vector buckets in a file. The default is 16384. | No | +| Parameter | Description | Required? | +| ------------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------- | +| `index_type` | The type of indexing method to query the collection. Please refer to [Index Types](https://www.milvus.io/docs/reference/index.md) for detailed introduction of supported indexes. The default is "FLAT". | No | +| `params` | The extra params of indexing method to query the collection. Please refer to [Index and search parameters](#Index-and-search-parameters) for detailed introduction of supported indexes. | No | ##### Query Parameters -| Parameter | Description | Required? | -|-----------------|---|------| -| `table_name` | Name of the table. | Yes | +| Parameter | Description | Required? | +| ----------------- | ----------------------- | --------- | +| `collection_name` | Name of the collection. | Yes | #### Response -| Status code | Description | -|-----------------|---| -| 201 | Created | -| 400 | The request is incorrect. Refer to the error message for details. | -| 404 | The required resource does not exist. | - -#### Response - -| Status code | Description | -|-----------------|---| -| 201 | Created | -| 400 | The request is incorrect. Refer to the error message for details. | -| 404 | The required resource does not exist. | +| Status code | Description | +| ----------- | ----------------------------------------------------------------- | +| 201 | Created | +| 400 | The request is incorrect. Refer to the error message for details. | +| 404 | The required resource does not exist. | #### Example ##### Request ```shell -$ curl -X POST "http://192.168.1.65:19121/tables/test_table/indexes" -H "accept: application/json" -H "Content-Type: application/json" -d "{\"index_type\":\"FLAT\",\"nlist\":16384}" +$ curl -X POST "http://127.0.0.1:19121/collections/test_collection/indexes" -H "accept: application/json" -H "Content-Type: application/json" -d "{\"index_type\":\"IVFFLAT\",\"params\": {\"nlist\":4096}}" ``` ##### Response ```json -{"message":"OK","code":0} +{ "message": "OK", "code": 0 } ``` -### `/tables/{table_name}/indexes` (DELETE) +### `/collections/{collection_name}/indexes` (DELETE) -Drops an index for a table. +Drops an index for a collection. #### Request -| Request Component | Value | -|-----------------|---| -| Name | `/tables/{table_name}/indexes` | -| Header | `accept: application/json` | -| Body | N/A | -| Method | DELETE | +| Request Component | Value | +| ----------------- | ---------------------------------------- | +| Name | `/collections/{collection_name}/indexes` | +| Header | `accept: application/json` | +| Body | N/A | +| Method | DELETE | ##### Query Parameters -| Parameter | Description | Required? | -|-----------------|---|------| -| `table_name` | Name of the table. | Yes | +| Parameter | Description | Required? | +| ----------------- | ----------------------- | --------- | +| `collection_name` | Name of the collection. | Yes | #### Response -| Status code | Description | -|-----------------|---| -| 204 | Deleted | -| 400 | The request is incorrect. Refer to the error message for details. | -| 404 | Resource not available | - +| Status code | Description | +| ----------- | ----------------------------------------------------------------- | +| 204 | Deleted | +| 400 | The request is incorrect. Refer to the error message for details. | +| 404 | Resource not available | #### Example ##### Request ```shell -$ curl -X DELETE "http://192.168.1.65:19121/tables/test_table/indexes" -H "accept: application/json" +$ curl -X DELETE "http://127.0.0.1:19121/collections/test_collection/indexes" -H "accept: application/json" ``` If the deletion is successful, no message will be returned. - -### `/tables/{table_name}/indexes` (OPTIONS) +### `/collections/{collection_name}/indexes` (OPTIONS) Use this API for Cross-Origin Resource Sharing (CORS). #### Request -| Request Component | Value | -|-----------------|---| -| Name | `/tables/{table_name}/indexes` | -| Header | N/A | -| Body | N/A | -| Method | OPTIONS | +| Request Component | Value | +| ----------------- | ---------------------------------------- | +| Name | `/collections/{collection_name}/indexes` | +| Header | N/A | +| Body | N/A | +| Method | OPTIONS | ##### Query Parameters -| Parameter | Description | Required? | -|-----------------|---|------| -| `table_name` | Name of the table. | Yes | +| Parameter | Description | Required? | +| ----------------- | ----------------------- | --------- | +| `collection_name` | Name of the collection. | Yes | #### Example ##### Request ```shell -$ curl -X OPTIONS "http://192.168.1.65:19121/tables/test_table/indexes" +$ curl -X OPTIONS "http://127.0.0.1:19121/collections/test_collection/indexes" ``` -### `/tables/{table_name}/partitions` (GET) +### `/collections/{collection_name}/partitions` (GET) -Gets all partitions in a table starting from `offset` and ends with `page_size`. +Gets all partitions in a collection starting from `offset` and ends with `page_size`. #### Request -| Request Component | Value | -|-----------------|-----------| -| Name | `/tables/{table_name}/partitions` | -| Header | `accept: application/json` | -| Body | N/A | -| Method | GET | +| Request Component | Value | +| ----------------- | ------------------------------------------- | +| Name | `/collections/{collection_name}/partitions` | +| Header | `accept: application/json` | +| Body | N/A | +| Method | GET | ##### Query Parameters -| Parameter | Description | Required? | -|-----------------|---|------| -| `table_name` | Name of the table. | Yes | -| `offset` | Row offset from which the data page starts. The default is 0. | No | -| `page_size` | Size of the data page. The default is 10. | No | +| Parameter | Description | Required? | +| ----------------- | ------------------------------------------------------------- | --------- | +| `collection_name` | Name of the collection. | Yes | +| `offset` | Row offset from which the data page starts. The default is 0. | No | +| `page_size` | Size of the data page. The default is 10. | No | #### Response -| Status code | Description | -|-----------------|---| -| 200 | The request is successful.| -| 400 | The request is incorrect. Refer to the error message for details. | -| 404 | The required resource does not exist. | +| Status code | Description | +| ----------- | ----------------------------------------------------------------- | +| 200 | The request is successful. | +| 400 | The request is incorrect. Refer to the error message for details. | +| 404 | The required resource does not exist. | #### Example ##### Request ```shell -$ curl -X GET "http://192.168.1.65:19121/tables/test_table/partitions?offset=0&page_size=3" -H "accept: application/json" +$ curl -X GET "http://127.0.0.1:19121/collections/test_collection/partitions?offset=0&page_size=3" -H "accept: application/json" ``` ##### Response ```json -{"partitions":[{"partition_name":"partition_1","partition_tag":"test_tag"},{"partition_name":"partition_2","partition_tag":"test_2"},{"partition_name":"partition_3","partition_tag":"test_3"}]} +{ + "partitions": [ + { "partition_tag": "_default" }, + { "partition_tag": "test_tag" }, + { "partition_tag": "test_2" } + ], + "count": 10 +} ``` -### `/tables/{table_name}/partitions` (POST) +### `/collections/{collection_name}/partitions` (POST) -Creates a partition in a table. +Creates a partition in a collection. #### Request -| Request Component | Value | -|-----------------|-----------| -| Name | `/tables/{table_name}/partitions` | -| Header | `accept: application/json` | -| Body | N/A | -| Method | POST | +| Request Component | Value | +| ----------------- | ------------------------------------------- | +| Name | `/collections/{collection_name}/partitions` | +| Header | `accept: application/json` | +| Body | N/A | +| Method | POST | #### Response -| Status code | Description | -|-----------------|---| -| 201 | Created | -| 400 | The request is incorrect. Refer to the error message for details. | -| 404 | The required resource does not exist. | +| Status code | Description | +| ----------- | ----------------------------------------------------------------- | +| 201 | Created | +| 400 | The request is incorrect. Refer to the error message for details. | +| 404 | The required resource does not exist. | #### Example ##### Request ```shell -$ curl -X POST "http://192.168.1.65:19121/tables/test_table/partitions" -H "accept: application/json" -H "Content-Type: application/json" -d "{\"partition_name\": \"partition_1\",\"partition_tag\": \"test\"}" +$ curl -X POST "http://127.0.0.1:19121/collections/test_collection/partitions" -H "accept: application/json" -H "Content-Type: application/json" -d "{\"partition_tag\": \"test\"}" ``` ##### Response ```json -{"message":"OK","code":0} +{ "message": "OK", "code": 0 } ``` -### `/tables/{table_name}/partitions` (OPTIONS) +### `/collections/{collection_name}/partitions` (OPTIONS) Use this API for Cross-Origin Resource Sharing (CORS). #### Request -| Request Component | Value | -|-----------------|---| -| Name | `/tables/{table_name}/partitions` | -| Header | N/A | -| Body | N/A | -| Method | OPTIONS | +| Request Component | Value | +| ----------------- | ------------------------------------------- | +| Name | `/collections/{collection_name}/partitions` | +| Header | N/A | +| Body | N/A | +| Method | OPTIONS | ##### Query Parameters -| Parameter | Description | Required? | -|-----------------|---|------| -| `table_name` | Name of the table. | Yes | - +| Parameter | Description | Required? | +| ----------------- | ----------------------- | --------- | +| `collection_name` | Name of the collection. | Yes | #### Example ##### Request ```shell -$ curl -X OPTIONS "http://192.168.1.65:19121/tables/test_table/partitions" +$ curl -X OPTIONS "http://127.0.0.1:19121/collections/test_collection/partitions" ``` -### `/tables/{table_name}/partitions/{partition_tag}` (DELETE) +### `/collections/{collection_name}/partitions` (DELETE) Deletes a partition by tag. #### Request -| Request Component | Value | -|-----------------|-----------| -| Name | `/tables/{table_name}/partitions/{partition_tag}` | -| Header | `accept: application/json` | -| Body | N/A | -| Method | DELETE | + + + + + + + +
Request ComponentValue
Name
/collections/{collection_name}/partitions
Header
accept: application/json
Body

+{
+  "partition_tag": string
+}
+
MethodPOST
##### Query Parameters -| Parameter | Description | Required? | -|-----------------|---|------| -| `table_name` | Name of the table that contains the partition. | Yes | -| `partition_tag` | Tag of the partition to delete. | yes | +| Parameter | Description | Required? | +| ----------------- | --------------------------------------------------- | --------- | +| `collection_name` | Name of the collection that contains the partition. | Yes | +| `partition_tag` | Tag of the partition to delete. | yes | #### Response -| Status code | Description | -|-----------------|---| -| 204 | Deleted | -| 400 | The request is incorrect. Refer to the error message for details. | -| 404 | The requested resource does not exist. | +| Status code | Description | +| ----------- | ----------------------------------------------------------------- | +| 204 | Deleted | +| 400 | The request is incorrect. Refer to the error message for details. | +| 404 | The requested resource does not exist. | #### Example ##### Request ```shell -$ curl -X DELETE "http://192.168.1.65:19121/tables/test_table/partitions/tags_01" -H "accept: application/json" +$ curl -X DELETE "http://127.0.0.1:19121/collections/test_collection/partitions -H "accept: application/json" -d "{\"partition_tag\": \"tags_01\"}" ``` The deletion is successful if no information is returned. -### `/tables/{table_name}/partitions/{partition_tag}` (OPTIONS) +### `/collections/{collection_name}/segments` (GET) -Use this API for Cross-Origin Resource Sharing (CORS). +Gets all segments in a collection starting from `offset` and ends with `page_size`. #### Request -| Request Component | Value | -|-----------------|---| -| Name | `/tables/{table_name}/partitions/{partition_tag}` | -| Header | N/A | -| Body | N/A | -| Method | OPTIONS | +| Request Component | Value | +| ----------------- | ----------------------------------------- | +| Name | `/collections/{collection_name}/segments` | +| Header | `accept: application/json` | +| Body | N/A | +| Method | GET | ##### Query Parameters -| Parameter | Description | Required? | -|-----------------|---|------| -| `table_name` | Name of the table. | Yes | -| `partition_tag` | Tag of the partition | yes | +| Parameter | Description | Required? | +| ----------------- | ------------------------------------------------------------- | --------- | +| `collection_name` | Name of the collection. | Yes | +| `offset` | Row offset from which the data page starts. The default is 0. | No | +| `page_size` | Size of the data page. The default is 10. | No | + +#### Response + +| Status code | Description | +| ----------- | ----------------------------------------------------------------- | +| 200 | The request is successful. | +| 400 | The request is incorrect. Refer to the error message for details. | +| 404 | The required resource does not exist. | #### Example ##### Request ```shell -$ curl -X OPTIONS "http://192.168.1.65:19121/tables/test_table/partitions/tag" +$ curl -X GET "http://127.0.0.1:19121/collections/test_collection/segments?offset=0&page_size=1" -H "accept: application/json" ``` -### `/tables/{table_name}/vectors` (PUT) +##### Response -Searches vectors in a table. +```json +{ + "code": 0, + "message": "OK", + "count": 2, + "segments": [ + { + "count": 10000, + "index": "IVFFLAT", + "partition_tag": "_default", + "segment_name": "1583727470444700000", + "size": 5284922 + } + ] +} +``` + +### `/collections/{collection_name}/segments/{segment_name}/vectors` (GET) + +Gets all vectors of segment in a collection starting from `offset` and ends with `page_size`. + +#### Request + +| Request Component | Value | +| ----------------- | ----------------------------------------- | +| Name | `/collections/{collection_name}/segments` | +| Header | `accept: application/json` | +| Body | N/A | +| Method | GET | + +##### Query Parameters + +| Parameter | Description | Required? | +| ----------------- | ------------------------------------------------------------- | --------- | +| `collection_name` | Name of the collection. | Yes | +| `segment_name` | Name of the segment. | Yes | +| `offset` | Row offset from which the data page starts. The default is 0. | No | +| `page_size` | Size of the data page. The default is 10. | No | + +#### Response + +| Status code | Description | +| ----------- | ----------------------------------------------------------------- | +| 200 | The request is successful. | +| 400 | The request is incorrect. Refer to the error message for details. | +| 404 | The required resource does not exist. | + +#### Example + +##### Request + +```shell +$ curl -X GET "http://127.0.0.1:19121/collections/test_collection/segments/1583727470444700000/vectors?offset=0&page_size=1" -H "accept: application/json" +``` + +##### Response + +```json +{ + "code": 0, + "message": "OK", + "count": 2, + "vectors": [ + { + "vector": [0.1], + "id": "1583727470435045000" + } + ] +} +``` + +### `/collections/{collection_name}/segments/{segment_name}/ids` (GET) + +Gets all vector ids of segment in a collection starting from `offset` and ends with `page_size`. + +#### Request + +| Request Component | Value | +| ----------------- | ----------------------------------------- | +| Name | `/collections/{collection_name}/segments` | +| Header | `accept: application/json` | +| Body | N/A | +| Method | GET | + +##### Query Parameters + +| Parameter | Description | Required? | +| ----------------- | ------------------------------------------------------------- | --------- | +| `collection_name` | Name of the collection. | Yes | +| `segment_name` | Name of the segment. | Yes | +| `offset` | Row offset from which the data page starts. The default is 0. | No | +| `page_size` | Size of the data page. The default is 10. | No | + +#### Response + +| Status code | Description | +| ----------- | ----------------------------------------------------------------- | +| 200 | The request is successful. | +| 400 | The request is incorrect. Refer to the error message for details. | +| 404 | The required resource does not exist. | + +#### Example + +##### Request + +```shell +$ curl -X GET "http://127.0.0.1:19121/collections/test_collection/segments/1583727470444700000/ids?offset=0&page_size=1" -H "accept: application/json" +``` + +##### Response + +```json +{ + "ids": ["1583727470435045000"], + "count": 10000 +} +``` + +### `/collections/{collection_name}/vectors` (PUT) + +1. Searches vectors in a collection. #### Request @@ -916,12 +1100,15 @@ Searches vectors in a table. Header
accept: application/json
Body

 {
-  "topk": integer($int64),
-  "nprobe": integer($int64),
-  "tags": [string],
-  "file_ids": [string],
-  "records": [[number($float)]],
-  "records_bin": [[number($uint64)]]
+  "search": {
+      "topk": integer($int64),
+      "partition_tags": [string],
+      "file_ids": [string],
+      "vectors": [[number($float/$uint8)]]
+      "params": {
+          "nprobe": 16
+      }
+  }
 }
 
MethodPUT @@ -929,49 +1116,107 @@ Searches vectors in a table. ##### Body Parameters -| Parameter | Description | Required? | -|-----------------|---|------| -| `topk` | The top k most similar results of each query vector. | Yes | -| `nprobe` | Number of queried vector buckets. | Yes | -| `tags` | Tags of partitions that you need to search. You do not have to specify this value if the table is not partitioned or you wish to search the whole table. | No | -| `file_ids` | IDs of the vector files. You do not have to specify this value if you do not use Milvus in distributed scenarios. Also, if you assign a value to `file_ids`, the value of `tags` is ignored. | No | -| `records` | Numeric vectors to insert to the table. | Yes | -| `records_bin` | Binary vectors to insert to the table. | Yes | - -> Note: Select `records` or `records_bin` depending on the metric used by the table. If the table uses `L2` or `IP`, you must use `records`. If the table uses `HAMMING`, `JACCARD`, or `TANIMOTO`, you must use `records_bin`. +| Parameter | Description | Required? | +| ---------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------- | +| `topk` | The top k most similar results of each query vector. | Yes | +| `tags` | Tags of partitions that you need to search. You do not have to specify this value if the collection is not partitioned or you wish to search the whole collection. | No | +| `file_ids` | IDs of the vector files. You do not have to specify this value if you do not use Milvus in distributed scenarios. Also, if you assign a value to `file_ids`, the value of `tags` is ignored. | No | +| `vectors` | Vectors to query. | Yes | +| `params` | Extra params for search. Please refer to [Index and search parameters](#Index-and-search-parameters) to get more detail information. | Yes | +> Note: Type of items of vectors depends on the metric used by the collection. If the collection uses `L2` or `IP`, you must use `float`. If the collection uses `HAMMING`, `JACCARD`, or `TANIMOTO`, you must use `uint8`. ##### Query Parameters -| Parameter | Description | Required? | -|-----------------|---|------| -| `table_name` | Name of the table. | Yes | +| Parameter | Description | Required? | +| ----------------- | ----------------------- | --------- | +| `collection_name` | Name of the collection. | Yes | #### Response -| Status code | Description | -|-----------------|---| -| 200 | The request is successful.| -| 400 | The request is incorrect. Refer to the error message for details. | -| 404 | The required resource does not exist. | +| Status code | Description | +| ----------- | ----------------------------------------------------------------- | +| 200 | The request is successful. | +| 400 | The request is incorrect. Refer to the error message for details. | +| 404 | The required resource does not exist. | #### Example ##### Request ```shell -$ curl -X PUT "http://192.168.1.65:19121/tables/test_table/vectors" -H "accept: application/json" -H "Content-Type: application/json" -d "{\"topk\":2,\"nprobe\":16,\"records\":[[0.1]]}" +$ curl -X PUT "http://127.0.0.1:19121/collections/test_collection/vectors" -H "accept: application/json" -H "Content-Type: application/json" -d "{\"topk\":2,\"vectors\":[[0.1]], \"params\":{\"nprobe\":16}}" ``` ##### Response ```json -{"num":1,"results":[[{"id":"1578989029645098000","distance":"0.000000"},{"id":"1578989029645098001","distance":"0.010000"}]]} +{ + "num": 1, + "results": [ + [ + { "id": "1578989029645098000", "distance": "0.000000" }, + { "id": "1578989029645098001", "distance": "0.010000" } + ] + ] +} ``` -### `/tables/{table_name}/vectors` (POST) +2. Delete vectors -Inserts vectors to a table. +#### Request + + + + + + + +
Request ComponentValue
Name
/tables/{table_name}/vectors
Header
accept: application/json
Body

+{
+  "delete": {
+     "ids": [$string]
+  }
+}
+
MethodPUT
+ +##### Body Parameters + +| Parameter | Description | Required? | +| --------- | --------------- | --------- | +| ids | IDs of vectors. | Yes | + +##### Query Parameters + +| Parameter | Description | Required? | +| ----------------- | ----------------------- | --------- | +| `collection_name` | Name of the collection. | Yes | + +#### Response + +| Status code | Description | +| ----------- | ----------------------------------------------------------------- | +| 200 | The request is successful. | +| 400 | The request is incorrect. Refer to the error message for details. | +| 404 | The required resource does not exist. | + +#### Example + +##### Request + +```shell +$ curl -X PUT "http://127.0.0.1:19121/collections/test_collection/vectors" -H "accept: application/json" -H "Content-Type: application/json" -d "{"delete": {"ids": ["1578989029645098000"]}}" +``` + +##### Response + +```json +{ "code": 0, "message": "success" } +``` + +### `/collections/{collection_name}/vectors` (POST) + +Inserts vectors to a collection. > Note: It is recommended that you do not insert more than 1 million vectors per request. @@ -983,9 +1228,8 @@ Inserts vectors to a table. Header
accept: application/json
Body

 {
-  "tag": string,
-  "records": [[number($float)]],
-  “records_bin”:[[number($uint64)]]
+  "partition_tag": string,
+  "vectors": [[number($float/$uint8)]],
   "ids": [integer($int64)]
 }
 
@@ -994,62 +1238,117 @@ Inserts vectors to a table. ##### Body Parameters -| Parameter | Description | Required? | -|-----------------|---|------| -| `tag` | Tag of the partition to insert vectors to. | No | -| `records` | Numeric vectors to insert to the table. | Yes | -| `records_bin` | Binary vectors to insert to the table. | Yes | -| `ids` | IDs of the vectors to insert to the table. If you assign IDs to the vectors, you must provide IDs for all vectors in the table. If you do not specify this parameter, Milvus automatically assigns IDs to the vectors. | No | +| Parameter | Description | Required? | +| --------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------- | +| `partition_tag` | Tag of the partition to insert vectors to. | No | +| `vectors` | Vectors to insert to the collection. | Yes | +| `ids` | IDs of the vectors to insert to the collection. If you assign IDs to the vectors, you must provide IDs for all vectors in the collection. If you do not specify this parameter, Milvus automatically assigns IDs to the vectors. | No | -> Note: Select `records` or `records_bin` depending on the metric used by the table. If the table uses `L2` or `IP`, you must use `records`. If the table uses `HAMMING`, `JACCARD`, or `TANIMOTO`, you must use `records_bin`. +> Note: Type of items of `vectors` depends on the metric used by the collection. If the collection uses `L2` or `IP`, you must use `float`. If the collection uses `HAMMING`, `JACCARD`, or `TANIMOTO`, you must use `uint8`. ##### Query Parameters -| Parameter | Description | Required? | -|-----------------|---|------| -| `table_name` | Name of the table. | Yes | +| Parameter | Description | Required? | +| ----------------- | ----------------------- | --------- | +| `collection_name` | Name of the collection. | Yes | #### Response -| Status code | Description | -|-----------------|---| -| 201 | Created | -| 400 | The request is incorrect. Refer to the error message for details. | -| 404 | The required resource does not exist. | +| Status code | Description | +| ----------- | ----------------------------------------------------------------- | +| 201 | Created | +| 400 | The request is incorrect. Refer to the error message for details. | +| 404 | The required resource does not exist. | #### Example ##### Request ```shell -$ curl -X POST "http://192.168.1.65:19121/tables/test_table/vectors" -H "accept: application/json" -H "Content-Type: application/json" -d "{\"records\":[[0.1],[0.2],[0.3],[0.4]]}" +$ curl -X POST "http://127.0.0.1:19121/collections/test_collection/vectors" -H "accept: application/json" -H "Content-Type: application/json" -d "{\"vectors\":[[0.1],[0.2],[0.3],[0.4]]}" ``` ##### Response ```json -{"ids":["1578989029645098000","1578989029645098001","1578989029645098002","1578989029645098003"]} +{ + "ids": [ + "1578989029645098000", + "1578989029645098001", + "1578989029645098002", + "1578989029645098003" + ] +} ``` -### `/tables/{table_name}/vectors` (OPTIONS) +### `/collections/{collection_name}/vectors?id={vector_id}` (GET) -Use this API for Cross-Origin Resource Sharing (CORS). +Obtain a vector to by ID. #### Request -| Request Component | Value | -|-----------------|---| -| Name | `/tables/{table_name}/vectors` | -| Header | N/A | -| Body | N/A | -| Method | OPTIONS | +| Request Component | Value | +| ----------------- | ---------------------------------------- | +| Name | `/collections/{collection_name}/vectors` | +| Header | `accept: application/json` | +| Body | N/A | +| Method | GET | + +#### Query Parameters + +| Parameter | Description | Required? | +| ----------------- | ----------------------- | --------- | +| `collection_name` | Name of the collection. | Yes | +| `vector_id` | Vector id. | Yes | + +#### Response + +| Status code | Description | +| ----------- | ----------------------------------------------------------------- | +| 201 | Created | +| 400 | The request is incorrect. Refer to the error message for details. | +| 404 | The required resource does not exist. | #### Example ##### Request ```shell -$ curl -X OPTIONS "http://192.168.1.65:19121/tables/test_table/vectors" +$ curl -X POST "http://127.0.0.1:19121/collections/test_collection/vectors?id=1578989029645098000" -H "accept: application/json" -H "Content-Type: application/json" +``` + +##### Response + +```json +{ + "vectors": [ + { + "id": "1578989029645098000", + "vector": [0.1] + } + ] +} +``` + +### `/collections/{collection_name}/vectors` (OPTIONS) + +Use this API for Cross-Origin Resource Sharing (CORS). + +#### Request + +| Request Component | Value | +| ----------------- | ---------------------------------------- | +| Name | `/collections/{collection_name}/vectors` | +| Header | N/A | +| Body | N/A | +| Method | OPTIONS | + +#### Example + +##### Request + +```shell +$ curl -X OPTIONS "http://127.0.0.1:19121/collections/test_collection/vectors" ``` ### `/system/{msg}` (GET) @@ -1058,72 +1357,229 @@ Gets information about the Milvus server. #### Request -| Request Component | Value | -|-----------------|---| -| Name | `/system/{msg}` | -| Header | `accept: application/json` | -| Body | N/A | -| Method | GET | +| Request Component | Value | +| ----------------- | -------------------------- | +| Name | `/system/{msg}` | +| Header | `accept: application/json` | +| Body | N/A | +| Method | GET | ##### Query Parameters -| Parameter | Description | Required? | -|-----------------|---|------| -| `msg` | Type of the message to return. You can use `status` or `version`. | Yes | +| Parameter | Description | Required? | +| --------- | ----------------------------------------------------------------- | --------- | +| `msg` | Type of the message to return. You can use `status` or `version`. | Yes | #### Response -| Status code | Description | -|-----------------|---| -| 200 | The request is successful.| -| 400 | The request is incorrect. Refer to the error message for details. | +| Status code | Description | +| ----------- | ----------------------------------------------------------------- | +| 200 | The request is successful. | +| 400 | The request is incorrect. Refer to the error message for details. | #### Example ##### Request ```shell -$ curl -X GET "http://192.168.1.65:19121/system/version" -H "accept: application/json" +$ curl -X GET "http://127.0.0.1:19121/system/version" -H "accept: application/json" ``` ##### Response ```json -{"reply":"0.6.0"} +{ "reply": "0.7.0" } ``` +### `system/{op}` (PUT) + +#### Flush a collection + +##### Request + + + + + + + +
Request ComponentValue
Name
/system/task
Header
accept: application/json
Body

+{
+  "flush": {
+     "collection_names": [$string]
+  }
+}
+
MethodPUT
+ +##### Response + +| Status code | Description | +| ----------- | ----------------------------------------------------------------- | +| 200 | The request is successful. | +| 400 | The request is incorrect. Refer to the error message for details. | + +##### Example + +###### Request + +```shell +$ curl -X PUT "http://127.0.0.1:19121/system/task" -H "accept: application/json" -d "{\"flush\": {\"collection_names\": [\"test_collection\"]}}" +``` + +###### Response + +```json +{ "code": 0, "message": "success" } +``` + +#### Compact segments in a collection + +##### Request + + + + + + + +
Request ComponentValue
Name
/system/task
Header
accept: application/json
Body

+{
+  "compact": {
+     "collection_name": $string
+  }
+}
+
MethodPUT
+ +##### Response + +| Status code | Description | +| ----------- | ----------------------------------------------------------------- | +| 200 | The request is successful. | +| 400 | The request is incorrect. Refer to the error message for details. | + +##### Example + +###### Request + +```shell +$ curl -X PUT "http://127.0.0.1:19121/system/task" -H "accept: application/json" -d "{\"compact\": {\"collection_name\": \"test_collection\"}}" +``` + +###### Response + +```json +{ "code": 0, "message": "success" } +``` + +#### Load a collection to memory + +##### Request + + + + + + + +
Request ComponentValue
Name
/system/task
Header
accept: application/json
Body

+{
+  "load": {
+     "collection_name": $string
+  }
+}
+
MethodPUT
+ +##### Response + +| Status code | Description | +| ----------- | ----------------------------------------------------------------- | +| 200 | The request is successful. | +| 400 | The request is incorrect. Refer to the error message for details. | + +##### Example + +###### Request + +```shell +$ curl -X PUT "http://127.0.0.1:19121/system/task" -H "accept: application/json" -d "{\"load\": {\"collection_name\": \"test_collection\"}}" +``` + +###### Response + +```json +{ "code": 0, "message": "success" } +``` + +## Index and search parameters + +For each index type, the RESTful API has specific index parameters and search parameters. + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Index typeCreate index paramSearch param
IVFFLAT
{"nlist": $int}
{"nprobe": $int}
IVFPQ
{"m": $int, "nlist": $int}
{"nprobe": $int}
IVFSQ8
{"nlist": $int}
{"nprobe": $int}
IVFSQ8H
{"nlist": $int}
{"nprobe": $int}
HNSW
{"M": $int, "efConstruction": $int}
{"ef": $int}
+ +For detailed information about the parameters above, refer to [Index Types](https://milvus.io/docs/v0.7.0/reference/index.md) + ## Error Codes The RESTful API returns error messages as JSON text. Each type of error message has a specific error code. -| Type | Code | -|----------|------| -SUCCESS | 0 | -UNEXPECTED_ERROR | 1 | -CONNECT_FAILED | 2 | -PERMISSION_DENIED | 3 | -TABLE_NOT_EXISTS | 4 | -ILLEGAL_ARGUMENT | 5 | -ILLEGAL_RANGE | 6 | -ILLEGAL_DIMENSION | 7 | -ILLEGAL_INDEX_TYPE | 8 | -ILLEGAL_TABLE_NAME | 9 | -ILLEGAL_TOPK | 10 | -ILLEGAL_ROWRECORD | 11 | -ILLEGAL_VECTOR_ID | 12 | -ILLEGAL_SEARCH_RESULT | 13 | -FILE_NOT_FOUND | 14 | -META_FAILED | 15 | -CACHE_FAILED | 16 | -CANNOT_CREATE_FOLDER | 17 | -CANNOT_CREATE_FILE | 18 | -CANNOT_DELETE_FOLDER | 19 | -CANNOT_DELETE_FILE | 20 | -BUILD_INDEX_ERROR | 21 | -ILLEGAL_NLIST | 22 | -ILLEGAL_METRIC_TYPE | 23 | -OUT_OF_MEMORY | 24 | -PATH_PARAM_LOSS | 31 | -QUERY_PARAM_LOSS | 32 | -BODY_FIELD_LOSS | 33 | -ILLEGAL_QUERY_PARAM | 36 | +| Type | Code | +| --------------------- | ---- | +| SUCCESS | 0 | +| UNEXPECTED_ERROR | 1 | +| CONNECT_FAILED | 2 | +| PERMISSION_DENIED | 3 | +| TABLE_NOT_EXISTS | 4 | +| ILLEGAL_ARGUMENT | 5 | +| ILLEGAL_RANGE | 6 | +| ILLEGAL_DIMENSION | 7 | +| ILLEGAL_INDEX_TYPE | 8 | +| ILLEGAL_TABLE_NAME | 9 | +| ILLEGAL_TOPK | 10 | +| ILLEGAL_ROWRECORD | 11 | +| ILLEGAL_VECTOR_ID | 12 | +| ILLEGAL_SEARCH_RESULT | 13 | +| FILE_NOT_FOUND | 14 | +| META_FAILED | 15 | +| CACHE_FAILED | 16 | +| CANNOT_CREATE_FOLDER | 17 | +| CANNOT_CREATE_FILE | 18 | +| CANNOT_DELETE_FOLDER | 19 | +| CANNOT_DELETE_FILE | 20 | +| BUILD_INDEX_ERROR | 21 | +| ILLEGAL_NLIST | 22 | +| ILLEGAL_METRIC_TYPE | 23 | +| OUT_OF_MEMORY | 24 | +| PATH_PARAM_LOSS | 31 | +| UNKNOWN_PATH | 32 | +| QUERY_PARAM_LOSS | 33 | +| BODY_FIELD_LOSS | 34 | +| ILLEGAL_BODY | 35 | +| BODY_PARSE_FAIL | 36 | +| ILLEGAL_QUERY_PARAM | 37 | diff --git a/core/src/server/web_impl/controller/WebController.hpp b/core/src/server/web_impl/controller/WebController.hpp index ab2a26c2ca..f326578405 100644 --- a/core/src/server/web_impl/controller/WebController.hpp +++ b/core/src/server/web_impl/controller/WebController.hpp @@ -336,7 +336,7 @@ class WebController : public oatpp::web::server::api::ApiController { ADD_CORS(CreateIndex) - ENDPOINT("POST", "/tables/{collection_name}/indexes", CreateIndex, + ENDPOINT("POST", "/collections/{collection_name}/indexes", CreateIndex, PATH(String, collection_name), BODY_STRING(String, body)) { TimeRecorder tr(std::string(WEB_LOG_PREFIX) + "POST \'/tables/" + collection_name->std_str() + "/indexes\'"); tr.RecordSection("Received request."); @@ -617,10 +617,7 @@ class WebController : public oatpp::web::server::api::ApiController { } ADD_CORS(VectorsOp) - /************* - * Search - * Delete by ID - * */ + ENDPOINT("PUT", "/collections/{collection_name}/vectors", VectorsOp, PATH(String, collection_name), BODY_STRING(String, body)) { TimeRecorder tr(std::string(WEB_LOG_PREFIX) + "PUT \'/collections/" + collection_name->std_str() + "/vectors\'"); @@ -648,6 +645,12 @@ class WebController : public oatpp::web::server::api::ApiController { return response; } + ADD_CORS(SystemOptions) + + ENDPOINT("OPTIONS", "/system/{info}", SystemOptions) { + return createResponse(Status::CODE_204, "No Content"); + } + ADD_CORS(SystemInfo) ENDPOINT("GET", "/system/{info}", SystemInfo, PATH(String, info), QUERIES(const QueryParams&, query_params)) { @@ -674,15 +677,15 @@ class WebController : public oatpp::web::server::api::ApiController { ADD_CORS(SystemOp) - ENDPOINT("PUT", "/system/{Op}", SystemOp, PATH(String, Op), BODY_STRING(String, body_str)) { - TimeRecorder tr(std::string(WEB_LOG_PREFIX) + "PUT \'/system/" + Op->std_str() + "\'"); + ENDPOINT("PUT", "/system/{op}", SystemOp, PATH(String, op), BODY_STRING(String, body_str)) { + TimeRecorder tr(std::string(WEB_LOG_PREFIX) + "PUT \'/system/" + op->std_str() + "\'"); tr.RecordSection("Received request."); WebRequestHandler handler = WebRequestHandler(); handler.RegisterRequestHandler(::milvus::server::RequestHandler()); String response_str; - auto status_dto = handler.SystemOp(Op, body_str, response_str); + auto status_dto = handler.SystemOp(op, body_str, response_str); std::shared_ptr response; switch (status_dto->code->getValue()) { diff --git a/core/src/server/web_impl/handler/WebRequestHandler.cpp b/core/src/server/web_impl/handler/WebRequestHandler.cpp index a7b61790a9..d08bc4d985 100644 --- a/core/src/server/web_impl/handler/WebRequestHandler.cpp +++ b/core/src/server/web_impl/handler/WebRequestHandler.cpp @@ -1046,7 +1046,9 @@ WebRequestHandler::CreateIndex(const OString& table_name, const OString& body) { auto status = request_handler_.CreateIndex(context_ptr_, table_name->std_str(), index, request_json["params"]); ASSIGN_RETURN_STATUS_DTO(status); } catch (nlohmann::detail::parse_error& e) { + RETURN_STATUS_DTO(BODY_PARSE_FAIL, e.what()) } catch (nlohmann::detail::type_error& e) { + RETURN_STATUS_DTO(BODY_PARSE_FAIL, e.what()) } ASSIGN_RETURN_STATUS_DTO(Status::OK()) diff --git a/core/src/utils/ValidationUtil.cpp b/core/src/utils/ValidationUtil.cpp index 84f260c508..50a6536fd2 100644 --- a/core/src/utils/ValidationUtil.cpp +++ b/core/src/utils/ValidationUtil.cpp @@ -174,14 +174,14 @@ ValidationUtil::ValidateIndexParams(const milvus::json& index_params, const engi case (int32_t)engine::EngineType::FAISS_IVFSQ8: case (int32_t)engine::EngineType::FAISS_IVFSQ8H: case (int32_t)engine::EngineType::FAISS_BIN_IVFFLAT: { - auto status = CheckParameterRange(index_params, knowhere::IndexParams::nlist, 0, 999999, false); + auto status = CheckParameterRange(index_params, knowhere::IndexParams::nlist, 1, 999999); if (!status.ok()) { return status; } break; } case (int32_t)engine::EngineType::FAISS_PQ: { - auto status = CheckParameterRange(index_params, knowhere::IndexParams::nlist, 0, 999999, false); + auto status = CheckParameterRange(index_params, knowhere::IndexParams::nlist, 1, 999999); if (!status.ok()) { return status; } @@ -254,7 +254,7 @@ ValidationUtil::ValidateSearchParams(const milvus::json& search_params, const en break; } case (int32_t)engine::EngineType::HNSW: { - auto status = CheckParameterRange(search_params, knowhere::IndexParams::ef, topk, 1000); + auto status = CheckParameterRange(search_params, knowhere::IndexParams::ef, topk, 4096); if (!status.ok()) { return status; } @@ -264,12 +264,6 @@ ValidationUtil::ValidateSearchParams(const milvus::json& search_params, const en return Status::OK(); } -bool -ValidationUtil::IsBinaryIndexType(int32_t index_type) { - return (index_type == static_cast(engine::EngineType::FAISS_BIN_IDMAP)) || - (index_type == static_cast(engine::EngineType::FAISS_BIN_IVFFLAT)); -} - Status ValidationUtil::ValidateTableIndexFileSize(int64_t index_file_size) { if (index_file_size <= 0 || index_file_size > INDEX_FILE_SIZE_LIMIT) { @@ -294,13 +288,6 @@ ValidationUtil::ValidateTableIndexMetricType(int32_t metric_type) { return Status::OK(); } -bool -ValidationUtil::IsBinaryMetricType(int32_t metric_type) { - return (metric_type == static_cast(engine::MetricType::HAMMING)) || - (metric_type == static_cast(engine::MetricType::JACCARD)) || - (metric_type == static_cast(engine::MetricType::TANIMOTO)); -} - Status ValidationUtil::ValidateSearchTopk(int64_t top_k, const engine::meta::TableSchema& table_schema) { if (top_k <= 0 || top_k > 2048) { diff --git a/core/src/utils/ValidationUtil.h b/core/src/utils/ValidationUtil.h index 4fa2df1619..481cb31f1b 100644 --- a/core/src/utils/ValidationUtil.h +++ b/core/src/utils/ValidationUtil.h @@ -43,18 +43,12 @@ class ValidationUtil { ValidateSearchParams(const milvus::json& search_params, const engine::meta::TableSchema& table_schema, int64_t topk); - static bool - IsBinaryIndexType(int32_t index_type); - static Status ValidateTableIndexFileSize(int64_t index_file_size); static Status ValidateTableIndexMetricType(int32_t metric_type); - static bool - IsBinaryMetricType(int32_t metric_type); - static Status ValidateSearchTopk(int64_t top_k, const engine::meta::TableSchema& table_schema); diff --git a/core/src/wrapper/ConfAdapter.cpp b/core/src/wrapper/ConfAdapter.cpp index 07ba5c56dd..dbeb724b33 100644 --- a/core/src/wrapper/ConfAdapter.cpp +++ b/core/src/wrapper/ConfAdapter.cpp @@ -31,7 +31,7 @@ namespace engine { #define GPU_MAX_NRPOBE 1024 #endif -#define DEFAULT_MAX_DIM 16384 +#define DEFAULT_MAX_DIM 32768 #define DEFAULT_MIN_DIM 1 #define DEFAULT_MAX_K 16384 #define DEFAULT_MIN_K 1 @@ -225,7 +225,9 @@ NSGConfAdapter::CheckTrain(milvus::json& oricfg) { // auto tune params oricfg[knowhere::IndexParams::nlist] = MatchNlist(oricfg[knowhere::meta::ROWS].get(), 8192, 8192); - oricfg[knowhere::IndexParams::nprobe] = int(oricfg[knowhere::IndexParams::nlist].get() * 0.01); + + int64_t nprobe = int(oricfg[knowhere::IndexParams::nlist].get() * 0.1); + oricfg[knowhere::IndexParams::nprobe] = nprobe < 1 ? 1 : nprobe; return true; } diff --git a/core/src/wrapper/VecIndex.h b/core/src/wrapper/VecIndex.h index f7cf50b5bc..7a98e971fe 100644 --- a/core/src/wrapper/VecIndex.h +++ b/core/src/wrapper/VecIndex.h @@ -12,10 +12,10 @@ #pragma once #include -#include #include #include +#include #include #include @@ -180,13 +180,14 @@ class VecIndex : public cache::DataObj { virtual Status SetBlacklist(faiss::ConcurrentBitsetPtr list) { - ENGINE_LOG_ERROR << "SetBlacklist not support"; + // ENGINE_LOG_ERROR << "SetBlacklist not support"; return Status::OK(); } virtual Status GetBlacklist(faiss::ConcurrentBitsetPtr& list) { - ENGINE_LOG_ERROR << "GetBlacklist not support"; + // ENGINE_LOG_ERROR << "GetBlacklist not support"; + ENGINE_LOG_WARNING << "Deletion on unsupported index type"; return Status::OK(); } diff --git a/core/unittest/db/test_delete.cpp b/core/unittest/db/test_delete.cpp index 4d6051455d..97c91fab64 100644 --- a/core/unittest/db/test_delete.cpp +++ b/core/unittest/db/test_delete.cpp @@ -68,7 +68,7 @@ TEST_F(DeleteTest, delete_in_mem) { auto stat = db_->CreateTable(table_info); milvus::engine::meta::TableSchema table_info_get; - table_info_get.table_id_ = GetTableName(); + table_info_get.table_id_ = table_info.table_id_; stat = db_->DescribeTable(table_info_get); ASSERT_TRUE(stat.ok()); ASSERT_EQ(table_info_get.dimension_, TABLE_DIM); @@ -81,7 +81,7 @@ TEST_F(DeleteTest, delete_in_mem) { xb.id_array_.push_back(i); } - stat = db_->InsertVectors(GetTableName(), "", xb); + stat = db_->InsertVectors(table_info.table_id_, "", xb); ASSERT_TRUE(stat.ok()); std::random_device rd; @@ -105,7 +105,7 @@ TEST_F(DeleteTest, delete_in_mem) { ids_to_delete.emplace_back(kv.first); } - stat = db_->DeleteVectors(GetTableName(), ids_to_delete); + stat = db_->DeleteVectors(table_info.table_id_, ids_to_delete); ASSERT_TRUE(stat.ok()); // std::this_thread::sleep_for(std::chrono::seconds(3)); // ensure raw data write to disk @@ -113,7 +113,7 @@ TEST_F(DeleteTest, delete_in_mem) { ASSERT_TRUE(stat.ok()); uint64_t row_count; - stat = db_->GetTableRowCount(GetTableName(), row_count); + stat = db_->GetTableRowCount(table_info.table_id_, row_count); ASSERT_TRUE(stat.ok()); ASSERT_EQ(row_count, nb - search_vectors.size()); @@ -124,7 +124,8 @@ TEST_F(DeleteTest, delete_in_mem) { std::vector tags; milvus::engine::ResultIds result_ids; milvus::engine::ResultDistances result_distances; - stat = db_->Query(dummy_context_, GetTableName(), tags, topk, nprobe, search, result_ids, result_distances); + stat = db_->Query(dummy_context_, table_info.table_id_, tags, topk, {{"nprobe", nprobe}}, search, result_ids, + result_distances); ASSERT_NE(result_ids[0], pair.first); // ASSERT_LT(result_distances[0], 1e-4); ASSERT_GT(result_distances[0], 1); @@ -136,7 +137,7 @@ TEST_F(DeleteTest, delete_on_disk) { auto stat = db_->CreateTable(table_info); milvus::engine::meta::TableSchema table_info_get; - table_info_get.table_id_ = GetTableName(); + table_info_get.table_id_ = table_info.table_id_; stat = db_->DescribeTable(table_info_get); ASSERT_TRUE(stat.ok()); ASSERT_EQ(table_info_get.dimension_, TABLE_DIM); @@ -149,7 +150,7 @@ TEST_F(DeleteTest, delete_on_disk) { xb.id_array_.push_back(i); } - stat = db_->InsertVectors(GetTableName(), "", xb); + stat = db_->InsertVectors(table_info.table_id_, "", xb); ASSERT_TRUE(stat.ok()); std::random_device rd; @@ -173,7 +174,7 @@ TEST_F(DeleteTest, delete_on_disk) { ASSERT_TRUE(stat.ok()); for (auto& kv : search_vectors) { - stat = db_->DeleteVector(GetTableName(), kv.first); + stat = db_->DeleteVector(table_info.table_id_, kv.first); ASSERT_TRUE(stat.ok()); } @@ -181,7 +182,7 @@ TEST_F(DeleteTest, delete_on_disk) { ASSERT_TRUE(stat.ok()); uint64_t row_count; - stat = db_->GetTableRowCount(GetTableName(), row_count); + stat = db_->GetTableRowCount(table_info.table_id_, row_count); ASSERT_TRUE(stat.ok()); ASSERT_EQ(row_count, nb - search_vectors.size()); @@ -192,7 +193,8 @@ TEST_F(DeleteTest, delete_on_disk) { std::vector tags; milvus::engine::ResultIds result_ids; milvus::engine::ResultDistances result_distances; - stat = db_->Query(dummy_context_, GetTableName(), tags, topk, nprobe, search, result_ids, result_distances); + stat = db_->Query(dummy_context_, table_info.table_id_, tags, topk, {{"nprobe", nprobe}}, search, result_ids, + result_distances); ASSERT_NE(result_ids[0], pair.first); // ASSERT_LT(result_distances[0], 1e-4); ASSERT_GT(result_distances[0], 1); @@ -204,7 +206,7 @@ TEST_F(DeleteTest, delete_multiple_times) { auto stat = db_->CreateTable(table_info); milvus::engine::meta::TableSchema table_info_get; - table_info_get.table_id_ = GetTableName(); + table_info_get.table_id_ = table_info.table_id_; stat = db_->DescribeTable(table_info_get); ASSERT_TRUE(stat.ok()); ASSERT_EQ(table_info_get.dimension_, TABLE_DIM); @@ -217,7 +219,7 @@ TEST_F(DeleteTest, delete_multiple_times) { xb.id_array_.push_back(i); } - stat = db_->InsertVectors(GetTableName(), "", xb); + stat = db_->InsertVectors(table_info.table_id_, "", xb); ASSERT_TRUE(stat.ok()); std::random_device rd; @@ -243,7 +245,7 @@ TEST_F(DeleteTest, delete_multiple_times) { int topk = 10, nprobe = 10; for (auto& pair : search_vectors) { std::vector to_delete{pair.first}; - stat = db_->DeleteVectors(GetTableName(), to_delete); + stat = db_->DeleteVectors(table_info.table_id_, to_delete); ASSERT_TRUE(stat.ok()); stat = db_->Flush(); @@ -254,20 +256,21 @@ TEST_F(DeleteTest, delete_multiple_times) { std::vector tags; milvus::engine::ResultIds result_ids; milvus::engine::ResultDistances result_distances; - stat = db_->Query(dummy_context_, GetTableName(), tags, topk, nprobe, search, result_ids, result_distances); + stat = db_->Query(dummy_context_, table_info.table_id_, tags, topk, {{"nprobe", nprobe}}, search, result_ids, + result_distances); ASSERT_NE(result_ids[0], pair.first); // ASSERT_LT(result_distances[0], 1e-4); ASSERT_GT(result_distances[0], 1); } } -TEST_F(DeleteTest, delete_with_index) { +TEST_F(DeleteTest, delete_before_create_index) { milvus::engine::meta::TableSchema table_info = BuildTableSchema(); table_info.engine_type_ = (int32_t)milvus::engine::EngineType::FAISS_IVFFLAT; auto stat = db_->CreateTable(table_info); milvus::engine::meta::TableSchema table_info_get; - table_info_get.table_id_ = GetTableName(); + table_info_get.table_id_ = table_info.table_id_; stat = db_->DescribeTable(table_info_get); ASSERT_TRUE(stat.ok()); ASSERT_EQ(table_info_get.dimension_, TABLE_DIM); @@ -280,7 +283,83 @@ TEST_F(DeleteTest, delete_with_index) { xb.id_array_.push_back(i); } - stat = db_->InsertVectors(GetTableName(), "", xb); + stat = db_->InsertVectors(table_info.table_id_, "", xb); + ASSERT_TRUE(stat.ok()); + + stat = db_->Flush(); + ASSERT_TRUE(stat.ok()); + + std::random_device rd; + std::mt19937 gen(rd()); + std::uniform_int_distribution dis(0, nb - 1); + + int64_t num_query = 10; + std::map search_vectors; + for (int64_t i = 0; i < num_query; ++i) { + int64_t index = dis(gen); + milvus::engine::VectorsData search; + search.vector_count_ = 1; + for (int64_t j = 0; j < TABLE_DIM; j++) { + search.float_data_.push_back(xb.float_data_[index * TABLE_DIM + j]); + } + search_vectors.insert(std::make_pair(xb.id_array_[index], search)); + } + + milvus::engine::IDNumbers ids_to_delete; + for (auto& kv : search_vectors) { + ids_to_delete.emplace_back(kv.first); + } + stat = db_->DeleteVectors(table_info.table_id_, ids_to_delete); + + stat = db_->Flush(); + ASSERT_TRUE(stat.ok()); + + milvus::engine::TableIndex index; + index.engine_type_ = (int)milvus::engine::EngineType::FAISS_IVFSQ8; + index.extra_params_ = {{"nlist", 100}}; + stat = db_->CreateIndex(table_info.table_id_, index); + ASSERT_TRUE(stat.ok()); + + uint64_t row_count; + stat = db_->GetTableRowCount(table_info.table_id_, row_count); + ASSERT_TRUE(stat.ok()); + ASSERT_EQ(row_count, nb - ids_to_delete.size()); + + int topk = 10, nprobe = 10; + for (auto& pair : search_vectors) { + auto& search = pair.second; + + std::vector tags; + milvus::engine::ResultIds result_ids; + milvus::engine::ResultDistances result_distances; + stat = db_->Query(dummy_context_, table_info.table_id_, tags, topk, {{"nprobe", nprobe}}, search, result_ids, + result_distances); + ASSERT_NE(result_ids[0], pair.first); + // ASSERT_LT(result_distances[0], 1e-4); + ASSERT_GT(result_distances[0], 1); + } +} + +TEST_F(DeleteTest, delete_with_index) { + milvus::engine::meta::TableSchema table_info = BuildTableSchema(); + table_info.engine_type_ = (int32_t)milvus::engine::EngineType::FAISS_IVFFLAT; + auto stat = db_->CreateTable(table_info); + + milvus::engine::meta::TableSchema table_info_get; + table_info_get.table_id_ = table_info.table_id_; + stat = db_->DescribeTable(table_info_get); + ASSERT_TRUE(stat.ok()); + ASSERT_EQ(table_info_get.dimension_, TABLE_DIM); + + int64_t nb = 10000; + milvus::engine::VectorsData xb; + BuildVectors(nb, xb); + + for (int64_t i = 0; i < nb; i++) { + xb.id_array_.push_back(i); + } + + stat = db_->InsertVectors(table_info.table_id_, "", xb); ASSERT_TRUE(stat.ok()); std::random_device rd; @@ -302,7 +381,7 @@ TEST_F(DeleteTest, delete_with_index) { milvus::engine::TableIndex index; index.engine_type_ = (int)milvus::engine::EngineType::FAISS_IVFSQ8; index.extra_params_ = {{"nlist", 100}}; - stat = db_->CreateIndex(GetTableName(), index); + stat = db_->CreateIndex(table_info.table_id_, index); ASSERT_TRUE(stat.ok()); // std::this_thread::sleep_for(std::chrono::seconds(3)); // ensure raw data write to disk @@ -313,13 +392,13 @@ TEST_F(DeleteTest, delete_with_index) { for (auto& kv : search_vectors) { ids_to_delete.emplace_back(kv.first); } - stat = db_->DeleteVectors(GetTableName(), ids_to_delete); + stat = db_->DeleteVectors(table_info.table_id_, ids_to_delete); stat = db_->Flush(); ASSERT_TRUE(stat.ok()); uint64_t row_count; - stat = db_->GetTableRowCount(GetTableName(), row_count); + stat = db_->GetTableRowCount(table_info.table_id_, row_count); ASSERT_TRUE(stat.ok()); ASSERT_EQ(row_count, nb - ids_to_delete.size()); @@ -330,7 +409,86 @@ TEST_F(DeleteTest, delete_with_index) { std::vector tags; milvus::engine::ResultIds result_ids; milvus::engine::ResultDistances result_distances; - stat = db_->Query(dummy_context_, GetTableName(), tags, topk, nprobe, search, result_ids, result_distances); + stat = db_->Query(dummy_context_, table_info.table_id_, tags, topk, {{"nprobe", nprobe}}, search, result_ids, + result_distances); + ASSERT_NE(result_ids[0], pair.first); + // ASSERT_LT(result_distances[0], 1e-4); + ASSERT_GT(result_distances[0], 1); + } +} + +TEST_F(DeleteTest, delete_multiple_times_with_index) { + milvus::engine::meta::TableSchema table_info = BuildTableSchema(); + auto stat = db_->CreateTable(table_info); + + milvus::engine::meta::TableSchema table_info_get; + table_info_get.table_id_ = table_info.table_id_; + stat = db_->DescribeTable(table_info_get); + ASSERT_TRUE(stat.ok()); + ASSERT_EQ(table_info_get.dimension_, TABLE_DIM); + + int64_t nb = 100000; + milvus::engine::VectorsData xb; + BuildVectors(nb, xb); + + for (int64_t i = 0; i < nb; i++) { + xb.id_array_.push_back(i); + } + + stat = db_->InsertVectors(table_info.table_id_, "", xb); + ASSERT_TRUE(stat.ok()); + + std::random_device rd; + std::mt19937 gen(rd()); + std::uniform_int_distribution dis(0, nb - 1); + + int64_t num_query = 10; + std::map search_vectors; + for (int64_t i = 0; i < num_query; ++i) { + int64_t index = dis(gen); + milvus::engine::VectorsData search; + search.vector_count_ = 1; + for (int64_t j = 0; j < TABLE_DIM; j++) { + search.float_data_.push_back(xb.float_data_[index * TABLE_DIM + j]); + } + search_vectors.insert(std::make_pair(xb.id_array_[index], search)); + } + + // std::this_thread::sleep_for(std::chrono::seconds(3)); // ensure raw data write to disk + stat = db_->Flush(); + ASSERT_TRUE(stat.ok()); + + milvus::engine::TableIndex index; + index.engine_type_ = (int)milvus::engine::EngineType::FAISS_IVFFLAT; + index.extra_params_ = {{"nlist", 1}}; + stat = db_->CreateIndex(table_info.table_id_, index); + ASSERT_TRUE(stat.ok()); + + int topk = 10, nprobe = 10; + int deleted = 0; + for (auto& pair : search_vectors) { + std::vector to_delete{pair.first}; + stat = db_->DeleteVectors(table_info.table_id_, to_delete); + ASSERT_TRUE(stat.ok()); + + stat = db_->Flush(); + ASSERT_TRUE(stat.ok()); + + ++deleted; + + uint64_t row_count; + stat = db_->GetTableRowCount(table_info.table_id_, row_count); + ASSERT_TRUE(stat.ok()); + ASSERT_EQ(row_count, nb - deleted); + + auto& search = pair.second; + + std::vector tags; + milvus::engine::ResultIds result_ids; + milvus::engine::ResultDistances result_distances; + stat = db_->Query(dummy_context_, table_info.table_id_, tags, topk, {{"nprobe", nprobe}}, search, result_ids, + result_distances); + ASSERT_TRUE(stat.ok()); ASSERT_NE(result_ids[0], pair.first); // ASSERT_LT(result_distances[0], 1e-4); ASSERT_GT(result_distances[0], 1); @@ -342,7 +500,7 @@ TEST_F(DeleteTest, delete_single_vector) { auto stat = db_->CreateTable(table_info); milvus::engine::meta::TableSchema table_info_get; - table_info_get.table_id_ = GetTableName(); + table_info_get.table_id_ = table_info.table_id_; stat = db_->DescribeTable(table_info_get); ASSERT_TRUE(stat.ok()); ASSERT_EQ(table_info_get.dimension_, TABLE_DIM); @@ -351,21 +509,21 @@ TEST_F(DeleteTest, delete_single_vector) { milvus::engine::VectorsData xb; BuildVectors(nb, xb); - stat = db_->InsertVectors(GetTableName(), "", xb); + stat = db_->InsertVectors(table_info.table_id_, "", xb); ASSERT_TRUE(stat.ok()); // std::this_thread::sleep_for(std::chrono::seconds(3)); // ensure raw data write to disk stat = db_->Flush(); ASSERT_TRUE(stat.ok()); - stat = db_->DeleteVectors(GetTableName(), xb.id_array_); + stat = db_->DeleteVectors(table_info.table_id_, xb.id_array_); ASSERT_TRUE(stat.ok()); stat = db_->Flush(); ASSERT_TRUE(stat.ok()); uint64_t row_count; - stat = db_->GetTableRowCount(GetTableName(), row_count); + stat = db_->GetTableRowCount(table_info.table_id_, row_count); ASSERT_TRUE(stat.ok()); ASSERT_EQ(row_count, 0); @@ -375,7 +533,7 @@ TEST_F(DeleteTest, delete_single_vector) { std::vector tags; milvus::engine::ResultIds result_ids; milvus::engine::ResultDistances result_distances; - stat = db_->Query(dummy_context_, GetTableName(), tags, topk, json_params, xb, result_ids, result_distances); + stat = db_->Query(dummy_context_, table_info.table_id_, tags, topk, json_params, xb, result_ids, result_distances); ASSERT_TRUE(result_ids.empty()); ASSERT_TRUE(result_distances.empty()); // ASSERT_EQ(result_ids[0], -1); @@ -388,7 +546,7 @@ TEST_F(DeleteTest, delete_add_create_index) { auto stat = db_->CreateTable(table_info); milvus::engine::meta::TableSchema table_info_get; - table_info_get.table_id_ = GetTableName(); + table_info_get.table_id_ = table_info.table_id_; stat = db_->DescribeTable(table_info_get); ASSERT_TRUE(stat.ok()); ASSERT_EQ(table_info_get.dimension_, TABLE_DIM); @@ -397,7 +555,7 @@ TEST_F(DeleteTest, delete_add_create_index) { milvus::engine::VectorsData xb; BuildVectors(nb, xb); - stat = db_->InsertVectors(GetTableName(), "", xb); + stat = db_->InsertVectors(table_info.table_id_, "", xb); ASSERT_TRUE(stat.ok()); // stat = db_->Flush(); @@ -405,27 +563,27 @@ TEST_F(DeleteTest, delete_add_create_index) { milvus::engine::TableIndex index; index.engine_type_ = (int)milvus::engine::EngineType::FAISS_IVFSQ8; index.extra_params_ = {{"nlist", 100}}; - stat = db_->CreateIndex(GetTableName(), index); + stat = db_->CreateIndex(table_info.table_id_, index); ASSERT_TRUE(stat.ok()); std::vector ids_to_delete; ids_to_delete.emplace_back(xb.id_array_.front()); - stat = db_->DeleteVectors(GetTableName(), ids_to_delete); + stat = db_->DeleteVectors(table_info.table_id_, ids_to_delete); ASSERT_TRUE(stat.ok()); milvus::engine::VectorsData xb2 = xb; xb2.id_array_.clear(); // same vector, different id - stat = db_->InsertVectors(GetTableName(), "", xb2); + stat = db_->InsertVectors(table_info.table_id_, "", xb2); ASSERT_TRUE(stat.ok()); // stat = db_->Flush(); // ASSERT_TRUE(stat.ok()); - stat = db_->CreateIndex(GetTableName(), index); + stat = db_->CreateIndex(table_info.table_id_, index); ASSERT_TRUE(stat.ok()); uint64_t row_count; - stat = db_->GetTableRowCount(GetTableName(), row_count); + stat = db_->GetTableRowCount(table_info.table_id_, row_count); ASSERT_TRUE(stat.ok()); ASSERT_EQ(row_count, nb * 2 - 1); @@ -439,15 +597,15 @@ TEST_F(DeleteTest, delete_add_create_index) { qb.float_data_.resize(TABLE_DIM); qb.vector_count_ = 1; qb.id_array_.clear(); - stat = db_->Query(dummy_context_, GetTableName(), tags, topk, json_params, qb, result_ids, result_distances); + stat = db_->Query(dummy_context_, table_info.table_id_, tags, topk, json_params, qb, result_ids, result_distances); ASSERT_EQ(result_ids[0], xb2.id_array_.front()); ASSERT_LT(result_distances[0], 1e-4); result_ids.clear(); result_distances.clear(); - stat = db_->QueryByID(dummy_context_, GetTableName(), tags, topk, json_params, ids_to_delete.front(), result_ids, - result_distances); + stat = db_->QueryByID(dummy_context_, table_info.table_id_, tags, topk, json_params, ids_to_delete.front(), + result_ids, result_distances); ASSERT_EQ(result_ids[0], -1); ASSERT_EQ(result_distances[0], std::numeric_limits::max()); } @@ -457,7 +615,7 @@ TEST_F(DeleteTest, delete_add_auto_flush) { auto stat = db_->CreateTable(table_info); milvus::engine::meta::TableSchema table_info_get; - table_info_get.table_id_ = GetTableName(); + table_info_get.table_id_ = table_info.table_id_; stat = db_->DescribeTable(table_info_get); ASSERT_TRUE(stat.ok()); ASSERT_EQ(table_info_get.dimension_, TABLE_DIM); @@ -466,7 +624,7 @@ TEST_F(DeleteTest, delete_add_auto_flush) { milvus::engine::VectorsData xb; BuildVectors(nb, xb); - stat = db_->InsertVectors(GetTableName(), "", xb); + stat = db_->InsertVectors(table_info.table_id_, "", xb); ASSERT_TRUE(stat.ok()); std::this_thread::sleep_for(std::chrono::seconds(2)); @@ -475,28 +633,28 @@ TEST_F(DeleteTest, delete_add_auto_flush) { // ASSERT_TRUE(stat.ok()); // milvus::engine::TableIndex index; // index.engine_type_ = (int)milvus::engine::EngineType::FAISS_IVFSQ8; - // stat = db_->CreateIndex(GetTableName(), index); + // stat = db_->CreateIndex(table_info.table_id_, index); // ASSERT_TRUE(stat.ok()); std::vector ids_to_delete; ids_to_delete.emplace_back(xb.id_array_.front()); - stat = db_->DeleteVectors(GetTableName(), ids_to_delete); + stat = db_->DeleteVectors(table_info.table_id_, ids_to_delete); ASSERT_TRUE(stat.ok()); milvus::engine::VectorsData xb2 = xb; xb2.id_array_.clear(); // same vector, different id - stat = db_->InsertVectors(GetTableName(), "", xb2); + stat = db_->InsertVectors(table_info.table_id_, "", xb2); ASSERT_TRUE(stat.ok()); std::this_thread::sleep_for(std::chrono::seconds(2)); // stat = db_->Flush(); // ASSERT_TRUE(stat.ok()); - // stat = db_->CreateIndex(GetTableName(), index); + // stat = db_->CreateIndex(table_info.table_id_, index); // ASSERT_TRUE(stat.ok()); uint64_t row_count; - stat = db_->GetTableRowCount(GetTableName(), row_count); + stat = db_->GetTableRowCount(table_info.table_id_, row_count); ASSERT_TRUE(stat.ok()); ASSERT_EQ(row_count, nb * 2 - 1); @@ -510,15 +668,15 @@ TEST_F(DeleteTest, delete_add_auto_flush) { qb.float_data_.resize(TABLE_DIM); qb.vector_count_ = 1; qb.id_array_.clear(); - stat = db_->Query(dummy_context_, GetTableName(), tags, topk, json_params, qb, result_ids, result_distances); + stat = db_->Query(dummy_context_, table_info.table_id_, tags, topk, json_params, qb, result_ids, result_distances); ASSERT_EQ(result_ids[0], xb2.id_array_.front()); ASSERT_LT(result_distances[0], 1e-4); result_ids.clear(); result_distances.clear(); - stat = db_->QueryByID(dummy_context_, GetTableName(), tags, topk, nprobe, ids_to_delete.front(), result_ids, - result_distances); + stat = db_->QueryByID(dummy_context_, table_info.table_id_, tags, topk, {{"nprobe", nprobe}}, ids_to_delete.front(), + result_ids, result_distances); ASSERT_EQ(result_ids[0], -1); ASSERT_EQ(result_distances[0], std::numeric_limits::max()); } @@ -528,7 +686,7 @@ TEST_F(CompactTest, compact_basic) { auto stat = db_->CreateTable(table_info); milvus::engine::meta::TableSchema table_info_get; - table_info_get.table_id_ = GetTableName(); + table_info_get.table_id_ = table_info.table_id_; stat = db_->DescribeTable(table_info_get); ASSERT_TRUE(stat.ok()); ASSERT_EQ(table_info_get.dimension_, TABLE_DIM); @@ -537,7 +695,7 @@ TEST_F(CompactTest, compact_basic) { milvus::engine::VectorsData xb; BuildVectors(nb, xb); - stat = db_->InsertVectors(GetTableName(), "", xb); + stat = db_->InsertVectors(table_info.table_id_, "", xb); ASSERT_TRUE(stat.ok()); stat = db_->Flush(); @@ -546,18 +704,18 @@ TEST_F(CompactTest, compact_basic) { std::vector ids_to_delete; ids_to_delete.emplace_back(xb.id_array_.front()); ids_to_delete.emplace_back(xb.id_array_.back()); - stat = db_->DeleteVectors(GetTableName(), ids_to_delete); + stat = db_->DeleteVectors(table_info.table_id_, ids_to_delete); ASSERT_TRUE(stat.ok()); stat = db_->Flush(); ASSERT_TRUE(stat.ok()); uint64_t row_count; - stat = db_->GetTableRowCount(GetTableName(), row_count); + stat = db_->GetTableRowCount(table_info.table_id_, row_count); ASSERT_TRUE(stat.ok()); ASSERT_EQ(row_count, nb - 2); - stat = db_->Compact(GetTableName()); + stat = db_->Compact(table_info.table_id_); ASSERT_TRUE(stat.ok()); const int topk = 1, nprobe = 1; @@ -569,8 +727,8 @@ TEST_F(CompactTest, compact_basic) { milvus::engine::VectorsData qb = xb; for (auto& id : ids_to_delete) { - stat = - db_->QueryByID(dummy_context_, GetTableName(), tags, topk, json_params, id, result_ids, result_distances); + stat = db_->QueryByID(dummy_context_, table_info.table_id_, tags, topk, json_params, id, result_ids, + result_distances); ASSERT_EQ(result_ids[0], -1); ASSERT_EQ(result_distances[0], std::numeric_limits::max()); } @@ -583,7 +741,7 @@ TEST_F(CompactTest, compact_with_index) { auto stat = db_->CreateTable(table_info); milvus::engine::meta::TableSchema table_info_get; - table_info_get.table_id_ = GetTableName(); + table_info_get.table_id_ = table_info.table_id_; stat = db_->DescribeTable(table_info_get); ASSERT_TRUE(stat.ok()); ASSERT_EQ(table_info_get.dimension_, TABLE_DIM); @@ -597,7 +755,7 @@ TEST_F(CompactTest, compact_with_index) { xb.id_array_.emplace_back(i); } - stat = db_->InsertVectors(GetTableName(), "", xb); + stat = db_->InsertVectors(table_info.table_id_, "", xb); ASSERT_TRUE(stat.ok()); std::random_device rd; @@ -618,7 +776,7 @@ TEST_F(CompactTest, compact_with_index) { milvus::engine::TableIndex index; index.engine_type_ = (int)milvus::engine::EngineType::FAISS_IVFSQ8; - stat = db_->CreateIndex(GetTableName(), index); + stat = db_->CreateIndex(table_info.table_id_, index); ASSERT_TRUE(stat.ok()); stat = db_->Flush(); @@ -628,25 +786,25 @@ TEST_F(CompactTest, compact_with_index) { for (auto& kv : search_vectors) { ids_to_delete.emplace_back(kv.first); } - stat = db_->DeleteVectors(GetTableName(), ids_to_delete); + stat = db_->DeleteVectors(table_info.table_id_, ids_to_delete); stat = db_->Flush(); ASSERT_TRUE(stat.ok()); uint64_t row_count; - stat = db_->GetTableRowCount(GetTableName(), row_count); + stat = db_->GetTableRowCount(table_info.table_id_, row_count); ASSERT_TRUE(stat.ok()); ASSERT_EQ(row_count, nb - ids_to_delete.size()); - stat = db_->Compact(GetTableName()); + stat = db_->Compact(table_info.table_id_); ASSERT_TRUE(stat.ok()); - stat = db_->GetTableRowCount(GetTableName(), row_count); + stat = db_->GetTableRowCount(table_info.table_id_, row_count); ASSERT_TRUE(stat.ok()); ASSERT_EQ(row_count, nb - ids_to_delete.size()); milvus::engine::TableIndex table_index; - stat = db_->DescribeIndex(GetTableName(), table_index); + stat = db_->DescribeIndex(table_info.table_id_, table_index); ASSERT_TRUE(stat.ok()); ASSERT_FLOAT_EQ(table_index.engine_type_, index.engine_type_); @@ -659,8 +817,8 @@ TEST_F(CompactTest, compact_with_index) { std::vector tags; milvus::engine::ResultIds result_ids; milvus::engine::ResultDistances result_distances; - stat = - db_->Query(dummy_context_, GetTableName(), tags, topk, json_params, search, result_ids, result_distances); + stat = db_->Query(dummy_context_, table_info.table_id_, tags, topk, json_params, search, result_ids, + result_distances); ASSERT_NE(result_ids[0], pair.first); // ASSERT_LT(result_distances[0], 1e-4); ASSERT_GT(result_distances[0], 1); diff --git a/core/unittest/db/test_search_by_id.cpp b/core/unittest/db/test_search_by_id.cpp index a4edb301b0..568626296b 100644 --- a/core/unittest/db/test_search_by_id.cpp +++ b/core/unittest/db/test_search_by_id.cpp @@ -68,7 +68,7 @@ TEST_F(SearchByIdTest, basic) { auto stat = db_->CreateTable(table_info); milvus::engine::meta::TableSchema table_info_get; - table_info_get.table_id_ = GetTableName(); + table_info_get.table_id_ = table_info.table_id_; stat = db_->DescribeTable(table_info_get); ASSERT_TRUE(stat.ok()); ASSERT_EQ(table_info_get.dimension_, TABLE_DIM); @@ -81,7 +81,7 @@ TEST_F(SearchByIdTest, basic) { xb.id_array_.push_back(i); } - stat = db_->InsertVectors(GetTableName(), "", xb); + stat = db_->InsertVectors(table_info.table_id_, "", xb); ASSERT_TRUE(stat.ok()); std::random_device rd; @@ -108,7 +108,8 @@ TEST_F(SearchByIdTest, basic) { milvus::engine::ResultIds result_ids; milvus::engine::ResultDistances result_distances; - stat = db_->QueryByID(dummy_context_, GetTableName(), tags, topk, json_params, i, result_ids, result_distances); + stat = db_->QueryByID(dummy_context_, table_info.table_id_, tags, topk, json_params, i, result_ids, + result_distances); ASSERT_EQ(result_ids[0], i); ASSERT_LT(result_distances[0], 1e-4); } @@ -119,7 +120,7 @@ TEST_F(SearchByIdTest, with_index) { auto stat = db_->CreateTable(table_info); milvus::engine::meta::TableSchema table_info_get; - table_info_get.table_id_ = GetTableName(); + table_info_get.table_id_ = table_info.table_id_; stat = db_->DescribeTable(table_info_get); ASSERT_TRUE(stat.ok()); ASSERT_EQ(table_info_get.dimension_, TABLE_DIM); @@ -132,7 +133,7 @@ TEST_F(SearchByIdTest, with_index) { xb.id_array_.push_back(i); } - stat = db_->InsertVectors(GetTableName(), "", xb); + stat = db_->InsertVectors(table_info.table_id_, "", xb); ASSERT_TRUE(stat.ok()); std::random_device rd; @@ -153,7 +154,7 @@ TEST_F(SearchByIdTest, with_index) { milvus::engine::TableIndex index; index.engine_type_ = (int)milvus::engine::EngineType::FAISS_IVFSQ8; index.extra_params_ = {{"nlist", 10}}; - stat = db_->CreateIndex(GetTableName(), index); + stat = db_->CreateIndex(table_info.table_id_, index); ASSERT_TRUE(stat.ok()); const int topk = 10, nprobe = 10; @@ -165,7 +166,8 @@ TEST_F(SearchByIdTest, with_index) { milvus::engine::ResultIds result_ids; milvus::engine::ResultDistances result_distances; - stat = db_->QueryByID(dummy_context_, GetTableName(), tags, topk, json_params, i, result_ids, result_distances); + stat = db_->QueryByID(dummy_context_, table_info.table_id_, tags, topk, json_params, i, result_ids, + result_distances); ASSERT_EQ(result_ids[0], i); ASSERT_LT(result_distances[0], 1e-3); } @@ -176,7 +178,7 @@ TEST_F(SearchByIdTest, with_delete) { auto stat = db_->CreateTable(table_info); milvus::engine::meta::TableSchema table_info_get; - table_info_get.table_id_ = GetTableName(); + table_info_get.table_id_ = table_info.table_id_; stat = db_->DescribeTable(table_info_get); ASSERT_TRUE(stat.ok()); ASSERT_EQ(table_info_get.dimension_, TABLE_DIM); @@ -189,7 +191,7 @@ TEST_F(SearchByIdTest, with_delete) { xb.id_array_.push_back(i); } - stat = db_->InsertVectors(GetTableName(), "", xb); + stat = db_->InsertVectors(table_info.table_id_, "", xb); ASSERT_TRUE(stat.ok()); std::random_device rd; @@ -211,7 +213,7 @@ TEST_F(SearchByIdTest, with_delete) { for (auto& id : ids_to_search) { ids_to_delete.emplace_back(id); } - stat = db_->DeleteVectors(GetTableName(), ids_to_delete); + stat = db_->DeleteVectors(table_info.table_id_, ids_to_delete); stat = db_->Flush(); ASSERT_TRUE(stat.ok()); @@ -225,7 +227,8 @@ TEST_F(SearchByIdTest, with_delete) { milvus::engine::ResultIds result_ids; milvus::engine::ResultDistances result_distances; - stat = db_->QueryByID(dummy_context_, GetTableName(), tags, topk, json_params, i, result_ids, result_distances); + stat = db_->QueryByID(dummy_context_, table_info.table_id_, tags, topk, json_params, i, result_ids, + result_distances); ASSERT_EQ(result_ids[0], -1); ASSERT_EQ(result_distances[0], std::numeric_limits::max()); } @@ -236,7 +239,7 @@ TEST_F(GetVectorByIdTest, basic) { auto stat = db_->CreateTable(table_info); milvus::engine::meta::TableSchema table_info_get; - table_info_get.table_id_ = GetTableName(); + table_info_get.table_id_ = table_info.table_id_; stat = db_->DescribeTable(table_info_get); ASSERT_TRUE(stat.ok()); ASSERT_EQ(table_info_get.dimension_, TABLE_DIM); @@ -249,7 +252,7 @@ TEST_F(GetVectorByIdTest, basic) { xb.id_array_.push_back(i); } - stat = db_->InsertVectors(GetTableName(), "", xb); + stat = db_->InsertVectors(table_info.table_id_, "", xb); ASSERT_TRUE(stat.ok()); std::random_device rd; @@ -277,11 +280,11 @@ TEST_F(GetVectorByIdTest, basic) { milvus::engine::ResultDistances result_distances; milvus::engine::VectorsData vector; - stat = db_->GetVectorByID(GetTableName(), id, vector); + stat = db_->GetVectorByID(table_info.table_id_, id, vector); ASSERT_TRUE(stat.ok()); - stat = - db_->Query(dummy_context_, GetTableName(), tags, topk, json_params, vector, result_ids, result_distances); + stat = db_->Query(dummy_context_, table_info.table_id_, tags, topk, json_params, vector, result_ids, + result_distances); ASSERT_TRUE(stat.ok()); ASSERT_EQ(result_ids[0], id); ASSERT_LT(result_distances[0], 1e-4); @@ -293,7 +296,7 @@ TEST_F(GetVectorByIdTest, with_index) { auto stat = db_->CreateTable(table_info); milvus::engine::meta::TableSchema table_info_get; - table_info_get.table_id_ = GetTableName(); + table_info_get.table_id_ = table_info.table_id_; stat = db_->DescribeTable(table_info_get); ASSERT_TRUE(stat.ok()); ASSERT_EQ(table_info_get.dimension_, TABLE_DIM); @@ -306,7 +309,7 @@ TEST_F(GetVectorByIdTest, with_index) { xb.id_array_.push_back(i); } - stat = db_->InsertVectors(GetTableName(), "", xb); + stat = db_->InsertVectors(table_info.table_id_, "", xb); ASSERT_TRUE(stat.ok()); std::random_device rd; @@ -327,7 +330,7 @@ TEST_F(GetVectorByIdTest, with_index) { milvus::engine::TableIndex index; index.extra_params_ = {{"nlist", 10}}; index.engine_type_ = (int)milvus::engine::EngineType::FAISS_IVFSQ8; - stat = db_->CreateIndex(GetTableName(), index); + stat = db_->CreateIndex(table_info.table_id_, index); ASSERT_TRUE(stat.ok()); const int topk = 10, nprobe = 10; @@ -340,11 +343,11 @@ TEST_F(GetVectorByIdTest, with_index) { milvus::engine::ResultDistances result_distances; milvus::engine::VectorsData vector; - stat = db_->GetVectorByID(GetTableName(), id, vector); + stat = db_->GetVectorByID(table_info.table_id_, id, vector); ASSERT_TRUE(stat.ok()); - stat = - db_->Query(dummy_context_, GetTableName(), tags, topk, json_params, vector, result_ids, result_distances); + stat = db_->Query(dummy_context_, table_info.table_id_, tags, topk, json_params, vector, result_ids, + result_distances); ASSERT_EQ(result_ids[0], id); ASSERT_LT(result_distances[0], 1e-3); } @@ -355,7 +358,7 @@ TEST_F(GetVectorByIdTest, with_delete) { auto stat = db_->CreateTable(table_info); milvus::engine::meta::TableSchema table_info_get; - table_info_get.table_id_ = GetTableName(); + table_info_get.table_id_ = table_info.table_id_; stat = db_->DescribeTable(table_info_get); ASSERT_TRUE(stat.ok()); ASSERT_EQ(table_info_get.dimension_, TABLE_DIM); @@ -368,7 +371,7 @@ TEST_F(GetVectorByIdTest, with_delete) { xb.id_array_.push_back(i); } - stat = db_->InsertVectors(GetTableName(), "", xb); + stat = db_->InsertVectors(table_info.table_id_, "", xb); ASSERT_TRUE(stat.ok()); std::random_device rd; @@ -390,7 +393,7 @@ TEST_F(GetVectorByIdTest, with_delete) { for (auto& id : ids_to_search) { ids_to_delete.emplace_back(id); } - stat = db_->DeleteVectors(GetTableName(), ids_to_delete); + stat = db_->DeleteVectors(table_info.table_id_, ids_to_delete); stat = db_->Flush(); ASSERT_TRUE(stat.ok()); @@ -402,7 +405,7 @@ TEST_F(GetVectorByIdTest, with_delete) { milvus::engine::ResultDistances result_distances; milvus::engine::VectorsData vector; - stat = db_->GetVectorByID(GetTableName(), id, vector); + stat = db_->GetVectorByID(table_info.table_id_, id, vector); ASSERT_TRUE(stat.ok()); ASSERT_TRUE(vector.float_data_.empty()); ASSERT_EQ(vector.vector_count_, 0); @@ -419,7 +422,7 @@ TEST_F(SearchByIdTest, BINARY) { ASSERT_TRUE(stat.ok()); milvus::engine::meta::TableSchema table_info_get; - table_info_get.table_id_ = GetTableName(); + table_info_get.table_id_ = table_info.table_id_; stat = db_->DescribeTable(table_info_get); ASSERT_TRUE(stat.ok()); ASSERT_EQ(table_info_get.dimension_, TABLE_DIM); @@ -445,7 +448,7 @@ TEST_F(SearchByIdTest, BINARY) { vectors.id_array_.emplace_back(k * nb + i); } - stat = db_->InsertVectors(GetTableName(), "", vectors); + stat = db_->InsertVectors(table_info.table_id_, "", vectors); ASSERT_TRUE(stat.ok()); } @@ -465,7 +468,7 @@ TEST_F(SearchByIdTest, BINARY) { ASSERT_TRUE(stat.ok()); uint64_t row_count; - stat = db_->GetTableRowCount(GetTableName(), row_count); + stat = db_->GetTableRowCount(table_info.table_id_, row_count); ASSERT_TRUE(stat.ok()); ASSERT_EQ(row_count, nb * insert_loop); @@ -479,12 +482,12 @@ TEST_F(SearchByIdTest, BINARY) { milvus::engine::ResultDistances result_distances; milvus::engine::VectorsData vector; - stat = db_->GetVectorByID(GetTableName(), id, vector); + stat = db_->GetVectorByID(table_info.table_id_, id, vector); ASSERT_TRUE(stat.ok()); ASSERT_EQ(vector.vector_count_, 1); - stat = - db_->Query(dummy_context_, GetTableName(), tags, topk, json_params, vector, result_ids, result_distances); + stat = db_->Query(dummy_context_, table_info.table_id_, tags, topk, json_params, vector, result_ids, + result_distances); ASSERT_TRUE(stat.ok()); ASSERT_EQ(result_ids[0], id); ASSERT_LT(result_distances[0], 1e-4); @@ -493,8 +496,8 @@ TEST_F(SearchByIdTest, BINARY) { result_ids.clear(); result_distances.clear(); - stat = - db_->QueryByID(dummy_context_, GetTableName(), tags, topk, json_params, id, result_ids, result_distances); + stat = db_->QueryByID(dummy_context_, table_info.table_id_, tags, topk, json_params, id, result_ids, + result_distances); ASSERT_TRUE(stat.ok()); ASSERT_EQ(result_ids[0], id); ASSERT_LT(result_distances[0], 1e-4); diff --git a/core/unittest/server/test_config.cpp b/core/unittest/server/test_config.cpp index f60fd3dc62..9c547b5036 100644 --- a/core/unittest/server/test_config.cpp +++ b/core/unittest/server/test_config.cpp @@ -685,10 +685,10 @@ TEST_F(ConfigTest, SERVER_CONFIG_VALID_FAIL_TEST) { milvus::Status s = config.LoadConfigFile(config_path + VALID_CONFIG_FILE); ASSERT_TRUE(s.ok()); - // fiu_enable("check_config_version_fail", 1, NULL, 0); - // s = config.ValidateConfig(); - // ASSERT_FALSE(s.ok()); - // fiu_disable("check_config_version_fail"); + fiu_enable("check_config_version_fail", 1, NULL, 0); + s = config.ValidateConfig(); + ASSERT_FALSE(s.ok()); + fiu_disable("check_config_version_fail"); /* server config */ fiu_enable("check_config_address_fail", 1, NULL, 0); diff --git a/core/unittest/server/test_web.cpp b/core/unittest/server/test_web.cpp index 0749aad75a..34becf6348 100644 --- a/core/unittest/server/test_web.cpp +++ b/core/unittest/server/test_web.cpp @@ -646,7 +646,7 @@ class TestClient : public oatpp::web::client::ApiClient { API_CALL("OPTIONS", "/collections/{collection_name}/indexes", optionsIndexes, PATH(String, collection_name, "collection_name")) - API_CALL("POST", "/tables/{table_name}/indexes", createIndex, PATH(String, table_name, "table_name"), + API_CALL("POST", "/collections/{collection_name}/indexes", createIndex, PATH(String, collection_name, "collection_name"), BODY_STRING(OString, body)) API_CALL("GET", "/collections/{collection_name}/indexes", getIndex, PATH(String, collection_name, "collection_name")) diff --git a/core/unittest/wrapper/test_wrapper.cpp b/core/unittest/wrapper/test_wrapper.cpp index ba3ed0f1a1..65e672e8c6 100644 --- a/core/unittest/wrapper/test_wrapper.cpp +++ b/core/unittest/wrapper/test_wrapper.cpp @@ -85,7 +85,7 @@ INSTANTIATE_TEST_CASE_P( // std::make_tuple(milvus::engine::IndexType::NSG_MIX, "Default", 128, 250000, 10, 10), #endif // std::make_tuple(milvus::engine::IndexType::SPTAG_KDT_RNT_CPU, "Default", 128, 100, 10, 10), - // std::make_tuple(milvus::engine::IndexType::SPTAG_BKT_RNT_CPU, "Default", 128, 100, 10, 10), + // std::make_tuple(milvus::engine::IndexType::SPTAG_BKT_RNT_CPU, "Default", 126, 100, 10, 10), std::make_tuple(milvus::engine::IndexType::HNSW, "Default", 64, 10000, 5, 10), std::make_tuple(milvus::engine::IndexType::FAISS_IDMAP, "Default", 64, 1000, 10, 10), std::make_tuple(milvus::engine::IndexType::FAISS_IVFFLAT_CPU, "Default", 64, 1000, 10, 10), diff --git a/sdk/examples/binary_vector/src/ClientTest.cpp b/sdk/examples/binary_vector/src/ClientTest.cpp index f6dbd4b8d2..db16575f61 100644 --- a/sdk/examples/binary_vector/src/ClientTest.cpp +++ b/sdk/examples/binary_vector/src/ClientTest.cpp @@ -22,52 +22,53 @@ namespace { -const char* TABLE_NAME = milvus_sdk::Utils::GenTableName().c_str(); +const char* COLLECTION_NAME = milvus_sdk::Utils::GenCollectionName().c_str(); -constexpr int64_t TABLE_DIMENSION = 512; -constexpr int64_t TABLE_INDEX_FILE_SIZE = 128; -constexpr milvus::MetricType TABLE_METRIC_TYPE = milvus::MetricType::TANIMOTO; -constexpr int64_t BATCH_ROW_COUNT = 100000; +constexpr int64_t COLLECTION_DIMENSION = 512; +constexpr int64_t COLLECTION_INDEX_FILE_SIZE = 128; +constexpr milvus::MetricType COLLECTION_METRIC_TYPE = milvus::MetricType::TANIMOTO; +constexpr int64_t BATCH_ENTITY_COUNT = 100000; constexpr int64_t NQ = 5; constexpr int64_t TOP_K = 10; constexpr int64_t NPROBE = 32; -constexpr int64_t SEARCH_TARGET = 5000; // change this value, result is different, ensure less than BATCH_ROW_COUNT -constexpr int64_t ADD_VECTOR_LOOP = 20; +constexpr int64_t SEARCH_TARGET = 5000; // change this value, result is different, ensure less than BATCH_ENTITY_COUNT +constexpr int64_t ADD_ENTITY_LOOP = 20; constexpr milvus::IndexType INDEX_TYPE = milvus::IndexType::IVFFLAT; -milvus::TableSchema -BuildTableSchema() { - milvus::TableSchema tb_schema = {TABLE_NAME, TABLE_DIMENSION, TABLE_INDEX_FILE_SIZE, TABLE_METRIC_TYPE}; - return tb_schema; +milvus::CollectionParam +BuildCollectionParam() { + milvus::CollectionParam + collection_param = {COLLECTION_NAME, COLLECTION_DIMENSION, COLLECTION_INDEX_FILE_SIZE, COLLECTION_METRIC_TYPE}; + return collection_param; } milvus::IndexParam BuildIndexParam() { JSON json_params = {{"nlist", 1024}}; - milvus::IndexParam index_param = {TABLE_NAME, INDEX_TYPE, json_params.dump()}; + milvus::IndexParam index_param = {COLLECTION_NAME, INDEX_TYPE, json_params.dump()}; return index_param; } void -BuildBinaryVectors(int64_t from, int64_t to, std::vector& vector_record_array, - std::vector& record_ids, int64_t dimension) { +BuildBinaryVectors(int64_t from, int64_t to, std::vector& entity_array, + std::vector& entity_ids, int64_t dimension) { if (to <= from) { return; } - vector_record_array.clear(); - record_ids.clear(); + entity_array.clear(); + entity_ids.clear(); int64_t dim_byte = dimension/8; for (int64_t k = from; k < to; k++) { - milvus::RowRecord record; - record.binary_data.resize(dim_byte); + milvus::Entity entity; + entity.binary_data.resize(dim_byte); for (int64_t i = 0; i < dim_byte; i++) { - record.binary_data[i] = (uint8_t)lrand48(); + entity.binary_data[i] = (uint8_t)lrand48(); } - vector_record_array.emplace_back(record); - record_ids.push_back(k); + entity_array.emplace_back(entity); + entity_ids.push_back(k); } } @@ -84,54 +85,54 @@ ClientTest::Test(const std::string& address, const std::string& port) { std::cout << "Connect function call status: " << stat.message() << std::endl; } - { // create table - milvus::TableSchema tb_schema = BuildTableSchema(); - stat = conn->CreateTable(tb_schema); - std::cout << "CreateTable function call status: " << stat.message() << std::endl; - milvus_sdk::Utils::PrintTableSchema(tb_schema); + { // create collection + milvus::CollectionParam collection_param = BuildCollectionParam(); + stat = conn->CreateCollection(collection_param); + std::cout << "CreateCollection function call status: " << stat.message() << std::endl; + milvus_sdk::Utils::PrintCollectionParam(collection_param); - bool has_table = conn->HasTable(tb_schema.table_name); - if (has_table) { - std::cout << "Table is created" << std::endl; + bool has_collection = conn->HasCollection(collection_param.collection_name); + if (has_collection) { + std::cout << "Collection is created" << std::endl; } } - std::vector> search_record_array; + std::vector> search_entity_array; { // insert vectors - for (int i = 0; i < ADD_VECTOR_LOOP; i++) { - std::vector record_array; - std::vector record_ids; - int64_t begin_index = i * BATCH_ROW_COUNT; + for (int i = 0; i < ADD_ENTITY_LOOP; i++) { + std::vector entity_array; + std::vector entity_ids; + int64_t begin_index = i * BATCH_ENTITY_COUNT; { // generate vectors - milvus_sdk::TimeRecorder rc("Build vectors No." + std::to_string(i)); + milvus_sdk::TimeRecorder rc("Build entities No." + std::to_string(i)); BuildBinaryVectors(begin_index, - begin_index + BATCH_ROW_COUNT, - record_array, - record_ids, - TABLE_DIMENSION); + begin_index + BATCH_ENTITY_COUNT, + entity_array, + entity_ids, + COLLECTION_DIMENSION); } - if (search_record_array.size() < NQ) { - search_record_array.push_back(std::make_pair(record_ids[SEARCH_TARGET], record_array[SEARCH_TARGET])); + if (search_entity_array.size() < NQ) { + search_entity_array.push_back(std::make_pair(entity_ids[SEARCH_TARGET], entity_array[SEARCH_TARGET])); } - std::string title = "Insert " + std::to_string(record_array.size()) + " vectors No." + std::to_string(i); + std::string title = "Insert " + std::to_string(entity_array.size()) + " entities No." + std::to_string(i); milvus_sdk::TimeRecorder rc(title); - stat = conn->Insert(TABLE_NAME, "", record_array, record_ids); - std::cout << "InsertVector function call status: " << stat.message() << std::endl; - std::cout << "Returned id array count: " << record_ids.size() << std::endl; + stat = conn->Insert(COLLECTION_NAME, "", entity_array, entity_ids); + std::cout << "Insert function call status: " << stat.message() << std::endl; + std::cout << "Returned id array count: " << entity_ids.size() << std::endl; } } { // flush buffer - stat = conn->FlushTable(TABLE_NAME); - std::cout << "FlushTable function call status: " << stat.message() << std::endl; + stat = conn->FlushCollection(COLLECTION_NAME); + std::cout << "FlushCollection function call status: " << stat.message() << std::endl; } { // search vectors std::vector partition_tags; milvus::TopKQueryResult topk_query_result; - milvus_sdk::Utils::DoSearch(conn, TABLE_NAME, partition_tags, TOP_K, NPROBE, search_record_array, + milvus_sdk::Utils::DoSearch(conn, COLLECTION_NAME, partition_tags, TOP_K, NPROBE, search_entity_array, topk_query_result); } @@ -144,7 +145,7 @@ ClientTest::Test(const std::string& address, const std::string& port) { std::cout << "CreateIndex function call status: " << stat.message() << std::endl; milvus::IndexParam index2; - stat = conn->DescribeIndex(TABLE_NAME, index2); + stat = conn->DescribeIndex(COLLECTION_NAME, index2); std::cout << "DescribeIndex function call status: " << stat.message() << std::endl; milvus_sdk::Utils::PrintIndexParam(index2); } @@ -152,13 +153,13 @@ ClientTest::Test(const std::string& address, const std::string& port) { { // search vectors std::vector partition_tags; milvus::TopKQueryResult topk_query_result; - milvus_sdk::Utils::DoSearch(conn, TABLE_NAME, partition_tags, TOP_K, NPROBE, search_record_array, + milvus_sdk::Utils::DoSearch(conn, COLLECTION_NAME, partition_tags, TOP_K, NPROBE, search_entity_array, topk_query_result); } - { // drop table - stat = conn->DropTable(TABLE_NAME); - std::cout << "DropTable function call status: " << stat.message() << std::endl; + { // drop collection + stat = conn->DropCollection(COLLECTION_NAME); + std::cout << "DropCollection function call status: " << stat.message() << std::endl; } milvus::Connection::Destroy(conn); diff --git a/sdk/examples/partition/src/ClientTest.cpp b/sdk/examples/partition/src/ClientTest.cpp index aca815c6ed..e5841fed70 100644 --- a/sdk/examples/partition/src/ClientTest.cpp +++ b/sdk/examples/partition/src/ClientTest.cpp @@ -22,12 +22,12 @@ namespace { -const char* TABLE_NAME = milvus_sdk::Utils::GenTableName().c_str(); +const char* COLLECTION_NAME = milvus_sdk::Utils::GenCollectionName().c_str(); -constexpr int64_t TABLE_DIMENSION = 512; -constexpr int64_t TABLE_INDEX_FILE_SIZE = 1024; -constexpr milvus::MetricType TABLE_METRIC_TYPE = milvus::MetricType::L2; -constexpr int64_t BATCH_ROW_COUNT = 10000; +constexpr int64_t COLLECTION_DIMENSION = 512; +constexpr int64_t COLLECTION_INDEX_FILE_SIZE = 1024; +constexpr milvus::MetricType COLLECTION_METRIC_TYPE = milvus::MetricType::L2; +constexpr int64_t BATCH_ENTITY_COUNT = 10000; constexpr int64_t NQ = 5; constexpr int64_t TOP_K = 10; constexpr int64_t NPROBE = 32; @@ -36,27 +36,45 @@ constexpr milvus::IndexType INDEX_TYPE = milvus::IndexType::IVFSQ8; constexpr int32_t PARTITION_COUNT = 5; constexpr int32_t TARGET_PARTITION = 3; -milvus::TableSchema -BuildTableSchema() { - milvus::TableSchema tb_schema = {TABLE_NAME, TABLE_DIMENSION, TABLE_INDEX_FILE_SIZE, TABLE_METRIC_TYPE}; - return tb_schema; +milvus::CollectionParam +BuildCollectionParam() { + milvus::CollectionParam + collection_param = {COLLECTION_NAME, COLLECTION_DIMENSION, COLLECTION_INDEX_FILE_SIZE, COLLECTION_METRIC_TYPE}; + return collection_param; } milvus::PartitionParam BuildPartitionParam(int32_t index) { std::string tag = std::to_string(index); - std::string partition_name = std::string(TABLE_NAME) + "_" + tag; - milvus::PartitionParam partition_param = {TABLE_NAME, tag}; + std::string partition_name = std::string(COLLECTION_NAME) + "_" + tag; + milvus::PartitionParam partition_param = {COLLECTION_NAME, tag}; return partition_param; } milvus::IndexParam BuildIndexParam() { JSON json_params = {{"nlist", 16384}}; - milvus::IndexParam index_param = {TABLE_NAME, INDEX_TYPE, json_params.dump()}; + milvus::IndexParam index_param = {COLLECTION_NAME, INDEX_TYPE, json_params.dump()}; return index_param; } +void +CountCollection(std::shared_ptr& conn) { + int64_t entity_count = 0; + auto stat = conn->CountCollection(COLLECTION_NAME, entity_count); + std::cout << COLLECTION_NAME << "(" << entity_count << " entities)" << std::endl; +} + +void +ShowCollectionInfo(std::shared_ptr& conn) { + CountCollection(conn); + + milvus::CollectionInfo collection_info; + auto stat = conn->ShowCollectionInfo(COLLECTION_NAME, collection_info); + milvus_sdk::Utils::PrintCollectionInfo(collection_info); + std::cout << "ShowCollectionInfo function call status: " << stat.message() << std::endl; +} + } // namespace void @@ -70,11 +88,11 @@ ClientTest::Test(const std::string& address, const std::string& port) { std::cout << "Connect function call status: " << stat.message() << std::endl; } - { // create table - milvus::TableSchema tb_schema = BuildTableSchema(); - stat = conn->CreateTable(tb_schema); - std::cout << "CreateTable function call status: " << stat.message() << std::endl; - milvus_sdk::Utils::PrintTableSchema(tb_schema); + { // create collection + milvus::CollectionParam tb_schema = BuildCollectionParam(); + stat = conn->CreateCollection(tb_schema); + std::cout << "CreateCollection function call status: " << stat.message() << std::endl; + milvus_sdk::Utils::PrintCollectionParam(tb_schema); } { // create partition @@ -87,7 +105,7 @@ ClientTest::Test(const std::string& address, const std::string& port) { // show partitions milvus::PartitionTagList partition_array; - stat = conn->ShowPartitions(TABLE_NAME, partition_array); + stat = conn->ShowPartitions(COLLECTION_NAME, partition_array); std::cout << partition_array.size() << " partitions created:" << std::endl; for (auto& partition_tag : partition_array) { @@ -96,64 +114,56 @@ ClientTest::Test(const std::string& address, const std::string& port) { } { // insert vectors - milvus_sdk::TimeRecorder rc("All vectors"); + milvus_sdk::TimeRecorder rc("All entities"); for (int i = 0; i < PARTITION_COUNT * 5; i++) { - std::vector record_array; - std::vector record_ids; - int64_t begin_index = i * BATCH_ROW_COUNT; + std::vector entity_array; + std::vector entity_ids; + int64_t begin_index = i * BATCH_ENTITY_COUNT; { // generate vectors - milvus_sdk::TimeRecorder rc("Build vectors No." + std::to_string(i)); - milvus_sdk::Utils::BuildVectors(begin_index, begin_index + BATCH_ROW_COUNT, record_array, record_ids, - TABLE_DIMENSION); + milvus_sdk::TimeRecorder rc("Build entities No." + std::to_string(i)); + milvus_sdk::Utils::BuildEntities(begin_index, + begin_index + BATCH_ENTITY_COUNT, + entity_array, + entity_ids, + COLLECTION_DIMENSION); } - std::string title = "Insert " + std::to_string(record_array.size()) + " vectors No." + std::to_string(i); + std::string title = "Insert " + std::to_string(entity_array.size()) + " entities No." + std::to_string(i); milvus_sdk::TimeRecorder rc(title); - stat = conn->Insert(TABLE_NAME, std::to_string(i % PARTITION_COUNT), record_array, record_ids); + stat = conn->Insert(COLLECTION_NAME, std::to_string(i % PARTITION_COUNT), entity_array, entity_ids); } } { // flush buffer - stat = conn->FlushTable(TABLE_NAME); - std::cout << "FlushTable function call status: " << stat.message() << std::endl; + stat = conn->FlushCollection(COLLECTION_NAME); + std::cout << "FlushCollection function call status: " << stat.message() << std::endl; } - { // table row count - int64_t row_count = 0; - stat = conn->CountTable(TABLE_NAME, row_count); - std::cout << TABLE_NAME << "(" << row_count << " rows)" << std::endl; - } + ShowCollectionInfo(conn); - { // get table information - milvus::TableInfo table_info; - stat = conn->ShowTableInfo(TABLE_NAME, table_info); - milvus_sdk::Utils::PrintTableInfo(table_info); - std::cout << "ShowTableInfo function call status: " << stat.message() << std::endl; - } - - std::vector> search_record_array; + std::vector> search_entity_array; { // build search vectors - std::vector record_array; - std::vector record_ids; - int64_t index = TARGET_PARTITION * BATCH_ROW_COUNT + SEARCH_TARGET; - milvus_sdk::Utils::BuildVectors(index, index + 1, record_array, record_ids, TABLE_DIMENSION); - search_record_array.push_back(std::make_pair(record_ids[0], record_array[0])); + std::vector entity_array; + std::vector entity_ids; + int64_t index = TARGET_PARTITION * BATCH_ENTITY_COUNT + SEARCH_TARGET; + milvus_sdk::Utils::BuildEntities(index, index + 1, entity_array, entity_ids, COLLECTION_DIMENSION); + search_entity_array.push_back(std::make_pair(entity_ids[0], entity_array[0])); } { // search vectors std::cout << "Search in correct partition" << std::endl; std::vector partition_tags = {std::to_string(TARGET_PARTITION)}; milvus::TopKQueryResult topk_query_result; - milvus_sdk::Utils::DoSearch(conn, TABLE_NAME, partition_tags, TOP_K, NPROBE, search_record_array, + milvus_sdk::Utils::DoSearch(conn, COLLECTION_NAME, partition_tags, TOP_K, NPROBE, search_entity_array, topk_query_result); std::cout << "Search in wrong partition" << std::endl; partition_tags = {"0"}; - milvus_sdk::Utils::DoSearch(conn, TABLE_NAME, partition_tags, TOP_K, NPROBE, search_record_array, + milvus_sdk::Utils::DoSearch(conn, COLLECTION_NAME, partition_tags, TOP_K, NPROBE, search_entity_array, topk_query_result); std::cout << "Search by regex matched partition tag" << std::endl; partition_tags = {"\\d"}; - milvus_sdk::Utils::DoSearch(conn, TABLE_NAME, partition_tags, TOP_K, NPROBE, search_record_array, + milvus_sdk::Utils::DoSearch(conn, COLLECTION_NAME, partition_tags, TOP_K, NPROBE, search_entity_array, topk_query_result); } @@ -166,57 +176,42 @@ ClientTest::Test(const std::string& address, const std::string& port) { std::cout << "CreateIndex function call status: " << stat.message() << std::endl; milvus::IndexParam index2; - stat = conn->DescribeIndex(TABLE_NAME, index2); + stat = conn->DescribeIndex(COLLECTION_NAME, index2); std::cout << "DescribeIndex function call status: " << stat.message() << std::endl; milvus_sdk::Utils::PrintIndexParam(index2); } - { // table row count - int64_t row_count = 0; - stat = conn->CountTable(TABLE_NAME, row_count); - std::cout << TABLE_NAME << "(" << row_count << " rows)" << std::endl; - } - - { // get table information - milvus::TableInfo table_info; - stat = conn->ShowTableInfo(TABLE_NAME, table_info); - milvus_sdk::Utils::PrintTableInfo(table_info); - std::cout << "ShowTableInfo function call status: " << stat.message() << std::endl; - } + ShowCollectionInfo(conn); { // drop partition - milvus::PartitionParam param1 = {TABLE_NAME, std::to_string(TARGET_PARTITION)}; + milvus::PartitionParam param1 = {COLLECTION_NAME, std::to_string(TARGET_PARTITION)}; milvus_sdk::Utils::PrintPartitionParam(param1); stat = conn->DropPartition(param1); std::cout << "DropPartition function call status: " << stat.message() << std::endl; } - { // table row count - int64_t row_count = 0; - stat = conn->CountTable(TABLE_NAME, row_count); - std::cout << TABLE_NAME << "(" << row_count << " rows)" << std::endl; - } + CountCollection(conn); - { // search vectors - std::cout << "Search in whole table" << std::endl; + { // search vectors, will get search error since we delete a partition + std::cout << "Search in whole collection after delete one partition" << std::endl; std::vector partition_tags; milvus::TopKQueryResult topk_query_result; - milvus_sdk::Utils::DoSearch(conn, TABLE_NAME, partition_tags, TOP_K, NPROBE, search_record_array, + milvus_sdk::Utils::DoSearch(conn, COLLECTION_NAME, partition_tags, TOP_K, NPROBE, search_entity_array, topk_query_result); } { // drop index - stat = conn->DropIndex(TABLE_NAME); + stat = conn->DropIndex(COLLECTION_NAME); std::cout << "DropIndex function call status: " << stat.message() << std::endl; - int64_t row_count = 0; - stat = conn->CountTable(TABLE_NAME, row_count); - std::cout << TABLE_NAME << "(" << row_count << " rows)" << std::endl; + int64_t entity_count = 0; + stat = conn->CountCollection(COLLECTION_NAME, entity_count); + std::cout << COLLECTION_NAME << "(" << entity_count << " entities)" << std::endl; } - { // drop table - stat = conn->DropTable(TABLE_NAME); - std::cout << "DropTable function call status: " << stat.message() << std::endl; + { // drop collection + stat = conn->DropCollection(COLLECTION_NAME); + std::cout << "DropCollection function call status: " << stat.message() << std::endl; } milvus::Connection::Destroy(conn); diff --git a/sdk/examples/simple/src/ClientTest.cpp b/sdk/examples/simple/src/ClientTest.cpp index 572f120dcc..fb2e2b138d 100644 --- a/sdk/examples/simple/src/ClientTest.cpp +++ b/sdk/examples/simple/src/ClientTest.cpp @@ -19,20 +19,19 @@ #include #include - namespace { -const char* TABLE_NAME = milvus_sdk::Utils::GenTableName().c_str(); +const char* COLLECTION_NAME = milvus_sdk::Utils::GenCollectionName().c_str(); -constexpr int64_t TABLE_DIMENSION = 512; -constexpr int64_t TABLE_INDEX_FILE_SIZE = 1024; -constexpr milvus::MetricType TABLE_METRIC_TYPE = milvus::MetricType::L2; -constexpr int64_t BATCH_ROW_COUNT = 100000; +constexpr int64_t COLLECTION_DIMENSION = 512; +constexpr int64_t COLLECTION_INDEX_FILE_SIZE = 1024; +constexpr milvus::MetricType COLLECTION_METRIC_TYPE = milvus::MetricType::L2; +constexpr int64_t BATCH_ENTITY_COUNT = 100000; constexpr int64_t NQ = 5; constexpr int64_t TOP_K = 10; constexpr int64_t NPROBE = 32; constexpr int64_t SEARCH_TARGET = 5000; // change this value, result is different -constexpr int64_t ADD_VECTOR_LOOP = 5; +constexpr int64_t ADD_ENTITY_LOOP = 5; constexpr milvus::IndexType INDEX_TYPE = milvus::IndexType::IVFSQ8; constexpr int32_t NLIST = 16384; @@ -63,199 +62,201 @@ ClientTest::ShowSdkVersion() { } void -ClientTest::ShowTables(std::vector& tables) { - milvus::Status stat = conn_->ShowTables(tables); - std::cout << "ShowTables function call status: " << stat.message() << std::endl; - std::cout << "All tables: " << std::endl; - for (auto& table : tables) { - int64_t row_count = 0; - stat = conn_->CountTable(table, row_count); - std::cout << "\t" << table << "(" << row_count << " rows)" << std::endl; +ClientTest::ShowCollections(std::vector& collection_array) { + milvus::Status stat = conn_->ShowCollections(collection_array); + std::cout << "ShowCollections function call status: " << stat.message() << std::endl; + std::cout << "All collections: " << std::endl; + for (auto& collection : collection_array) { + int64_t entity_count = 0; + stat = conn_->CountCollection(collection, entity_count); + std::cout << "\t" << collection << "(" << entity_count << " entities)" << std::endl; } } void -ClientTest::CreateTable(const std::string& table_name, int64_t dim, milvus::MetricType type) { - milvus::TableSchema tb_schema = {table_name, dim, TABLE_INDEX_FILE_SIZE, type}; - milvus::Status stat = conn_->CreateTable(tb_schema); - std::cout << "CreateTable function call status: " << stat.message() << std::endl; - milvus_sdk::Utils::PrintTableSchema(tb_schema); +ClientTest::CreateCollection(const std::string& collection_name, int64_t dim, milvus::MetricType type) { + milvus::CollectionParam collection_param = {collection_name, dim, COLLECTION_INDEX_FILE_SIZE, type}; + milvus::Status stat = conn_->CreateCollection(collection_param); + std::cout << "CreateCollection function call status: " << stat.message() << std::endl; + milvus_sdk::Utils::PrintCollectionParam(collection_param); - bool has_table = conn_->HasTable(tb_schema.table_name); + bool has_table = conn_->HasCollection(collection_param.collection_name); if (has_table) { - std::cout << "Table is created" << std::endl; + std::cout << "Collection is created" << std::endl; } } void -ClientTest::DescribeTable(const std::string& table_name) { - milvus::TableSchema tb_schema; - milvus::Status stat = conn_->DescribeTable(table_name, tb_schema); - std::cout << "DescribeTable function call status: " << stat.message() << std::endl; - milvus_sdk::Utils::PrintTableSchema(tb_schema); +ClientTest::DescribeCollection(const std::string& collection_name) { + milvus::CollectionParam collection_param; + milvus::Status stat = conn_->DescribeCollection(collection_name, collection_param); + std::cout << "DescribeCollection function call status: " << stat.message() << std::endl; + milvus_sdk::Utils::PrintCollectionParam(collection_param); } void -ClientTest::InsertVectors(const std::string& table_name, int64_t dim) { - for (int i = 0; i < ADD_VECTOR_LOOP; i++) { - std::vector record_array; +ClientTest::InsertEntities(const std::string& collection_name, int64_t dim) { + for (int i = 0; i < ADD_ENTITY_LOOP; i++) { + std::vector entity_array; std::vector record_ids; - int64_t begin_index = i * BATCH_ROW_COUNT; + int64_t begin_index = i * BATCH_ENTITY_COUNT; { // generate vectors - milvus_sdk::TimeRecorder rc("Build vectors No." + std::to_string(i)); - milvus_sdk::Utils::BuildVectors(begin_index, begin_index + BATCH_ROW_COUNT, record_array, record_ids, dim); + milvus_sdk::TimeRecorder rc("Build entities No." + std::to_string(i)); + milvus_sdk::Utils::BuildEntities(begin_index, + begin_index + BATCH_ENTITY_COUNT, + entity_array, + record_ids, + dim); } - std::string title = "Insert " + std::to_string(record_array.size()) + " vectors No." + std::to_string(i); + std::string title = "Insert " + std::to_string(entity_array.size()) + " entities No." + std::to_string(i); milvus_sdk::TimeRecorder rc(title); - milvus::Status stat = conn_->Insert(table_name, "", record_array, record_ids); - std::cout << "InsertVector function call status: " << stat.message() << std::endl; + milvus::Status stat = conn_->Insert(collection_name, "", entity_array, record_ids); + std::cout << "InsertEntities function call status: " << stat.message() << std::endl; std::cout << "Returned id array count: " << record_ids.size() << std::endl; } } void -ClientTest::BuildSearchVectors(int64_t nq, int64_t dim) { - search_record_array_.clear(); +ClientTest::BuildSearchEntities(int64_t nq, int64_t dim) { + search_entity_array_.clear(); search_id_array_.clear(); for (int64_t i = 0; i < nq; i++) { - std::vector record_array; + std::vector entity_array; std::vector record_ids; - int64_t index = i * BATCH_ROW_COUNT + SEARCH_TARGET; - milvus_sdk::Utils::BuildVectors(index, index + 1, record_array, record_ids, dim); - search_record_array_.push_back(std::make_pair(record_ids[0], record_array[0])); + int64_t index = i * BATCH_ENTITY_COUNT + SEARCH_TARGET; + milvus_sdk::Utils::BuildEntities(index, index + 1, entity_array, record_ids, dim); + search_entity_array_.push_back(std::make_pair(record_ids[0], entity_array[0])); search_id_array_.push_back(record_ids[0]); } } void -ClientTest::Flush(const std::string& table_name) { +ClientTest::Flush(const std::string& collection_name) { milvus_sdk::TimeRecorder rc("Flush"); - milvus::Status stat = conn_->FlushTable(table_name); - std::cout << "FlushTable function call status: " << stat.message() << std::endl; + milvus::Status stat = conn_->FlushCollection(collection_name); + std::cout << "FlushCollection function call status: " << stat.message() << std::endl; } void -ClientTest::ShowTableInfo(const std::string& table_name) { - milvus::TableInfo table_info; - milvus::Status stat = conn_->ShowTableInfo(table_name, table_info); - milvus_sdk::Utils::PrintTableInfo(table_info); - std::cout << "ShowTableInfo function call status: " << stat.message() << std::endl; +ClientTest::ShowCollectionInfo(const std::string& collection_name) { + milvus::CollectionInfo collection_info; + milvus::Status stat = conn_->ShowCollectionInfo(collection_name, collection_info); + milvus_sdk::Utils::PrintCollectionInfo(collection_info); + std::cout << "ShowCollectionInfo function call status: " << stat.message() << std::endl; } void -ClientTest::GetVectorById(const std::string& table_name, int64_t id) { - milvus::RowRecord vector_data; - milvus::Status stat = conn_->GetVectorByID(table_name, id, vector_data); - std::cout << "The vector " << id << " has " << vector_data.float_data.size() << " float elements" << std::endl; - std::cout << "GetVectorByID function call status: " << stat.message() << std::endl; +ClientTest::GetEntityById(const std::string& collection_name, int64_t id) { + milvus::Entity entity; + milvus::Status stat = conn_->GetEntityByID(collection_name, id, entity); + std::cout << "The entity " << id << " has " << entity.float_data.size() << " float elements" << std::endl; + std::cout << "GetEntityById function call status: " << stat.message() << std::endl; } void -ClientTest::SearchVectors(const std::string& table_name, int64_t topk, int64_t nprobe) { +ClientTest::SearchEntities(const std::string& collection_name, int64_t topk, int64_t nprobe) { std::vector partition_tags; milvus::TopKQueryResult topk_query_result; - milvus_sdk::Utils::DoSearch(conn_, table_name, partition_tags, topk, nprobe, search_record_array_, + milvus_sdk::Utils::DoSearch(conn_, collection_name, partition_tags, topk, nprobe, search_entity_array_, topk_query_result); } void -ClientTest::SearchVectorsByIds(const std::string& table_name, int64_t topk, int64_t nprobe) { - std::vector partition_tags; - milvus::TopKQueryResult topk_query_result; - milvus_sdk::Utils::DoSearch(conn_, table_name, partition_tags, topk, nprobe, search_id_array_, topk_query_result); -} - -void -ClientTest::CreateIndex(const std::string& table_name, milvus::IndexType type, int64_t nlist) { +ClientTest::CreateIndex(const std::string& collection_name, milvus::IndexType type, int64_t nlist) { milvus_sdk::TimeRecorder rc("Create index"); std::cout << "Wait until create all index done" << std::endl; JSON json_params = {{"nlist", nlist}}; - milvus::IndexParam index1 = {table_name, type, json_params.dump()}; + milvus::IndexParam index1 = {collection_name, type, json_params.dump()}; milvus_sdk::Utils::PrintIndexParam(index1); milvus::Status stat = conn_->CreateIndex(index1); std::cout << "CreateIndex function call status: " << stat.message() << std::endl; milvus::IndexParam index2; - stat = conn_->DescribeIndex(table_name, index2); + stat = conn_->DescribeIndex(collection_name, index2); std::cout << "DescribeIndex function call status: " << stat.message() << std::endl; milvus_sdk::Utils::PrintIndexParam(index2); } void -ClientTest::PreloadTable(const std::string& table_name) { - milvus::Status stat = conn_->PreloadTable(table_name); - std::cout << "PreloadTable function call status: " << stat.message() << std::endl; +ClientTest::PreloadCollection(const std::string& collection_name) { + milvus::Status stat = conn_->PreloadCollection(collection_name); + std::cout << "PreloadCollection function call status: " << stat.message() << std::endl; } void -ClientTest::DeleteByIds(const std::string& table_name, const std::vector& id_array) { - milvus::Status stat = conn_->DeleteByID(table_name, id_array); +ClientTest::DeleteByIds(const std::string& collection_name, const std::vector& id_array) { + std::cout << "Delete entity: "; + for (auto id : id_array) { + std::cout << "\t" << id; + } + std::cout << std::endl; + + milvus::Status stat = conn_->DeleteByID(collection_name, id_array); std::cout << "DeleteByID function call status: " << stat.message() << std::endl; { milvus_sdk::TimeRecorder rc("Flush"); - stat = conn_->FlushTable(table_name); - std::cout << "FlushTable function call status: " << stat.message() << std::endl; + stat = conn_->FlushCollection(collection_name); + std::cout << "FlushCollection function call status: " << stat.message() << std::endl; } { // compact table milvus_sdk::TimeRecorder rc1("Compact"); - stat = conn_->CompactTable(table_name); - std::cout << "CompactTable function call status: " << stat.message() << std::endl; + stat = conn_->CompactCollection(collection_name); + std::cout << "CompactCollection function call status: " << stat.message() << std::endl; } } void -ClientTest::DropIndex(const std::string& table_name) { - milvus::Status stat = conn_->DropIndex(table_name); +ClientTest::DropIndex(const std::string& collection_name) { + milvus::Status stat = conn_->DropIndex(collection_name); std::cout << "DropIndex function call status: " << stat.message() << std::endl; int64_t row_count = 0; - stat = conn_->CountTable(table_name, row_count); - std::cout << table_name << "(" << row_count << " rows)" << std::endl; + stat = conn_->CountCollection(collection_name, row_count); + std::cout << collection_name << "(" << row_count << " rows)" << std::endl; } void -ClientTest::DropTable(const std::string& table_name) { - milvus::Status stat = conn_->DropTable(table_name); - std::cout << "DropTable function call status: " << stat.message() << std::endl; +ClientTest::DropCollection(const std::string& collection_name) { + milvus::Status stat = conn_->DropCollection(collection_name); + std::cout << "DropCollection function call status: " << stat.message() << std::endl; } void ClientTest::Test() { - std::string table_name = TABLE_NAME; - int64_t dim = TABLE_DIMENSION; - milvus::MetricType metric_type = TABLE_METRIC_TYPE; + std::string collection_name = COLLECTION_NAME; + int64_t dim = COLLECTION_DIMENSION; + milvus::MetricType metric_type = COLLECTION_METRIC_TYPE; ShowServerVersion(); ShowSdkVersion(); std::vector table_array; - ShowTables(table_array); + ShowCollections(table_array); - CreateTable(table_name, dim, metric_type); - DescribeTable(table_name); + CreateCollection(collection_name, dim, metric_type); + DescribeCollection(collection_name); - InsertVectors(table_name, dim); - BuildSearchVectors(NQ, dim); - Flush(table_name); - ShowTableInfo(table_name); + InsertEntities(collection_name, dim); + BuildSearchEntities(NQ, dim); + Flush(collection_name); + ShowCollectionInfo(collection_name); - GetVectorById(table_name, search_id_array_[0]); - SearchVectors(table_name, TOP_K, NPROBE); - SearchVectorsByIds(table_name, TOP_K, NPROBE); + GetEntityById(collection_name, search_id_array_[0]); + SearchEntities(collection_name, TOP_K, NPROBE); - CreateIndex(table_name, INDEX_TYPE, NLIST); - ShowTableInfo(table_name); + CreateIndex(collection_name, INDEX_TYPE, NLIST); + ShowCollectionInfo(collection_name); - PreloadTable(table_name); + PreloadCollection(collection_name); std::vector delete_ids = {search_id_array_[0], search_id_array_[1]}; - DeleteByIds(table_name, delete_ids); - SearchVectors(table_name, TOP_K, NPROBE); + DeleteByIds(collection_name, delete_ids); + SearchEntities(collection_name, TOP_K, NPROBE); // this line get two search error since we delete two entities - DropIndex(table_name); - DropTable(table_name); + DropIndex(collection_name); + DropCollection(collection_name); } diff --git a/sdk/examples/simple/src/ClientTest.h b/sdk/examples/simple/src/ClientTest.h index 1082878109..2e676353eb 100644 --- a/sdk/examples/simple/src/ClientTest.h +++ b/sdk/examples/simple/src/ClientTest.h @@ -29,41 +29,54 @@ class ClientTest { private: void ShowServerVersion(); + void ShowSdkVersion(); + void - ShowTables(std::vector&); + ShowCollections(std::vector&); + void - CreateTable(const std::string&, int64_t, milvus::MetricType); + CreateCollection(const std::string&, int64_t, milvus::MetricType); + void - DescribeTable(const std::string&); + DescribeCollection(const std::string&); + void - InsertVectors(const std::string&, int64_t); + InsertEntities(const std::string&, int64_t); + void - BuildSearchVectors(int64_t, int64_t); + BuildSearchEntities(int64_t, int64_t); + void Flush(const std::string&); + void - ShowTableInfo(const std::string&); + ShowCollectionInfo(const std::string&); + void - GetVectorById(const std::string&, int64_t); + GetEntityById(const std::string&, int64_t); + void - SearchVectors(const std::string&, int64_t, int64_t); - void - SearchVectorsByIds(const std::string&, int64_t, int64_t); + SearchEntities(const std::string&, int64_t, int64_t); + void CreateIndex(const std::string&, milvus::IndexType, int64_t); + void - PreloadTable(const std::string&); + PreloadCollection(const std::string&); + void - DeleteByIds(const std::string&, const std::vector&); + DeleteByIds(const std::string&, const std::vector& id_array); + void DropIndex(const std::string&); + void - DropTable(const std::string&); + DropCollection(const std::string&); private: std::shared_ptr conn_; - std::vector> search_record_array_; + std::vector> search_entity_array_; std::vector search_id_array_; }; diff --git a/sdk/examples/utils/Utils.cpp b/sdk/examples/utils/Utils.cpp index 068e5099b0..ed709f4ffe 100644 --- a/sdk/examples/utils/Utils.cpp +++ b/sdk/examples/utils/Utils.cpp @@ -64,8 +64,8 @@ Utils::Sleep(int seconds) { } const std::string& -Utils::GenTableName() { - static std::string s_id("tbl_" + CurrentTime()); +Utils::GenCollectionName() { + static std::string s_id("C_" + CurrentTime()); return s_id; } @@ -97,19 +97,19 @@ Utils::IndexTypeName(const milvus::IndexType& index_type) { } void -Utils::PrintTableSchema(const milvus::TableSchema& tb_schema) { +Utils::PrintCollectionParam(const milvus::CollectionParam& collection_param) { BLOCK_SPLITER - std::cout << "Table name: " << tb_schema.table_name << std::endl; - std::cout << "Table dimension: " << tb_schema.dimension << std::endl; - std::cout << "Table index file size: " << tb_schema.index_file_size << std::endl; - std::cout << "Table metric type: " << MetricTypeName(tb_schema.metric_type) << std::endl; + std::cout << "Collection name: " << collection_param.collection_name << std::endl; + std::cout << "Collection dimension: " << collection_param.dimension << std::endl; + std::cout << "Collection index file size: " << collection_param.index_file_size << std::endl; + std::cout << "Collection metric type: " << MetricTypeName(collection_param.metric_type) << std::endl; BLOCK_SPLITER } void Utils::PrintPartitionParam(const milvus::PartitionParam& partition_param) { BLOCK_SPLITER - std::cout << "Table name: " << partition_param.table_name << std::endl; + std::cout << "Collection name: " << partition_param.collection_name << std::endl; std::cout << "Partition tag: " << partition_param.partition_tag << std::endl; BLOCK_SPLITER } @@ -117,40 +117,40 @@ Utils::PrintPartitionParam(const milvus::PartitionParam& partition_param) { void Utils::PrintIndexParam(const milvus::IndexParam& index_param) { BLOCK_SPLITER - std::cout << "Index table name: " << index_param.table_name << std::endl; + std::cout << "Index collection name: " << index_param.collection_name << std::endl; std::cout << "Index type: " << IndexTypeName(index_param.index_type) << std::endl; std::cout << "Index extra_params: " << index_param.extra_params << std::endl; BLOCK_SPLITER } void -Utils::BuildVectors(int64_t from, int64_t to, std::vector& vector_record_array, - std::vector& record_ids, int64_t dimension) { +Utils::BuildEntities(int64_t from, int64_t to, std::vector& entity_array, + std::vector& entity_ids, int64_t dimension) { if (to <= from) { return; } - vector_record_array.clear(); - record_ids.clear(); + entity_array.clear(); + entity_ids.clear(); for (int64_t k = from; k < to; k++) { - milvus::RowRecord record; - record.float_data.resize(dimension); + milvus::Entity entity; + entity.float_data.resize(dimension); for (int64_t i = 0; i < dimension; i++) { - record.float_data[i] = (float)(k % (i + 1)); + entity.float_data[i] = (float)(k % (i + 1)); } - vector_record_array.emplace_back(record); - record_ids.push_back(k); + entity_array.emplace_back(entity); + entity_ids.push_back(k); } } void -Utils::PrintSearchResult(const std::vector>& search_record_array, +Utils::PrintSearchResult(const std::vector>& entity_array, const milvus::TopKQueryResult& topk_query_result) { BLOCK_SPLITER std::cout << "Returned result count: " << topk_query_result.size() << std::endl; - if (topk_query_result.size() != search_record_array.size()) { + if (topk_query_result.size() != entity_array.size()) { std::cout << "ERROR: Returned result count not equal nq" << std::endl; return; } @@ -158,8 +158,8 @@ Utils::PrintSearchResult(const std::vector for (size_t i = 0; i < topk_query_result.size(); i++) { const milvus::QueryResult& one_result = topk_query_result[i]; size_t topk = one_result.ids.size(); - auto search_id = search_record_array[i].first; - std::cout << "No." << i << " vector " << search_id << " top " << topk << " search result:" << std::endl; + auto search_id = entity_array[i].first; + std::cout << "No." << i << " entity " << search_id << " top " << topk << " search result:" << std::endl; for (size_t j = 0; j < topk; j++) { std::cout << "\t" << one_result.ids[j] << "\t" << one_result.distances[j] << std::endl; } @@ -168,13 +168,13 @@ Utils::PrintSearchResult(const std::vector } void -Utils::CheckSearchResult(const std::vector>& search_record_array, +Utils::CheckSearchResult(const std::vector>& entity_array, const milvus::TopKQueryResult& topk_query_result) { BLOCK_SPLITER size_t nq = topk_query_result.size(); for (size_t i = 0; i < nq; i++) { const milvus::QueryResult& one_result = topk_query_result[i]; - auto search_id = search_record_array[i].first; + auto search_id = entity_array[i].first; uint64_t match_index = one_result.ids.size(); for (uint64_t index = 0; index < one_result.ids.size(); index++) { @@ -195,15 +195,15 @@ Utils::CheckSearchResult(const std::vector } void -Utils::DoSearch(std::shared_ptr conn, const std::string& table_name, +Utils::DoSearch(std::shared_ptr conn, const std::string& collection_name, const std::vector& partition_tags, int64_t top_k, int64_t nprobe, - const std::vector>& search_record_array, + const std::vector>& entity_array, milvus::TopKQueryResult& topk_query_result) { topk_query_result.clear(); - std::vector record_array; - for (auto& pair : search_record_array) { - record_array.push_back(pair.second); + std::vector temp_entity_array; + for (auto& pair : entity_array) { + temp_entity_array.push_back(pair.second); } { @@ -211,90 +211,33 @@ Utils::DoSearch(std::shared_ptr conn, const std::string& tab JSON json_params = {{"nprobe", nprobe}}; milvus_sdk::TimeRecorder rc("search"); milvus::Status stat = - conn->Search(table_name, partition_tags, record_array, top_k, json_params.dump(), topk_query_result); - std::cout << "SearchVector function call status: " << stat.message() << std::endl; + conn->Search(collection_name, + partition_tags, + temp_entity_array, + top_k, + json_params.dump(), + topk_query_result); + std::cout << "Search function call status: " << stat.message() << std::endl; BLOCK_SPLITER } - PrintSearchResult(search_record_array, topk_query_result); - CheckSearchResult(search_record_array, topk_query_result); -} - -void -Utils::DoSearch(std::shared_ptr conn, const std::string& table_name, - const std::vector& partition_tags, int64_t top_k, int64_t nprobe, - const std::vector& search_id_array, milvus::TopKQueryResult& topk_query_result) { - topk_query_result.clear(); - - { - BLOCK_SPLITER - JSON json_params = {{"nprobe", nprobe}}; - for (auto& search_id : search_id_array) { - milvus_sdk::TimeRecorder rc("search by id " + std::to_string(search_id)); - milvus::TopKQueryResult result; - milvus::Status - stat = conn->SearchByID(table_name, partition_tags, search_id, top_k, json_params.dump(), result); - topk_query_result.insert(topk_query_result.end(), std::make_move_iterator(result.begin()), - std::make_move_iterator(result.end())); - std::cout << "SearchByID function call status: " << stat.message() << std::endl; - } - BLOCK_SPLITER - } - - if (topk_query_result.size() != search_id_array.size()) { - std::cout << "ERROR: Returned result count does not equal nq" << std::endl; - return; - } - - BLOCK_SPLITER - for (size_t i = 0; i < topk_query_result.size(); i++) { - const milvus::QueryResult& one_result = topk_query_result[i]; - size_t topk = one_result.ids.size(); - auto search_id = search_id_array[i]; - std::cout << "No." << i << " vector " << search_id << " top " << topk << " search result:" << std::endl; - for (size_t j = 0; j < topk; j++) { - std::cout << "\t" << one_result.ids[j] << "\t" << one_result.distances[j] << std::endl; - } - } - BLOCK_SPLITER - - BLOCK_SPLITER - size_t nq = topk_query_result.size(); - for (size_t i = 0; i < nq; i++) { - const milvus::QueryResult& one_result = topk_query_result[i]; - auto search_id = search_id_array[i]; - - uint64_t match_index = one_result.ids.size(); - for (uint64_t index = 0; index < one_result.ids.size(); index++) { - if (search_id == one_result.ids[index]) { - match_index = index; - break; - } - } - - if (match_index >= one_result.ids.size()) { - std::cout << "The topk result is wrong: not return search target in result set" << std::endl; - } else { - std::cout << "No." << i << " Check result successfully for target: " << search_id << " at top " - << match_index << std::endl; - } - } - BLOCK_SPLITER + PrintSearchResult(entity_array, topk_query_result); + CheckSearchResult(entity_array, topk_query_result); } void PrintPartitionStat(const milvus::PartitionStat& partition_stat) { - std::cout << "\tPartition " << partition_stat.tag << " row count: " << partition_stat.row_count << std::endl; + std::cout << "\tPartition " << partition_stat.tag << " entity count: " << partition_stat.row_count << std::endl; for (auto& seg_stat : partition_stat.segments_stat) { - std::cout << "\t\tsegment " << seg_stat.segment_name << " row count: " << seg_stat.row_count + std::cout << "\t\tsegment " << seg_stat.segment_name << " entity count: " << seg_stat.row_count << " index: " << seg_stat.index_name << " data size: " << seg_stat.data_size << std::endl; } } void -Utils::PrintTableInfo(const milvus::TableInfo& info) { +Utils::PrintCollectionInfo(const milvus::CollectionInfo& info) { BLOCK_SPLITER - std::cout << "Table " << " total row count: " << info.total_row_count << std::endl; + std::cout << "Collection " << " total entity count: " << info.total_row_count << std::endl; for (const milvus::PartitionStat& partition_stat : info.partitions_stat) { PrintPartitionStat(partition_stat); } diff --git a/sdk/examples/utils/Utils.h b/sdk/examples/utils/Utils.h index c89f5151a9..f83ee069b7 100644 --- a/sdk/examples/utils/Utils.h +++ b/sdk/examples/utils/Utils.h @@ -32,7 +32,7 @@ class Utils { CurrentTmDate(int64_t offset_day = 0); static const std::string& - GenTableName(); + GenCollectionName(); static void Sleep(int seconds); @@ -44,7 +44,7 @@ class Utils { IndexTypeName(const milvus::IndexType& index_type); static void - PrintTableSchema(const milvus::TableSchema& tb_schema); + PrintCollectionParam(const milvus::CollectionParam& collection_param); static void PrintPartitionParam(const milvus::PartitionParam& partition_param); @@ -53,31 +53,25 @@ class Utils { PrintIndexParam(const milvus::IndexParam& index_param); static void - BuildVectors(int64_t from, int64_t to, std::vector& vector_record_array, - std::vector& record_ids, int64_t dimension); + BuildEntities(int64_t from, int64_t to, std::vector& entity_array, + std::vector& entity_ids, int64_t dimension); static void - PrintSearchResult(const std::vector>& search_record_array, + PrintSearchResult(const std::vector>& entity_array, const milvus::TopKQueryResult& topk_query_result); static void - CheckSearchResult(const std::vector>& search_record_array, + CheckSearchResult(const std::vector>& entity_array, const milvus::TopKQueryResult& topk_query_result); static void - DoSearch(std::shared_ptr conn, const std::string& table_name, + DoSearch(std::shared_ptr conn, const std::string& collection_name, const std::vector& partition_tags, int64_t top_k, int64_t nprobe, - const std::vector>& search_record_array, + const std::vector>& entity_array, milvus::TopKQueryResult& topk_query_result); static void - DoSearch(std::shared_ptr conn, const std::string& table_name, - const std::vector& partition_tags, int64_t top_k, int64_t nprobe, - const std::vector& search_id_array, - milvus::TopKQueryResult& topk_query_result); - - static void - PrintTableInfo(const milvus::TableInfo& info); + PrintCollectionInfo(const milvus::CollectionInfo& collection_info); }; } // namespace milvus_sdk diff --git a/sdk/grpc/ClientProxy.cpp b/sdk/grpc/ClientProxy.cpp index 7dae7b3b16..219f78e4ad 100644 --- a/sdk/grpc/ClientProxy.cpp +++ b/sdk/grpc/ClientProxy.cpp @@ -31,12 +31,12 @@ UriCheck(const std::string& uri) { template void -ConstructSearchParam(const std::string& table_name, +ConstructSearchParam(const std::string& collection_name, const std::vector& partition_tag_array, int64_t topk, const std::string& extra_params, T& search_param) { - search_param.set_table_name(table_name); + search_param.set_table_name(collection_name); search_param.set_topk(topk); milvus::grpc::KeyValuePair* kv = search_param.add_extra_params(); kv->set_key(EXTRA_PARAM_KEY); @@ -48,7 +48,7 @@ ConstructSearchParam(const std::string& table_name, } void -CopyRowRecord(::milvus::grpc::RowRecord* target, const RowRecord& src) { +CopyRowRecord(::milvus::grpc::RowRecord* target, const Entity& src) { if (!src.float_data.empty()) { auto vector_data = target->mutable_float_data(); vector_data->Resize(static_cast(src.float_data.size()), 0.0); @@ -86,7 +86,7 @@ ClientProxy::Connect(const ConnectParam& param) { return Status::OK(); } - std::string reason = "connect Failed!"; + std::string reason = "Connect failed!"; connected_ = false; return Status(StatusCode::NotConnected, reason); } @@ -111,7 +111,7 @@ ClientProxy::Connected() const { std::string info; return client_ptr_->Cmd("", info); } catch (std::exception& ex) { - return Status(StatusCode::NotConnected, "connection lost: " + std::string(ex.what())); + return Status(StatusCode::NotConnected, "Connection lost: " + std::string(ex.what())); } } @@ -132,305 +132,6 @@ ClientProxy::ClientVersion() const { return MILVUS_SDK_VERSION; } -Status -ClientProxy::CreateTable(const TableSchema& param) { - try { - ::milvus::grpc::TableSchema schema; - schema.set_table_name(param.table_name); - schema.set_dimension(param.dimension); - schema.set_index_file_size(param.index_file_size); - schema.set_metric_type(static_cast(param.metric_type)); - - return client_ptr_->CreateTable(schema); - } catch (std::exception& ex) { - return Status(StatusCode::UnknownError, "Failed to create table: " + std::string(ex.what())); - } -} - -bool -ClientProxy::HasTable(const std::string& table_name) { - Status status = Status::OK(); - ::milvus::grpc::TableName grpc_table_name; - grpc_table_name.set_table_name(table_name); - bool result = client_ptr_->HasTable(grpc_table_name, status); - return result; -} - -Status -ClientProxy::DropTable(const std::string& table_name) { - try { - ::milvus::grpc::TableName grpc_table_name; - grpc_table_name.set_table_name(table_name); - return client_ptr_->DropTable(grpc_table_name); - } catch (std::exception& ex) { - return Status(StatusCode::UnknownError, "Failed to drop table: " + std::string(ex.what())); - } -} - -Status -ClientProxy::CreateIndex(const IndexParam& index_param) { - try { - ::milvus::grpc::IndexParam grpc_index_param; - grpc_index_param.set_table_name(index_param.table_name); - grpc_index_param.set_index_type(static_cast(index_param.index_type)); - milvus::grpc::KeyValuePair* kv = grpc_index_param.add_extra_params(); - kv->set_key(EXTRA_PARAM_KEY); - kv->set_value(index_param.extra_params); - return client_ptr_->CreateIndex(grpc_index_param); - } catch (std::exception& ex) { - return Status(StatusCode::UnknownError, "Failed to build index: " + std::string(ex.what())); - } -} - -Status -ClientProxy::Insert(const std::string& table_name, const std::string& partition_tag, - const std::vector& record_array, std::vector& id_array) { - Status status = Status::OK(); - try { - ::milvus::grpc::InsertParam insert_param; - insert_param.set_table_name(table_name); - insert_param.set_partition_tag(partition_tag); - - for (auto& record : record_array) { - ::milvus::grpc::RowRecord* grpc_record = insert_param.add_row_record_array(); - CopyRowRecord(grpc_record, record); - } - - // Single thread - ::milvus::grpc::VectorIds vector_ids; - if (!id_array.empty()) { - /* set user's ids */ - auto row_ids = insert_param.mutable_row_id_array(); - row_ids->Resize(static_cast(id_array.size()), -1); - memcpy(row_ids->mutable_data(), id_array.data(), id_array.size() * sizeof(int64_t)); - status = client_ptr_->Insert(insert_param, vector_ids); - } else { - status = client_ptr_->Insert(insert_param, vector_ids); - /* return Milvus generated ids back to user */ - id_array.insert(id_array.end(), vector_ids.vector_id_array().begin(), vector_ids.vector_id_array().end()); - } - } catch (std::exception& ex) { - return Status(StatusCode::UnknownError, "Failed to add vector: " + std::string(ex.what())); - } - - return status; -} - -Status -ClientProxy::GetVectorByID(const std::string& table_name, int64_t vector_id, RowRecord& vector_data) { - try { - ::milvus::grpc::VectorIdentity vector_identity; - vector_identity.set_table_name(table_name); - vector_identity.set_id(vector_id); - - ::milvus::grpc::VectorData grpc_data; - Status status = client_ptr_->GetVectorByID(vector_identity, grpc_data); - if (!status.ok()) { - return status; - } - - int float_size = grpc_data.vector_data().float_data_size(); - if (float_size > 0) { - vector_data.float_data.resize(float_size); - memcpy(vector_data.float_data.data(), grpc_data.vector_data().float_data().data(), - float_size * sizeof(float)); - } - - auto byte_size = grpc_data.vector_data().binary_data().length(); - if (byte_size > 0) { - vector_data.binary_data.resize(byte_size); - memcpy(vector_data.binary_data.data(), grpc_data.vector_data().binary_data().data(), byte_size); - } - - return status; - } catch (std::exception& ex) { - return Status(StatusCode::UnknownError, "Failed to get vector by id: " + std::string(ex.what())); - } -} - -Status -ClientProxy::GetIDsInSegment(const std::string& table_name, const std::string& segment_name, - std::vector& id_array) { - try { - ::milvus::grpc::GetVectorIDsParam param; - param.set_table_name(table_name); - param.set_segment_name(segment_name); - - ::milvus::grpc::VectorIds vector_ids; - Status status = client_ptr_->GetIDsInSegment(param, vector_ids); - if (!status.ok()) { - return status; - } - - id_array.insert(id_array.end(), vector_ids.vector_id_array().begin(), vector_ids.vector_id_array().end()); - - return status; - } catch (std::exception& ex) { - return Status(StatusCode::UnknownError, "Failed to get vector by id: " + std::string(ex.what())); - } -} - -Status -ClientProxy::Search(const std::string& table_name, const std::vector& partition_tag_array, - const std::vector& query_record_array, int64_t topk, const std::string& extra_params, - TopKQueryResult& topk_query_result) { - try { - // step 1: convert vectors data - ::milvus::grpc::SearchParam search_param; - ConstructSearchParam(table_name, - partition_tag_array, - topk, - extra_params, - search_param); - - for (auto& record : query_record_array) { - ::milvus::grpc::RowRecord* row_record = search_param.add_query_record_array(); - CopyRowRecord(row_record, record); - } - - // step 2: search vectors - ::milvus::grpc::TopKQueryResult result; - Status status = client_ptr_->Search(search_param, result); - if (result.row_num() == 0) { - return status; - } - - // step 3: convert result array - topk_query_result.reserve(result.row_num()); - int64_t nq = result.row_num(); - int64_t topk = result.ids().size() / nq; - for (int64_t i = 0; i < result.row_num(); i++) { - milvus::QueryResult one_result; - one_result.ids.resize(topk); - one_result.distances.resize(topk); - memcpy(one_result.ids.data(), result.ids().data() + topk * i, topk * sizeof(int64_t)); - memcpy(one_result.distances.data(), result.distances().data() + topk * i, topk * sizeof(float)); - topk_query_result.emplace_back(one_result); - } - - return status; - } catch (std::exception& ex) { - return Status(StatusCode::UnknownError, "Failed to search vectors: " + std::string(ex.what())); - } -} - -Status -ClientProxy::SearchByID(const std::string& table_name, - const std::vector& partition_tag_array, - int64_t query_id, - int64_t topk, - const std::string& extra_params, - TopKQueryResult& topk_query_result) { - try { - // step 1: convert vector id array - ::milvus::grpc::SearchByIDParam search_param; - ConstructSearchParam(table_name, - partition_tag_array, - topk, - extra_params, - search_param); - search_param.set_id(query_id); - - // step 2: search vectors - ::milvus::grpc::TopKQueryResult result; - Status status = client_ptr_->SearchByID(search_param, result); - if (result.row_num() == 0) { - return status; - } - - // step 4: convert result array - topk_query_result.reserve(result.row_num()); - int64_t nq = result.row_num(); - int64_t topk = result.ids().size() / nq; - for (int64_t i = 0; i < result.row_num(); i++) { - milvus::QueryResult one_result; - one_result.ids.resize(topk); - one_result.distances.resize(topk); - memcpy(one_result.ids.data(), result.ids().data() + topk * i, topk * sizeof(int64_t)); - memcpy(one_result.distances.data(), result.distances().data() + topk * i, topk * sizeof(float)); - topk_query_result.emplace_back(one_result); - } - - return status; - } catch (std::exception& ex) { - return Status(StatusCode::UnknownError, "Failed to search vectors: " + std::string(ex.what())); - } -} - -Status -ClientProxy::DescribeTable(const std::string& table_name, TableSchema& table_schema) { - try { - ::milvus::grpc::TableSchema grpc_schema; - - Status status = client_ptr_->DescribeTable(table_name, grpc_schema); - - table_schema.table_name = grpc_schema.table_name(); - table_schema.dimension = grpc_schema.dimension(); - table_schema.index_file_size = grpc_schema.index_file_size(); - table_schema.metric_type = static_cast(grpc_schema.metric_type()); - - return status; - } catch (std::exception& ex) { - return Status(StatusCode::UnknownError, "Failed to describe table: " + std::string(ex.what())); - } -} - -Status -ClientProxy::CountTable(const std::string& table_name, int64_t& row_count) { - try { - Status status; - ::milvus::grpc::TableName grpc_table_name; - grpc_table_name.set_table_name(table_name); - row_count = client_ptr_->CountTable(grpc_table_name, status); - return status; - } catch (std::exception& ex) { - return Status(StatusCode::UnknownError, "Failed to show tables: " + std::string(ex.what())); - } -} - -Status -ClientProxy::ShowTables(std::vector& table_array) { - try { - Status status; - milvus::grpc::TableNameList table_name_list; - status = client_ptr_->ShowTables(table_name_list); - - table_array.resize(table_name_list.table_names_size()); - for (uint64_t i = 0; i < table_name_list.table_names_size(); ++i) { - table_array[i] = table_name_list.table_names(i); - } - return status; - } catch (std::exception& ex) { - return Status(StatusCode::UnknownError, "Failed to show tables: " + std::string(ex.what())); - } -} - -Status -ClientProxy::ShowTableInfo(const std::string& table_name, TableInfo& table_info) { - try { - Status status; - ::milvus::grpc::TableName grpc_table_name; - grpc_table_name.set_table_name(table_name); - milvus::grpc::TableInfo grpc_table_info; - status = client_ptr_->ShowTableInfo(grpc_table_name, grpc_table_info); - - // get native info - table_info.total_row_count = grpc_table_info.total_row_count(); - - // get partitions info - for (int i = 0; i < grpc_table_info.partitions_stat_size(); i++) { - auto& grpc_partition_stat = grpc_table_info.partitions_stat(i); - PartitionStat partition_stat; - ConstructPartitionStat(grpc_partition_stat, partition_stat); - table_info.partitions_stat.emplace_back(partition_stat); - } - - return status; - } catch (std::exception& ex) { - return Status(StatusCode::UnknownError, "Failed to show table info: " + std::string(ex.what())); - } -} - std::string ClientProxy::ServerVersion() const { Status status = Status::OK(); @@ -458,127 +159,6 @@ ClientProxy::ServerStatus() const { } } -std::string -ClientProxy::DumpTaskTables() const { - if (channel_ == nullptr) { - return "not connected to server"; - } - - try { - std::string dummy; - Status status = client_ptr_->Cmd("tasktable", dummy); - return dummy; - } catch (std::exception& ex) { - return "connection lost"; - } -} - -Status -ClientProxy::DeleteByID(const std::string& table_name, const std::vector& id_array) { - try { - ::milvus::grpc::DeleteByIDParam delete_by_id_param; - delete_by_id_param.set_table_name(table_name); - - for (auto id : id_array) { - delete_by_id_param.add_id_array(id); - } - - return client_ptr_->DeleteByID(delete_by_id_param); - } catch (std::exception& ex) { - return Status(StatusCode::UnknownError, "Failed to delete by range: " + std::string(ex.what())); - } -} - -Status -ClientProxy::PreloadTable(const std::string& table_name) const { - try { - ::milvus::grpc::TableName grpc_table_name; - grpc_table_name.set_table_name(table_name); - Status status = client_ptr_->PreloadTable(grpc_table_name); - return status; - } catch (std::exception& ex) { - return Status(StatusCode::UnknownError, "Failed to preload tables: " + std::string(ex.what())); - } -} - -Status -ClientProxy::DescribeIndex(const std::string& table_name, IndexParam& index_param) const { - try { - ::milvus::grpc::TableName grpc_table_name; - grpc_table_name.set_table_name(table_name); - - ::milvus::grpc::IndexParam grpc_index_param; - Status status = client_ptr_->DescribeIndex(grpc_table_name, grpc_index_param); - index_param.index_type = static_cast(grpc_index_param.index_type()); - - for (int i = 0; i < grpc_index_param.extra_params_size(); i++) { - const milvus::grpc::KeyValuePair& kv = grpc_index_param.extra_params(i); - if (kv.key() == EXTRA_PARAM_KEY) { - index_param.extra_params = kv.value(); - } - } - - return status; - } catch (std::exception& ex) { - return Status(StatusCode::UnknownError, "Failed to describe index: " + std::string(ex.what())); - } -} - -Status -ClientProxy::DropIndex(const std::string& table_name) const { - try { - ::milvus::grpc::TableName grpc_table_name; - grpc_table_name.set_table_name(table_name); - Status status = client_ptr_->DropIndex(grpc_table_name); - return status; - } catch (std::exception& ex) { - return Status(StatusCode::UnknownError, "Failed to drop index: " + std::string(ex.what())); - } -} - -Status -ClientProxy::CreatePartition(const PartitionParam& partition_param) { - try { - ::milvus::grpc::PartitionParam grpc_partition_param; - grpc_partition_param.set_table_name(partition_param.table_name); - grpc_partition_param.set_tag(partition_param.partition_tag); - Status status = client_ptr_->CreatePartition(grpc_partition_param); - return status; - } catch (std::exception& ex) { - return Status(StatusCode::UnknownError, "Failed to create partition: " + std::string(ex.what())); - } -} - -Status -ClientProxy::ShowPartitions(const std::string& table_name, PartitionTagList& partition_tag_array) const { - try { - ::milvus::grpc::TableName grpc_table_name; - grpc_table_name.set_table_name(table_name); - ::milvus::grpc::PartitionList grpc_partition_list; - Status status = client_ptr_->ShowPartitions(grpc_table_name, grpc_partition_list); - partition_tag_array.resize(grpc_partition_list.partition_tag_array_size()); - for (uint64_t i = 0; i < grpc_partition_list.partition_tag_array_size(); ++i) { - partition_tag_array[i] = grpc_partition_list.partition_tag_array(i); - } - return status; - } catch (std::exception& ex) { - return Status(StatusCode::UnknownError, "Failed to show partitions: " + std::string(ex.what())); - } -} - -Status -ClientProxy::DropPartition(const PartitionParam& partition_param) { - try { - ::milvus::grpc::PartitionParam grpc_partition_param; - grpc_partition_param.set_table_name(partition_param.table_name); - grpc_partition_param.set_tag(partition_param.partition_tag); - Status status = client_ptr_->DropPartition(grpc_partition_param); - return status; - } catch (std::exception& ex) { - return Status(StatusCode::UnknownError, "Failed to drop partition: " + std::string(ex.what())); - } -} - Status ClientProxy::GetConfig(const std::string& node_name, std::string& value) const { try { @@ -599,12 +179,373 @@ ClientProxy::SetConfig(const std::string& node_name, const std::string& value) c } Status -ClientProxy::FlushTable(const std::string& table_name) { +ClientProxy::CreateCollection(const CollectionParam& param) { + try { + ::milvus::grpc::TableSchema schema; + schema.set_table_name(param.collection_name); + schema.set_dimension(param.dimension); + schema.set_index_file_size(param.index_file_size); + schema.set_metric_type(static_cast(param.metric_type)); + + return client_ptr_->CreateTable(schema); + } catch (std::exception& ex) { + return Status(StatusCode::UnknownError, "Failed to create collection: " + std::string(ex.what())); + } +} + +bool +ClientProxy::HasCollection(const std::string& collection_name) { + Status status = Status::OK(); + ::milvus::grpc::TableName grpc_collection_name; + grpc_collection_name.set_table_name(collection_name); + bool result = client_ptr_->HasTable(grpc_collection_name, status); + return result; +} + +Status +ClientProxy::DropCollection(const std::string& collection_name) { + try { + ::milvus::grpc::TableName grpc_collection_name; + grpc_collection_name.set_table_name(collection_name); + return client_ptr_->DropTable(grpc_collection_name); + } catch (std::exception& ex) { + return Status(StatusCode::UnknownError, "Failed to drop collection: " + std::string(ex.what())); + } +} + +Status +ClientProxy::CreateIndex(const IndexParam& index_param) { + try { + ::milvus::grpc::IndexParam grpc_index_param; + grpc_index_param.set_table_name(index_param.collection_name); + grpc_index_param.set_index_type(static_cast(index_param.index_type)); + milvus::grpc::KeyValuePair* kv = grpc_index_param.add_extra_params(); + kv->set_key(EXTRA_PARAM_KEY); + kv->set_value(index_param.extra_params); + return client_ptr_->CreateIndex(grpc_index_param); + } catch (std::exception& ex) { + return Status(StatusCode::UnknownError, "Failed to build index: " + std::string(ex.what())); + } +} + +Status +ClientProxy::Insert(const std::string& collection_name, const std::string& partition_tag, + const std::vector& entity_array, std::vector& id_array) { + Status status = Status::OK(); + try { + ::milvus::grpc::InsertParam insert_param; + insert_param.set_table_name(collection_name); + insert_param.set_partition_tag(partition_tag); + + for (auto& entity : entity_array) { + ::milvus::grpc::RowRecord* grpc_record = insert_param.add_row_record_array(); + CopyRowRecord(grpc_record, entity); + } + + // Single thread + ::milvus::grpc::VectorIds vector_ids; + if (!id_array.empty()) { + /* set user's ids */ + auto row_ids = insert_param.mutable_row_id_array(); + row_ids->Resize(static_cast(id_array.size()), -1); + memcpy(row_ids->mutable_data(), id_array.data(), id_array.size() * sizeof(int64_t)); + status = client_ptr_->Insert(insert_param, vector_ids); + } else { + status = client_ptr_->Insert(insert_param, vector_ids); + /* return Milvus generated ids back to user */ + id_array.insert(id_array.end(), vector_ids.vector_id_array().begin(), vector_ids.vector_id_array().end()); + } + } catch (std::exception& ex) { + return Status(StatusCode::UnknownError, "Failed to add entities: " + std::string(ex.what())); + } + + return status; +} + +Status +ClientProxy::GetEntityByID(const std::string& collection_name, int64_t entity_id, Entity& entity_data) { + try { + ::milvus::grpc::VectorIdentity vector_identity; + vector_identity.set_table_name(collection_name); + vector_identity.set_id(entity_id); + + ::milvus::grpc::VectorData grpc_data; + Status status = client_ptr_->GetVectorByID(vector_identity, grpc_data); + if (!status.ok()) { + return status; + } + + int float_size = grpc_data.vector_data().float_data_size(); + if (float_size > 0) { + entity_data.float_data.resize(float_size); + memcpy(entity_data.float_data.data(), grpc_data.vector_data().float_data().data(), + float_size * sizeof(float)); + } + + auto byte_size = grpc_data.vector_data().binary_data().length(); + if (byte_size > 0) { + entity_data.binary_data.resize(byte_size); + memcpy(entity_data.binary_data.data(), grpc_data.vector_data().binary_data().data(), byte_size); + } + + return status; + } catch (std::exception& ex) { + return Status(StatusCode::UnknownError, "Failed to get entity by id: " + std::string(ex.what())); + } +} + +Status +ClientProxy::GetIDsInSegment(const std::string& collection_name, const std::string& segment_name, + std::vector& id_array) { + try { + ::milvus::grpc::GetVectorIDsParam param; + param.set_table_name(collection_name); + param.set_segment_name(segment_name); + + ::milvus::grpc::VectorIds vector_ids; + Status status = client_ptr_->GetIDsInSegment(param, vector_ids); + if (!status.ok()) { + return status; + } + + id_array.insert(id_array.end(), vector_ids.vector_id_array().begin(), vector_ids.vector_id_array().end()); + + return status; + } catch (std::exception& ex) { + return Status(StatusCode::UnknownError, "Failed to get ids from segment: " + std::string(ex.what())); + } +} + +Status +ClientProxy::Search(const std::string& collection_name, const std::vector& partition_tag_array, + const std::vector& entity_array, int64_t topk, const std::string& extra_params, + TopKQueryResult& topk_query_result) { + try { + // step 1: convert vectors data + ::milvus::grpc::SearchParam search_param; + ConstructSearchParam(collection_name, + partition_tag_array, + topk, + extra_params, + search_param); + + for (auto& entity : entity_array) { + ::milvus::grpc::RowRecord* row_record = search_param.add_query_record_array(); + CopyRowRecord(row_record, entity); + } + + // step 2: search vectors + ::milvus::grpc::TopKQueryResult result; + Status status = client_ptr_->Search(search_param, result); + if (result.row_num() == 0) { + return status; + } + + // step 3: convert result array + topk_query_result.reserve(result.row_num()); + int64_t nq = result.row_num(); + int64_t topk = result.ids().size() / nq; + for (int64_t i = 0; i < result.row_num(); i++) { + milvus::QueryResult one_result; + one_result.ids.resize(topk); + one_result.distances.resize(topk); + memcpy(one_result.ids.data(), result.ids().data() + topk * i, topk * sizeof(int64_t)); + memcpy(one_result.distances.data(), result.distances().data() + topk * i, topk * sizeof(float)); + topk_query_result.emplace_back(one_result); + } + + return status; + } catch (std::exception& ex) { + return Status(StatusCode::UnknownError, "Failed to search entities: " + std::string(ex.what())); + } +} + +Status +ClientProxy::DescribeCollection(const std::string& collection_name, CollectionParam& collection_param) { + try { + ::milvus::grpc::TableSchema grpc_schema; + + Status status = client_ptr_->DescribeTable(collection_name, grpc_schema); + + collection_param.collection_name = grpc_schema.table_name(); + collection_param.dimension = grpc_schema.dimension(); + collection_param.index_file_size = grpc_schema.index_file_size(); + collection_param.metric_type = static_cast(grpc_schema.metric_type()); + + return status; + } catch (std::exception& ex) { + return Status(StatusCode::UnknownError, "Failed to describe collection: " + std::string(ex.what())); + } +} + +Status +ClientProxy::CountCollection(const std::string& collection_name, int64_t& row_count) { + try { + Status status; + ::milvus::grpc::TableName grpc_collection_name; + grpc_collection_name.set_table_name(collection_name); + row_count = client_ptr_->CountTable(grpc_collection_name, status); + return status; + } catch (std::exception& ex) { + return Status(StatusCode::UnknownError, "Failed to count collection: " + std::string(ex.what())); + } +} + +Status +ClientProxy::ShowCollections(std::vector& collection_array) { + try { + Status status; + milvus::grpc::TableNameList collection_name_list; + status = client_ptr_->ShowTables(collection_name_list); + + collection_array.resize(collection_name_list.table_names_size()); + for (uint64_t i = 0; i < collection_name_list.table_names_size(); ++i) { + collection_array[i] = collection_name_list.table_names(i); + } + return status; + } catch (std::exception& ex) { + return Status(StatusCode::UnknownError, "Failed to show collections: " + std::string(ex.what())); + } +} + +Status +ClientProxy::ShowCollectionInfo(const std::string& collection_name, CollectionInfo& collection_info) { + try { + Status status; + ::milvus::grpc::TableName grpc_collection_name; + grpc_collection_name.set_table_name(collection_name); + milvus::grpc::TableInfo grpc_collection_info; + status = client_ptr_->ShowTableInfo(grpc_collection_name, grpc_collection_info); + + // get native info + collection_info.total_row_count = grpc_collection_info.total_row_count(); + + // get partitions info + for (int i = 0; i < grpc_collection_info.partitions_stat_size(); i++) { + auto& grpc_partition_stat = grpc_collection_info.partitions_stat(i); + PartitionStat partition_stat; + ConstructPartitionStat(grpc_partition_stat, partition_stat); + collection_info.partitions_stat.emplace_back(partition_stat); + } + + return status; + } catch (std::exception& ex) { + return Status(StatusCode::UnknownError, "Failed to show collection info: " + std::string(ex.what())); + } +} + +Status +ClientProxy::DeleteByID(const std::string& collection_name, const std::vector& id_array) { + try { + ::milvus::grpc::DeleteByIDParam delete_by_id_param; + delete_by_id_param.set_table_name(collection_name); + for (auto id : id_array) { + delete_by_id_param.add_id_array(id); + } + + return client_ptr_->DeleteByID(delete_by_id_param); + } catch (std::exception& ex) { + return Status(StatusCode::UnknownError, "Failed to delete entity id: " + std::string(ex.what())); + } +} + +Status +ClientProxy::PreloadCollection(const std::string& collection_name) const { + try { + ::milvus::grpc::TableName grpc_collection_name; + grpc_collection_name.set_table_name(collection_name); + Status status = client_ptr_->PreloadTable(grpc_collection_name); + return status; + } catch (std::exception& ex) { + return Status(StatusCode::UnknownError, "Failed to preload collection: " + std::string(ex.what())); + } +} + +Status +ClientProxy::DescribeIndex(const std::string& collection_name, IndexParam& index_param) const { + try { + ::milvus::grpc::TableName grpc_collection_name; + grpc_collection_name.set_table_name(collection_name); + + ::milvus::grpc::IndexParam grpc_index_param; + Status status = client_ptr_->DescribeIndex(grpc_collection_name, grpc_index_param); + index_param.index_type = static_cast(grpc_index_param.index_type()); + + for (int i = 0; i < grpc_index_param.extra_params_size(); i++) { + const milvus::grpc::KeyValuePair& kv = grpc_index_param.extra_params(i); + if (kv.key() == EXTRA_PARAM_KEY) { + index_param.extra_params = kv.value(); + } + } + + return status; + } catch (std::exception& ex) { + return Status(StatusCode::UnknownError, "Failed to describe index: " + std::string(ex.what())); + } +} + +Status +ClientProxy::DropIndex(const std::string& collection_name) const { + try { + ::milvus::grpc::TableName grpc_collection_name; + grpc_collection_name.set_table_name(collection_name); + Status status = client_ptr_->DropIndex(grpc_collection_name); + return status; + } catch (std::exception& ex) { + return Status(StatusCode::UnknownError, "Failed to drop index: " + std::string(ex.what())); + } +} + +Status +ClientProxy::CreatePartition(const PartitionParam& partition_param) { + try { + ::milvus::grpc::PartitionParam grpc_partition_param; + grpc_partition_param.set_table_name(partition_param.collection_name); + grpc_partition_param.set_tag(partition_param.partition_tag); + Status status = client_ptr_->CreatePartition(grpc_partition_param); + return status; + } catch (std::exception& ex) { + return Status(StatusCode::UnknownError, "Failed to create partition: " + std::string(ex.what())); + } +} + +Status +ClientProxy::ShowPartitions(const std::string& collection_name, PartitionTagList& partition_tag_array) const { + try { + ::milvus::grpc::TableName grpc_collection_name; + grpc_collection_name.set_table_name(collection_name); + ::milvus::grpc::PartitionList grpc_partition_list; + Status status = client_ptr_->ShowPartitions(grpc_collection_name, grpc_partition_list); + partition_tag_array.resize(grpc_partition_list.partition_tag_array_size()); + for (uint64_t i = 0; i < grpc_partition_list.partition_tag_array_size(); ++i) { + partition_tag_array[i] = grpc_partition_list.partition_tag_array(i); + } + return status; + } catch (std::exception& ex) { + return Status(StatusCode::UnknownError, "Failed to show partitions: " + std::string(ex.what())); + } +} + +Status +ClientProxy::DropPartition(const PartitionParam& partition_param) { + try { + ::milvus::grpc::PartitionParam grpc_partition_param; + grpc_partition_param.set_table_name(partition_param.collection_name); + grpc_partition_param.set_tag(partition_param.partition_tag); + Status status = client_ptr_->DropPartition(grpc_partition_param); + return status; + } catch (std::exception& ex) { + return Status(StatusCode::UnknownError, "Failed to drop partition: " + std::string(ex.what())); + } +} + +Status +ClientProxy::FlushCollection(const std::string& collection_name) { try { std::string dummy; - return client_ptr_->Flush(table_name); + return client_ptr_->Flush(collection_name); } catch (std::exception& ex) { - return Status(StatusCode::UnknownError, "Failed to flush"); + return Status(StatusCode::UnknownError, "Failed to flush collection"); } } @@ -614,19 +555,19 @@ ClientProxy::Flush() { std::string dummy; return client_ptr_->Flush(""); } catch (std::exception& ex) { - return Status(StatusCode::UnknownError, "Failed to flush"); + return Status(StatusCode::UnknownError, "Failed to flush collections"); } } Status -ClientProxy::CompactTable(const std::string& table_name) { +ClientProxy::CompactCollection(const std::string& collection_name) { try { - ::milvus::grpc::TableName grpc_table_name; - grpc_table_name.set_table_name(table_name); - Status status = client_ptr_->Compact(grpc_table_name); + ::milvus::grpc::TableName grpc_collection_name; + grpc_collection_name.set_table_name(collection_name); + Status status = client_ptr_->Compact(grpc_collection_name); return status; } catch (std::exception& ex) { - return Status(StatusCode::UnknownError, "Failed to compact table: " + std::string(ex.what())); + return Status(StatusCode::UnknownError, "Failed to compact collection: " + std::string(ex.what())); } } diff --git a/sdk/grpc/ClientProxy.h b/sdk/grpc/ClientProxy.h index 30bffedf0f..f395ce32ca 100644 --- a/sdk/grpc/ClientProxy.h +++ b/sdk/grpc/ClientProxy.h @@ -24,7 +24,7 @@ class ClientProxy : public Connection { public: // Implementations of the Connection interface Status - Connect(const ConnectParam& param) override; + Connect(const ConnectParam& connect_param) override; Status Connect(const std::string& uri) override; @@ -35,51 +35,6 @@ class ClientProxy : public Connection { Status Disconnect() override; - Status - CreateTable(const TableSchema& param) override; - - bool - HasTable(const std::string& table_name) override; - - Status - DropTable(const std::string& table_name) override; - - Status - CreateIndex(const IndexParam& index_param) override; - - Status - Insert(const std::string& table_name, const std::string& partition_tag, const std::vector& record_array, - std::vector& id_array) override; - - Status - GetVectorByID(const std::string& table_name, int64_t vector_id, RowRecord& vector_data) override; - - Status - GetIDsInSegment(const std::string& table_name, const std::string& segment_name, - std::vector& id_array) override; - - Status - Search(const std::string& table_name, const std::vector& partition_tag_array, - const std::vector& query_record_array, int64_t topk, const std::string& extra_params, - TopKQueryResult& topk_query_result) override; - - Status - SearchByID(const std::string& table_name, const std::vector& partition_tag_array, - int64_t query_id, int64_t topk, - const std::string& extra_params, TopKQueryResult& topk_query_result) override; - - Status - DescribeTable(const std::string& table_name, TableSchema& table_schema) override; - - Status - CountTable(const std::string& table_name, int64_t& row_count) override; - - Status - ShowTables(std::vector& table_array) override; - - Status - ShowTableInfo(const std::string& table_name, TableInfo& table_info) override; - std::string ClientVersion() const override; @@ -89,30 +44,6 @@ class ClientProxy : public Connection { std::string ServerStatus() const override; - std::string - DumpTaskTables() const override; - - Status - DeleteByID(const std::string& table_name, const std::vector& id_array) override; - - Status - PreloadTable(const std::string& table_name) const override; - - Status - DescribeIndex(const std::string& table_name, IndexParam& index_param) const override; - - Status - DropIndex(const std::string& table_name) const override; - - Status - CreatePartition(const PartitionParam& partition_param) override; - - Status - ShowPartitions(const std::string& table_name, PartitionTagList& partition_tag_array) const override; - - Status - DropPartition(const PartitionParam& partition_param) override; - Status GetConfig(const std::string& node_name, std::string& value) const override; @@ -120,13 +51,76 @@ class ClientProxy : public Connection { SetConfig(const std::string& node_name, const std::string& value) const override; Status - FlushTable(const std::string& table_name) override; + CreateCollection(const CollectionParam& param) override; + + bool + HasCollection(const std::string& collection_name) override; + + Status + DropCollection(const std::string& collection_name) override; + + Status + CreateIndex(const IndexParam& index_param) override; + + Status + Insert(const std::string& collection_name, + const std::string& partition_tag, + const std::vector& entity_array, + std::vector& id_array) override; + + Status + GetEntityByID(const std::string& collection_name, int64_t entity_id, Entity& entity_data) override; + + Status + GetIDsInSegment(const std::string& collection_name, const std::string& segment_name, + std::vector& id_array) override; + + Status + Search(const std::string& collection_name, const std::vector& partition_tag_array, + const std::vector& entity_array, int64_t topk, + const std::string& extra_params, TopKQueryResult& topk_query_result) override; + + Status + DescribeCollection(const std::string& collection_name, CollectionParam& collection_schema) override; + + Status + CountCollection(const std::string& collection_name, int64_t& entity_count) override; + + Status + ShowCollections(std::vector& collection_array) override; + + Status + ShowCollectionInfo(const std::string& collection_name, CollectionInfo& collection_info) override; + + Status + DeleteByID(const std::string& collection_name, const std::vector& id_array) override; + + Status + PreloadCollection(const std::string& collection_name) const override; + + Status + DescribeIndex(const std::string& collection_name, IndexParam& index_param) const override; + + Status + DropIndex(const std::string& collection_name) const override; + + Status + CreatePartition(const PartitionParam& partition_param) override; + + Status + ShowPartitions(const std::string& collection_name, PartitionTagList& partition_tag_array) const override; + + Status + DropPartition(const PartitionParam& partition_param) override; + + Status + FlushCollection(const std::string& collection_name) override; Status Flush() override; Status - CompactTable(const std::string& table_name) override; + CompactCollection(const std::string& collection_name) override; private: std::shared_ptr<::grpc::Channel> channel_; diff --git a/sdk/grpc/GrpcClient.cpp b/sdk/grpc/GrpcClient.cpp index 47c6c9e8ab..96773f9c75 100644 --- a/sdk/grpc/GrpcClient.cpp +++ b/sdk/grpc/GrpcClient.cpp @@ -178,26 +178,6 @@ GrpcClient::Search( return Status::OK(); } -Status -GrpcClient::SearchByID(const ::milvus::grpc::SearchByIDParam& search_param, - ::milvus::grpc::TopKQueryResult& topk_query_result) { - ::milvus::grpc::TopKQueryResult query_result; - ClientContext context; - ::grpc::Status grpc_status = stub_->SearchByID(&context, search_param, &topk_query_result); - - if (!grpc_status.ok()) { - std::cerr << "SearchByID rpc failed!" << std::endl; - std::cerr << grpc_status.error_message() << std::endl; - return Status(StatusCode::RPCFailed, grpc_status.error_message()); - } - if (topk_query_result.status().error_code() != grpc::SUCCESS) { - std::cerr << topk_query_result.status().reason() << std::endl; - return Status(StatusCode::ServerFailed, topk_query_result.status().reason()); - } - - return Status::OK(); -} - Status GrpcClient::DescribeTable(const std::string& table_name, ::milvus::grpc::TableSchema& grpc_schema) { ClientContext context; diff --git a/sdk/grpc/GrpcClient.h b/sdk/grpc/GrpcClient.h index 51af08a2c7..d7b7ae42c5 100644 --- a/sdk/grpc/GrpcClient.h +++ b/sdk/grpc/GrpcClient.h @@ -59,9 +59,6 @@ class GrpcClient { Status Search(const grpc::SearchParam& search_param, ::milvus::grpc::TopKQueryResult& topk_query_result); - Status - SearchByID(const grpc::SearchByIDParam& search_param, ::milvus::grpc::TopKQueryResult& topk_query_result); - Status DescribeTable(const std::string& table_name, grpc::TableSchema& grpc_schema); diff --git a/sdk/include/MilvusApi.h b/sdk/include/MilvusApi.h index 8196bc676f..50f012ff62 100644 --- a/sdk/include/MilvusApi.h +++ b/sdk/include/MilvusApi.h @@ -54,20 +54,19 @@ struct ConnectParam { }; /** - * @brief Table Schema + * @brief Collection parameters */ -struct TableSchema { - std::string table_name; ///< Table name +struct CollectionParam { + std::string collection_name; ///< Collection_name name int64_t dimension = 0; ///< Vector dimension, must be a positive value int64_t index_file_size = 1024; ///< Index file size, must be a positive value, unit: MB MetricType metric_type = MetricType::L2; ///< Index metric type }; - /** - * @brief Record inserted + * @brief Entity inserted, currently each entity represent a vector */ -struct RowRecord { +struct Entity { std::vector float_data; ///< Vector raw float data std::vector binary_data; ///< Vector raw binary data }; @@ -76,7 +75,7 @@ struct RowRecord { * @brief TopK query result */ struct QueryResult { - std::vector ids; ///< Query ids result + std::vector ids; ///< Query entity ids result std::vector distances; ///< Query distances result }; using TopKQueryResult = std::vector; ///< Topk query result @@ -85,13 +84,22 @@ using TopKQueryResult = std::vector; ///< Topk query result * @brief Index parameters * Note: extra_params is extra parameters list, it must be json format * For different index type, parameter list is different accordingly, for example: - * FLAT/IVFLAT/SQ8: "{nlist: '16384'}" - * IVFPQ: "{nlist: '16384', nbits: "16"}" - * NSG: "{search_length: '100', out_degree:'40', pool_size:'66'}" - * HNSW "{M: '16', ef_construct:'500'}" + * FLAT/IVFLAT/SQ8: {nlist: 16384} + * ///< nlist range:[1, 999999] + * IVFPQ: {nlist: 16384, m: 12} + * ///< nlist range:[1, 999999] + * ///< m is decided by dim and have a couple of results. + * NSG: {search_length: 45, out_degree:50, candidate_pool_size:300, knng:100} + * ///< search_length range:[10, 300] + * ///< out_degree range:[5, 300] + * ///< candidate_pool_size range:[50, 1000] + * ///< knng range:[5, 300] + * HNSW {M: 16, efConstruction:300} + * ///< M range:[5, 48] + * ///< efConstruction range:[100, 500] */ struct IndexParam { - std::string table_name; ///< Table name for create index + std::string collection_name; ///< Collection name for create index IndexType index_type; ///< Index type std::string extra_params; ///< Extra parameters according to different index type, must be json format }; @@ -100,7 +108,7 @@ struct IndexParam { * @brief partition parameters */ struct PartitionParam { - std::string table_name; + std::string collection_name; std::string partition_tag; }; @@ -126,11 +134,11 @@ struct PartitionStat { }; /** - * @brief table info + * @brief collection info */ -struct TableInfo { - int64_t total_row_count; ///< Table total row count - std::vector partitions_stat; ///< Table's partitions statistics +struct CollectionInfo { + int64_t total_row_count; ///< Collection total entity count + std::vector partitions_stat; ///< Collection's partitions statistics }; /** @@ -139,18 +147,18 @@ struct TableInfo { class Connection { public: /** - * @brief CreateConnection + * @brief Create connection * * Create a connection instance and return it's shared pointer * - * @return Connection instance pointer + * @return connection instance pointer */ static std::shared_ptr Create(); /** - * @brief DestroyConnection + * @brief Destroy connection * * Destroy the connection instance * @@ -174,7 +182,7 @@ class Connection { */ virtual Status - Connect(const ConnectParam& param) = 0; + Connect(const ConnectParam& connect_param) = 0; /** * @brief Connect @@ -190,7 +198,7 @@ class Connection { Connect(const std::string& uri) = 0; /** - * @brief connected + * @brief Connected * * This method is used to test whether server is connected. * @@ -210,201 +218,7 @@ class Connection { Disconnect() = 0; /** - * @brief Create table method - * - * This method is used to create table. - * - * @param param, use to provide table information to be created. - * - * @return Indicate if table is created successfully - */ - virtual Status - CreateTable(const TableSchema& param) = 0; - - /** - * @brief Test table existence method - * - * This method is used to create table. - * - * @param table_name, target table's name. - * - * @return Indicate if table is cexist - */ - virtual bool - HasTable(const std::string& table_name) = 0; - - /** - * @brief Drop table method - * - * This method is used to drop table(and its partitions). - * - * @param table_name, target table's name. - * - * @return Indicate if table is drop successfully. - */ - virtual Status - DropTable(const std::string& table_name) = 0; - - /** - * @brief Create index method - * - * This method is used to create index for whole table(and its partitions). - * - * @param IndexParam - * table_name, table name is going to be create index. - * index type, - * nlist, - * index file size - * - * @return Indicate if build index successfully. - */ - virtual Status - CreateIndex(const IndexParam& index_param) = 0; - - /** - * @brief Insert vector to table - * - * This method is used to insert vector array to table. - * - * @param table_name, target table's name. - * @param partition_tag, target partition's tag, keep empty if no partition specified. - * @param record_array, vector array is inserted. - * @param id_array, - * specify id for each vector, - * if this array is empty, milvus will generate unique id for each vector, - * and return all ids by this parameter. - * - * @return Indicate if vector array are inserted successfully - */ - virtual Status - Insert(const std::string& table_name, const std::string& partition_tag, const std::vector& record_array, - std::vector& id_array) = 0; - - /** - * @brief Get vector data by id - * - * This method is used to get vector data by id from a table. - * Return the first found vector if there are vectors with duplicated id - * - * @param table_name, target table's name. - * @param vector_id, target vector id. - * @param vector_data, returned vector data. - * - * @return Indicate if the operation is succeed. - */ - virtual Status - GetVectorByID(const std::string& table_name, int64_t vector_id, RowRecord& vector_data) = 0; - - /** - * @brief Get vector ids from a segment - * - * This method is used to get vector ids from a segment - * Return all vector(not deleted) ids - * - * @param table_name, target table's name. - * @param segment_name, target segment name. - * @param id_array, returned vector id array. - * - * @return Indicate if the operation is succeed. - */ - virtual Status - GetIDsInSegment(const std::string& table_name, const std::string& segment_name, std::vector& id_array) = 0; - - /** - * @brief Search vector - * - * This method is used to query vector in table. - * - * @param table_name, target table's name. - * @param partition_tag_array, target partitions, keep empty if no partition specified. - * @param query_record_array, vectors to be queried. - * @param topk, how many similarity vectors will be returned. - * @param extra_params, extra search parameters according to different index type, must be json format. - * Note: extra_params is extra parameters list, it must be json format, for example: - * For different index type, parameter list is different accordingly - * FLAT/IVFLAT/SQ8/IVFPQ: "{nprobe: '32'}" - * NSG: "{search_length:'100'} - * HNSW "{ef: '64'} - * @param topk_query_result, result array. - * - * @return Indicate if query is successful. - */ - virtual Status - Search(const std::string& table_name, const PartitionTagList& partition_tag_array, - const std::vector& query_record_array, int64_t topk, - const std::string& extra_params, TopKQueryResult& topk_query_result) = 0; - - /** - * @brief Search vector by ID - * - * This method is used to query vector in table. - * - * @param table_name, target table's name. - * @param partition_tag_array, target partitions, keep empty if no partition. - * @param query_id, vector id to be queried. - * @param topk, how many similarity vectors will be returned. - * @param extra_params, extra search parameters according to different index type, must be json format. - * @param topk_query_result, result array. - * - * @return Indicate if query is successful. - */ - virtual Status - SearchByID(const std::string& table_name, const PartitionTagList& partition_tag_array, int64_t query_id, - int64_t topk, const std::string& extra_params, TopKQueryResult& topk_query_result) = 0; - - /** - * @brief Show table description - * - * This method is used to show table information. - * - * @param table_name, target table's name. - * @param table_schema, table_schema is given when operation is successful. - * - * @return Indicate if this operation is successful. - */ - virtual Status - DescribeTable(const std::string& table_name, TableSchema& table_schema) = 0; - - /** - * @brief Get table row count - * - * This method is used to get table row count. - * - * @param table_name, target table's name. - * @param row_count, table total row count(including partitions). - * - * @return Indicate if this operation is successful. - */ - virtual Status - CountTable(const std::string& table_name, int64_t& row_count) = 0; - - /** - * @brief Show all tables in database - * - * This method is used to list all tables. - * - * @param table_array, all tables in database. - * - * @return Indicate if this operation is successful. - */ - virtual Status - ShowTables(std::vector& table_array) = 0; - - /** - * @brief Show table information - * - * This method is used to get detail information of a table. - * - * @param table_name, target table's name. - * @param table_info, target table's information - * - * @return Indicate if this operation is successful. - */ - virtual Status - ShowTableInfo(const std::string& table_name, TableInfo& table_info) = 0; - - /** - * @brief Give the client version + * @brief Get the client version * * This method is used to give the client version. * @@ -414,7 +228,7 @@ class Connection { ClientVersion() const = 0; /** - * @brief Give the server version + * @brief Get the server version * * This method is used to give the server version. * @@ -424,7 +238,7 @@ class Connection { ServerVersion() const = 0; /** - * @brief Give the server status + * @brief Get the server status * * This method is used to give the server status. * @@ -433,106 +247,6 @@ class Connection { virtual std::string ServerStatus() const = 0; - /** - * @brief dump server tasks information - * - * This method is internal used. - * - * @return Task information in tasktables. - */ - virtual std::string - DumpTaskTables() const = 0; - - /** - * [deprecated] - * @brief delete tables by vector id - * - * This method is used to delete table data by date range. - * - * @param table_name, target table's name. - * @param id_array, vector ids to deleted. - * - * @return Indicate if this operation is successful. - */ - virtual Status - DeleteByID(const std::string& table_name, const std::vector& id_array) = 0; - - /** - * @brief preload table - * - * This method is used to preload table - * - * @param table_name - * - * @return Indicate if this operation is successful. - */ - virtual Status - PreloadTable(const std::string& table_name) const = 0; - - /** - * @brief describe index - * - * This method is used to describe index - * - * @param table_name, target table's name. - * @param index_param, returned index information. - * - * @return Indicate if this operation is successful. - */ - virtual Status - DescribeIndex(const std::string& table_name, IndexParam& index_param) const = 0; - - /** - * @brief drop index - * - * This method is used to drop index of table(and its partitions) - * - * @param table_name, target table's name. - * - * @return Indicate if this operation is successful. - */ - virtual Status - DropIndex(const std::string& table_name) const = 0; - - /** - * @brief Create partition method - * - * This method is used to create table partition - * - * @param param, use to provide partition information to be created. - * - * @return Indicate if partition is created successfully - */ - virtual Status - CreatePartition(const PartitionParam& param) = 0; - - /** - * @brief Test table existence method - * - * This method is used to create table - * - * @param table_name, table name is going to be tested. - * @param partition_tag_array, partition tag array of the table. - * - * @return Indicate if this operation is successful - */ - virtual Status - ShowPartitions(const std::string& table_name, PartitionTagList& partition_tag_array) const = 0; - - /** - * @brief Delete partition method - * - * This method is used to delete table partition. - * - * @param param, target partition to be deleted. - * NOTE: if param.table_name is empty, you must specify param.partition_name, - * else you can specify param.table_name and param.tag and let the param.partition_name be empty - * - * @return Indicate if partition is delete successfully. - */ - virtual Status - DropPartition(const PartitionParam& param) = 0; - /** * @brief Get config method * @@ -560,21 +274,287 @@ class Connection { SetConfig(const std::string& node_name, const std::string& value) const = 0; /** - * @brief flush table buffer into storage + * @brief Create collection method * - * This method is used to flush table buffer into storage + * This method is used to create collection. * - * @param table_name, target table's name. + * @param param, use to provide collection information to be created. + * + * @return Indicate if collection is created successfully + */ + virtual Status + CreateCollection(const CollectionParam& param) = 0; + + /** + * @brief Test collection existence method + * + * This method is used to test collection existence. + * + * @param collection_name, target collection's name. + * + * @return Indicate if collection is cexist + */ + virtual bool + HasCollection(const std::string& collection_name) = 0; + + /** + * @brief Drop collection method + * + * This method is used to drop collection(and its partitions). + * + * @param collection_name, target collection's name. + * + * @return Indicate if collection is drop successfully. + */ + virtual Status + DropCollection(const std::string& collection_name) = 0; + + /** + * @brief Create index method + * + * This method is used to create index for whole collection(and its partitions). + * + * @param index_param, use to provide index information to be created. + * + * @return Indicate if create index successfully. + */ + virtual Status + CreateIndex(const IndexParam& index_param) = 0; + + /** + * @brief Insert entity to collection + * + * This method is used to insert vector array to collection. + * + * @param collection_name, target collection's name. + * @param partition_tag, target partition's tag, keep empty if no partition specified. + * @param entity_array, entity array is inserted, each entitu represent a vector. + * @param id_array, + * specify id for each entity, + * if this array is empty, milvus will generate unique id for each entity, + * and return all ids by this parameter. + * + * @return Indicate if entity array are inserted successfully + */ + virtual Status + Insert(const std::string& collection_name, + const std::string& partition_tag, + const std::vector& entity_array, + std::vector& id_array) = 0; + + /** + * @brief Get entity data by id + * + * This method is used to get entity data by id from a collection. + * Return the first found entity if there are entities with duplicated id + * + * @param collection_name, target collection's name. + * @param entity_id, target entity id. + * @param entity_data, returned entity data. + * + * @return Indicate if the operation is succeed. + */ + virtual Status + GetEntityByID(const std::string& collection_name, int64_t entity_id, Entity& entity_data) = 0; + + /** + * @brief Get entity ids from a segment + * + * This method is used to get entity ids from a segment + * Return all entity(not deleted) ids + * + * @param collection_name, target collection's name. + * @param segment_name, target segment name. + * @param id_array, returned entity id array. + * + * @return Indicate if the operation is succeed. + */ + virtual Status + GetIDsInSegment(const std::string& collection_name, + const std::string& segment_name, + std::vector& id_array) = 0; + + /** + * @brief Search entities in a collection + * + * This method is used to query entity in collection. + * + * @param collection_name, target collection's name. + * @param partition_tag_array, target partitions, keep empty if no partition specified. + * @param query_entity_array, vectors to be queried. + * @param topk, how many similarity entities will be returned. + * @param extra_params, extra search parameters according to different index type, must be json format. + * Note: extra_params is extra parameters list, it must be json format, for example: + * For different index type, parameter list is different accordingly + * FLAT/IVFLAT/SQ8/IVFPQ: {nprobe: 32} + * ///< nprobe range:[1,999999] + * NSG: {search_length:100} + * ///< search_length range:[10, 300] + * HNSW {ef: 64} + * ///< ef range:[topk, 4096] + * @param topk_query_result, result array. + * + * @return Indicate if query is successful. + */ + virtual Status + Search(const std::string& collection_name, const PartitionTagList& partition_tag_array, + const std::vector& entity_array, int64_t topk, + const std::string& extra_params, TopKQueryResult& topk_query_result) = 0; + + /** + * @brief Show collection description + * + * This method is used to show collection information. + * + * @param collection_name, target collection's name. + * @param collection_param, collection_param is given when operation is successful. * * @return Indicate if this operation is successful. */ virtual Status - FlushTable(const std::string& table_name) = 0; + DescribeCollection(const std::string& collection_name, CollectionParam& collection_param) = 0; /** - * @brief flush all buffer into storage + * @brief Get collection entity count * - * This method is used to all table buffer into storage + * This method is used to get collection entity count. + * + * @param collection_name, target collection's name. + * @param entity_count, collection total entity count(including partitions). + * + * @return Indicate if this operation is successful. + */ + virtual Status + CountCollection(const std::string& collection_name, int64_t& entity_count) = 0; + + /** + * @brief Show all collections in database + * + * This method is used to list all collections. + * + * @param collection_array, all collections in database. + * + * @return Indicate if this operation is successful. + */ + virtual Status + ShowCollections(std::vector& collection_array) = 0; + + /** + * @brief Show collection information + * + * This method is used to get detail information of a collection. + * + * @param collection_name, target collection's name. + * @param collection_info, target collection's information + * + * @return Indicate if this operation is successful. + */ + virtual Status + ShowCollectionInfo(const std::string& collection_name, CollectionInfo& collection_info) = 0; + + /** + * @brief Delete entity by id + * + * This method is used to delete entity by id. + * + * @param collection_name, target collection's name. + * @param id_array, entity id array to be deleted. + * + * @return Indicate if this operation is successful. + */ + virtual Status + DeleteByID(const std::string& collection_name, const std::vector& id_array) = 0; + + /** + * @brief Preload collection + * + * This method is used to preload collection data into memory + * + * @param collection_name, target collection's name. + * + * @return Indicate if this operation is successful. + */ + virtual Status + PreloadCollection(const std::string& collection_name) const = 0; + + /** + * @brief Describe index + * + * This method is used to describe index + * + * @param collection_name, target collection's name. + * @param index_param, returned index information. + * + * @return Indicate if this operation is successful. + */ + virtual Status + DescribeIndex(const std::string& collection_name, IndexParam& index_param) const = 0; + + /** + * @brief Drop index + * + * This method is used to drop index of collection(and its partitions) + * + * @param collection_name, target collection's name. + * + * @return Indicate if this operation is successful. + */ + virtual Status + DropIndex(const std::string& collection_name) const = 0; + + /** + * @brief Create partition method + * + * This method is used to create collection's partition + * + * @param partition_param, use to provide partition information to be created. + * + * @return Indicate if partition is created successfully + */ + virtual Status + CreatePartition(const PartitionParam& partition_param) = 0; + + /** + * @brief Show all partitions method + * + * This method is used to show all partitions(return their tags) + * + * @param collection_name, target collection's name. + * @param partition_tag_array, partition tag array of the collection. + * + * @return Indicate if this operation is successful + */ + virtual Status + ShowPartitions(const std::string& collection_name, PartitionTagList& partition_tag_array) const = 0; + + /** + * @brief Delete partition method + * + * This method is used to delete collection's partition. + * + * @param partition_param, target partition to be deleted. + * + * @return Indicate if partition is delete successfully. + */ + virtual Status + DropPartition(const PartitionParam& partition_param) = 0; + + /** + * @brief Flush collection buffer into storage + * + * This method is used to flush collection buffer into storage + * + * @param collection_name, target collection's name. + * + * @return Indicate if this operation is successful. + */ + virtual Status + FlushCollection(const std::string& collection_name) = 0; + + /** + * @brief Flush all buffer into storage + * + * This method is used to all collection buffer into storage * * @return Indicate if this operation is successful. */ @@ -582,16 +562,16 @@ class Connection { Flush() = 0; /** - * @brief compact table, remove deleted vectors + * @brief Compact collection, permanently remove deleted vectors * - * This method is used to compact table + * This method is used to compact collection * - * @param table_name, target table's name. + * @param collection_name, target collection's name. * * @return Indicate if this operation is successful. */ virtual Status - CompactTable(const std::string& table_name) = 0; + CompactCollection(const std::string& collection_name) = 0; }; } // namespace milvus diff --git a/sdk/interface/ConnectionImpl.cpp b/sdk/interface/ConnectionImpl.cpp index 54c3aded92..2a51a11d50 100644 --- a/sdk/interface/ConnectionImpl.cpp +++ b/sdk/interface/ConnectionImpl.cpp @@ -56,80 +56,6 @@ ConnectionImpl::ClientVersion() const { return client_proxy_->ClientVersion(); } -Status -ConnectionImpl::CreateTable(const TableSchema& param) { - return client_proxy_->CreateTable(param); -} - -bool -ConnectionImpl::HasTable(const std::string& table_name) { - return client_proxy_->HasTable(table_name); -} - -Status -ConnectionImpl::DropTable(const std::string& table_name) { - return client_proxy_->DropTable(table_name); -} - -Status -ConnectionImpl::CreateIndex(const IndexParam& index_param) { - return client_proxy_->CreateIndex(index_param); -} - -Status -ConnectionImpl::Insert(const std::string& table_name, const std::string& partition_tag, - const std::vector& record_array, std::vector& id_array) { - return client_proxy_->Insert(table_name, partition_tag, record_array, id_array); -} - -Status -ConnectionImpl::GetVectorByID(const std::string& table_name, int64_t vector_id, RowRecord& vector_data) { - return client_proxy_->GetVectorByID(table_name, vector_id, vector_data); -} - -Status -ConnectionImpl::GetIDsInSegment(const std::string& table_name, const std::string& segment_name, - std::vector& id_array) { - return client_proxy_->GetIDsInSegment(table_name, segment_name, id_array); -} - -Status -ConnectionImpl::Search(const std::string& table_name, const std::vector& partition_tags, - const std::vector& query_record_array, int64_t topk, const std::string& extra_params, - TopKQueryResult& topk_query_result) { - return client_proxy_->Search(table_name, partition_tags, query_record_array, topk, extra_params, topk_query_result); -} - -Status -ConnectionImpl::SearchByID(const std::string& table_name, - const std::vector& partition_tags, - int64_t query_id, - int64_t topk, - const std::string& extra_params, - TopKQueryResult& topk_query_result) { - return client_proxy_->SearchByID(table_name, partition_tags, query_id, topk, extra_params, topk_query_result); -} - -Status -ConnectionImpl::DescribeTable(const std::string& table_name, TableSchema& table_schema) { - return client_proxy_->DescribeTable(table_name, table_schema); -} - -Status -ConnectionImpl::CountTable(const std::string& table_name, int64_t& row_count) { - return client_proxy_->CountTable(table_name, row_count); -} - -Status -ConnectionImpl::ShowTables(std::vector& table_array) { - return client_proxy_->ShowTables(table_array); -} - -Status -ConnectionImpl::ShowTableInfo(const std::string& table_name, TableInfo& table_info) { - return client_proxy_->ShowTableInfo(table_name, table_info); -} - std::string ConnectionImpl::ServerVersion() const { return client_proxy_->ServerVersion(); @@ -140,46 +66,6 @@ ConnectionImpl::ServerStatus() const { return client_proxy_->ServerStatus(); } -std::string -ConnectionImpl::DumpTaskTables() const { - return client_proxy_->DumpTaskTables(); -} - -Status -ConnectionImpl::DeleteByID(const std::string& table_name, const std::vector& id_array) { - return client_proxy_->DeleteByID(table_name, id_array); -} - -Status -ConnectionImpl::PreloadTable(const std::string& table_name) const { - return client_proxy_->PreloadTable(table_name); -} - -Status -ConnectionImpl::DescribeIndex(const std::string& table_name, IndexParam& index_param) const { - return client_proxy_->DescribeIndex(table_name, index_param); -} - -Status -ConnectionImpl::DropIndex(const std::string& table_name) const { - return client_proxy_->DropIndex(table_name); -} - -Status -ConnectionImpl::CreatePartition(const PartitionParam& param) { - return client_proxy_->CreatePartition(param); -} - -Status -ConnectionImpl::ShowPartitions(const std::string& table_name, PartitionTagList& partition_array) const { - return client_proxy_->ShowPartitions(table_name, partition_array); -} - -Status -ConnectionImpl::DropPartition(const PartitionParam& param) { - return client_proxy_->DropPartition(param); -} - Status ConnectionImpl::GetConfig(const std::string& node_name, std::string& value) const { return client_proxy_->GetConfig(node_name, value); @@ -191,8 +77,107 @@ ConnectionImpl::SetConfig(const std::string& node_name, const std::string& value } Status -ConnectionImpl::FlushTable(const std::string& Status) { - return client_proxy_->FlushTable(Status); +ConnectionImpl::CreateCollection(const CollectionParam& param) { + return client_proxy_->CreateCollection(param); +} + +bool +ConnectionImpl::HasCollection(const std::string& collection_name) { + return client_proxy_->HasCollection(collection_name); +} + +Status +ConnectionImpl::DropCollection(const std::string& collection_name) { + return client_proxy_->DropCollection(collection_name); +} + +Status +ConnectionImpl::CreateIndex(const IndexParam& index_param) { + return client_proxy_->CreateIndex(index_param); +} + +Status +ConnectionImpl::Insert(const std::string& collection_name, const std::string& partition_tag, + const std::vector& entity_array, std::vector& id_array) { + return client_proxy_->Insert(collection_name, partition_tag, entity_array, id_array); +} + +Status +ConnectionImpl::GetEntityByID(const std::string& collection_name, int64_t entity_id, Entity& entity_data) { + return client_proxy_->GetEntityByID(collection_name, entity_id, entity_data); +} + +Status +ConnectionImpl::GetIDsInSegment(const std::string& collection_name, const std::string& segment_name, + std::vector& id_array) { + return client_proxy_->GetIDsInSegment(collection_name, segment_name, id_array); +} + +Status +ConnectionImpl::Search(const std::string& collection_name, const std::vector& partition_tags, + const std::vector& entity_array, int64_t topk, const std::string& extra_params, + TopKQueryResult& topk_query_result) { + return client_proxy_->Search(collection_name, partition_tags, entity_array, topk, extra_params, topk_query_result); +} + +Status +ConnectionImpl::DescribeCollection(const std::string& collection_name, CollectionParam& collection_schema) { + return client_proxy_->DescribeCollection(collection_name, collection_schema); +} + +Status +ConnectionImpl::CountCollection(const std::string& collection_name, int64_t& row_count) { + return client_proxy_->CountCollection(collection_name, row_count); +} + +Status +ConnectionImpl::ShowCollections(std::vector& collection_array) { + return client_proxy_->ShowCollections(collection_array); +} + +Status +ConnectionImpl::ShowCollectionInfo(const std::string& collection_name, CollectionInfo& collection_info) { + return client_proxy_->ShowCollectionInfo(collection_name, collection_info); +} + +Status +ConnectionImpl::DeleteByID(const std::string& collection_name, const std::vector& id_array) { + return client_proxy_->DeleteByID(collection_name, id_array); +} + +Status +ConnectionImpl::PreloadCollection(const std::string& collection_name) const { + return client_proxy_->PreloadCollection(collection_name); +} + +Status +ConnectionImpl::DescribeIndex(const std::string& collection_name, IndexParam& index_param) const { + return client_proxy_->DescribeIndex(collection_name, index_param); +} + +Status +ConnectionImpl::DropIndex(const std::string& collection_name) const { + return client_proxy_->DropIndex(collection_name); +} + +Status +ConnectionImpl::CreatePartition(const PartitionParam& partition_param) { + return client_proxy_->CreatePartition(partition_param); +} + +Status +ConnectionImpl::ShowPartitions(const std::string& collection_name, PartitionTagList& partition_array) const { + return client_proxy_->ShowPartitions(collection_name, partition_array); +} + +Status +ConnectionImpl::DropPartition(const PartitionParam& partition_param) { + return client_proxy_->DropPartition(partition_param); +} + +Status +ConnectionImpl::FlushCollection(const std::string& Status) { + return client_proxy_->FlushCollection(Status); } Status @@ -201,8 +186,8 @@ ConnectionImpl::Flush() { } Status -ConnectionImpl::CompactTable(const std::string& table_name) { - return client_proxy_->CompactTable(table_name); +ConnectionImpl::CompactCollection(const std::string& collection_name) { + return client_proxy_->CompactCollection(collection_name); } } // namespace milvus diff --git a/sdk/interface/ConnectionImpl.h b/sdk/interface/ConnectionImpl.h index 0a89a7e9a7..15533b9de6 100644 --- a/sdk/interface/ConnectionImpl.h +++ b/sdk/interface/ConnectionImpl.h @@ -26,7 +26,7 @@ class ConnectionImpl : public Connection { // Implementations of the Connection interface Status - Connect(const ConnectParam& param) override; + Connect(const ConnectParam& connect_param) override; Status Connect(const std::string& uri) override; @@ -37,50 +37,6 @@ class ConnectionImpl : public Connection { Status Disconnect() override; - Status - CreateTable(const TableSchema& param) override; - - bool - HasTable(const std::string& table_name) override; - - Status - DropTable(const std::string& table_name) override; - - Status - CreateIndex(const IndexParam& index_param) override; - - Status - Insert(const std::string& table_name, const std::string& partition_tag, const std::vector& record_array, - std::vector& id_array) override; - - Status - GetVectorByID(const std::string& table_name, int64_t vector_id, RowRecord& vector_data) override; - - Status - GetIDsInSegment(const std::string& table_name, const std::string& segment_name, - std::vector& id_array) override; - - Status - Search(const std::string& table_name, const std::vector& partition_tag_array, - const std::vector& query_record_array, int64_t topk, - const std::string& extra_params, TopKQueryResult& topk_query_result) override; - - Status - SearchByID(const std::string& table_name, const std::vector& partition_tag_array, int64_t query_id, - int64_t topk, const std::string& extra_params, TopKQueryResult& topk_query_result) override; - - Status - DescribeTable(const std::string& table_name, TableSchema& table_schema) override; - - Status - CountTable(const std::string& table_name, int64_t& row_count) override; - - Status - ShowTables(std::vector& table_array) override; - - Status - ShowTableInfo(const std::string& table_name, TableInfo& table_info) override; - std::string ClientVersion() const override; @@ -90,30 +46,6 @@ class ConnectionImpl : public Connection { std::string ServerStatus() const override; - std::string - DumpTaskTables() const override; - - Status - DeleteByID(const std::string& table_name, const std::vector& id_array) override; - - Status - PreloadTable(const std::string& table_name) const override; - - Status - DescribeIndex(const std::string& table_name, IndexParam& index_param) const override; - - Status - DropIndex(const std::string& table_name) const override; - - Status - CreatePartition(const PartitionParam& param) override; - - Status - ShowPartitions(const std::string& table_name, PartitionTagList& partition_tag_array) const override; - - Status - DropPartition(const PartitionParam& param) override; - Status GetConfig(const std::string& node_name, std::string& value) const override; @@ -121,13 +53,76 @@ class ConnectionImpl : public Connection { SetConfig(const std::string& node_name, const std::string& value) const override; Status - FlushTable(const std::string& table_name) override; + CreateCollection(const CollectionParam& param) override; + + bool + HasCollection(const std::string& collection_name) override; + + Status + DropCollection(const std::string& collection_name) override; + + Status + CreateIndex(const IndexParam& index_param) override; + + Status + Insert(const std::string& collection_name, + const std::string& partition_tag, + const std::vector& entity_array, + std::vector& id_array) override; + + Status + GetEntityByID(const std::string& collection_name, int64_t entity_id, Entity& entity_data) override; + + Status + GetIDsInSegment(const std::string& collection_name, const std::string& segment_name, + std::vector& id_array) override; + + Status + Search(const std::string& collection_name, const std::vector& partition_tag_array, + const std::vector& entity_array, int64_t topk, + const std::string& extra_params, TopKQueryResult& topk_query_result) override; + + Status + DescribeCollection(const std::string& collection_name, CollectionParam& collection_schema) override; + + Status + CountCollection(const std::string& collection_name, int64_t& entity_count) override; + + Status + ShowCollections(std::vector& collection_array) override; + + Status + ShowCollectionInfo(const std::string& collection_name, CollectionInfo& collection_info) override; + + Status + DeleteByID(const std::string& collection_name, const std::vector& id_array) override; + + Status + PreloadCollection(const std::string& collection_name) const override; + + Status + DescribeIndex(const std::string& collection_name, IndexParam& index_param) const override; + + Status + DropIndex(const std::string& collection_name) const override; + + Status + CreatePartition(const PartitionParam& partition_param) override; + + Status + ShowPartitions(const std::string& collection_name, PartitionTagList& partition_tag_array) const override; + + Status + DropPartition(const PartitionParam& partition_param) override; + + Status + FlushCollection(const std::string& collection_name) override; Status Flush() override; Status - CompactTable(const std::string& table_name) override; + CompactCollection(const std::string& collection_name) override; private: std::shared_ptr client_proxy_;