diff --git a/CHANGELOG.md b/CHANGELOG.md index 933f248bf5..480f599c36 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,16 +11,16 @@ Please mark all change in change log and use the issue from GitHub - \#805 IVFTest.gpu_seal_test unittest failed - \#831 Judge branch error in CommonUtil.cpp - \#977 Server crash when create tables concurrently -- \#990 check gpu resources setting when assign repeated value +- \#990 Check gpu resources setting when assign repeated value - \#995 table count set to 0 if no tables found -- \#1010 improve error message when offset or page_size is equal 0 +- \#1010 Improve error message when offset or page_size is equal 0 - \#1022 check if partition name is legal - \#1028 check if table exists when show partitions - \#1029 check if table exists when try to delete partition - \#1066 optimize http insert and search speed - \#1067 Add binary vectors support in http server -- \#1075 improve error message when page size or offset is illegal -- \#1082 check page_size or offset value to avoid float +- \#1075 Improve error message when page size or offset is illegal +- \#1082 Check page_size or offset value to avoid float - \#1115 http server support load table into memory - \#1152 Error log output continuously after server start - \#1211 Server down caused by searching with index_type: HNSW @@ -29,18 +29,21 @@ Please mark all change in change log and use the issue from GitHub - \#1359 Negative distance value returned when searching with HNSW index type - \#1429 Server crashed when searching vectors with GPU - \#1476 Fix vectors results bug when getting vectors from segments -- \#1484 Index type changed to IDMAP after compacted +- \#1484 Index type changed to IDMAP after compacted +- \#1491 Server crashed during adding vectors - \#1499 Fix duplicated ID number issue -- \#1491 Server crashed during adding vectors -- \#1504 Avoid possible race condition between delete and search +- \#1504 Avoid possible race condition between delete and search - \#1507 set_config for insert_buffer_size is wrong - \#1510 Add set interfaces for WAL configurations - \#1511 Fix big integer cannot pass to server correctly - \#1518 Table count did not match after deleting vectors and compact - \#1521 Make cache_insert_data take effect in-service - \#1525 Add setter API for config preload_table +- \#1529 Fix server crash when cache_insert_data enabled - \#1530 Set table file with correct engine type in meta +- \#1532 Search with ivf_flat failed with open-dataset: sift-256-hamming - \#1535 Degradation searching performance with metric_type: binary_idmap +- \#1556 Index file not created after table and index created ## Feature - \#216 Add CLI to get server info @@ -86,9 +89,11 @@ Please mark all change in change log and use the issue from GitHub - \#1320 Remove debug logging from faiss - \#1426 Support to configure whether to enabled autoflush and the autoflush interval - \#1444 Improve delete +- \#1448 General proto api for NNS libraries - \#1480 Add return code for AVX512 selection - \#1524 Update config "preload_table" description - \#1537 Optimize raw vector and uids read/write +- \#1544 Update resources name in HTTP module ## Task - \#1327 Exclude third-party code from codebeat diff --git a/ci/jenkins/step/singleDevTest.groovy b/ci/jenkins/step/singleDevTest.groovy index 9f4882a56c..c744328598 100644 --- a/ci/jenkins/step/singleDevTest.groovy +++ b/ci/jenkins/step/singleDevTest.groovy @@ -2,6 +2,7 @@ timeout(time: 60, unit: 'MINUTES') { dir ("tests/milvus_python_test") { // sh 'python3 -m pip install -r requirements.txt -i http://pypi.douban.com/simple --trusted-host pypi.douban.com' sh 'python3 -m pip install -r requirements.txt' + sh 'python3 -m pip install git+https://github.com/BossZou/pymilvus.git@nns' sh "pytest . --alluredir=\"test_out/dev/single/sqlite\" --level=1 --ip ${env.HELM_RELEASE_NAME}.milvus.svc.cluster.local" } diff --git a/core/src/config/handler/ConfigHandler.h b/core/src/config/handler/ConfigHandler.h index 84e6e91dec..8f31757a7e 100644 --- a/core/src/config/handler/ConfigHandler.h +++ b/core/src/config/handler/ConfigHandler.h @@ -14,7 +14,6 @@ #include #include "server/Config.h" -#include "utils/Log.h" namespace milvus { namespace server { diff --git a/core/src/db/DB.h b/core/src/db/DB.h index f70b806473..8999ab2e74 100644 --- a/core/src/db/DB.h +++ b/core/src/db/DB.h @@ -112,18 +112,18 @@ class DB { virtual Status QueryByID(const std::shared_ptr& context, const std::string& table_id, - const std::vector& partition_tags, uint64_t k, uint64_t nprobe, IDNumber vector_id, - ResultIds& result_ids, ResultDistances& result_distances) = 0; + const std::vector& partition_tags, uint64_t k, const milvus::json& extra_params, + IDNumber vector_id, ResultIds& result_ids, ResultDistances& result_distances) = 0; virtual Status Query(const std::shared_ptr& context, const std::string& table_id, - const std::vector& partition_tags, uint64_t k, uint64_t nprobe, const VectorsData& vectors, - ResultIds& result_ids, ResultDistances& result_distances) = 0; + const std::vector& partition_tags, uint64_t k, const milvus::json& extra_params, + const VectorsData& vectors, ResultIds& result_ids, ResultDistances& result_distances) = 0; virtual Status QueryByFileID(const std::shared_ptr& context, const std::string& table_id, - const std::vector& file_ids, uint64_t k, uint64_t nprobe, const VectorsData& vectors, - ResultIds& result_ids, ResultDistances& result_distances) = 0; + const std::vector& file_ids, uint64_t k, const milvus::json& extra_params, + const VectorsData& vectors, ResultIds& result_ids, ResultDistances& result_distances) = 0; virtual Status Size(uint64_t& result) = 0; diff --git a/core/src/db/DBImpl.cpp b/core/src/db/DBImpl.cpp index 1da3d90bfe..29ed17308a 100644 --- a/core/src/db/DBImpl.cpp +++ b/core/src/db/DBImpl.cpp @@ -371,8 +371,10 @@ DBImpl::PreloadTable(const std::string& table_id) { } else { engine_type = (EngineType)file.engine_type_; } - ExecutionEnginePtr engine = EngineFactory::Build(file.dimension_, file.location_, engine_type, - (MetricType)file.metric_type_, file.nlist_); + + auto json = milvus::json::parse(file.index_params_); + ExecutionEnginePtr engine = + EngineFactory::Build(file.dimension_, file.location_, engine_type, (MetricType)file.metric_type_, json); fiu_do_on("DBImpl.PreloadTable.null_engine", engine = nullptr); if (engine == nullptr) { ENGINE_LOG_ERROR << "Invalid engine type"; @@ -382,7 +384,7 @@ DBImpl::PreloadTable(const std::string& table_id) { size += engine->PhysicalSize(); fiu_do_on("DBImpl.PreloadTable.exceed_cache", size = available_size + 1); if (size > available_size) { - ENGINE_LOG_DEBUG << "Pre-load canceled since cache almost full"; + ENGINE_LOG_DEBUG << "Pre-load cancelled since cache is almost full"; return Status(SERVER_CACHE_FULL, "Cache is full"); } else { try { @@ -810,7 +812,7 @@ DBImpl::CompactFile(const std::string& table_id, const meta::TableFileSchema& fi // Update table files state // if index type isn't IDMAP, set file type to TO_INDEX if file size exceed index_file_size // else set file type to RAW, no need to build index - if (compacted_file.engine_type_ != (int)EngineType::FAISS_IDMAP) { + if (!utils::IsRawIndexType(compacted_file.engine_type_)) { compacted_file.file_type_ = (segment_writer_ptr->Size() >= compacted_file.index_file_size_) ? meta::TableFileSchema::TO_INDEX : meta::TableFileSchema::RAW; @@ -1110,8 +1112,8 @@ DBImpl::DropIndex(const std::string& table_id) { Status DBImpl::QueryByID(const std::shared_ptr& context, const std::string& table_id, - const std::vector& partition_tags, uint64_t k, uint64_t nprobe, IDNumber vector_id, - ResultIds& result_ids, ResultDistances& result_distances) { + const std::vector& partition_tags, uint64_t k, const milvus::json& extra_params, + IDNumber vector_id, ResultIds& result_ids, ResultDistances& result_distances) { if (!initialized_.load(std::memory_order_acquire)) { return SHUTDOWN_ERROR; } @@ -1119,14 +1121,15 @@ DBImpl::QueryByID(const std::shared_ptr& context, const std::st VectorsData vectors_data = VectorsData(); vectors_data.id_array_.emplace_back(vector_id); vectors_data.vector_count_ = 1; - Status result = Query(context, table_id, partition_tags, k, nprobe, vectors_data, result_ids, result_distances); + Status result = + Query(context, table_id, partition_tags, k, extra_params, vectors_data, result_ids, result_distances); return result; } Status DBImpl::Query(const std::shared_ptr& context, const std::string& table_id, - const std::vector& partition_tags, uint64_t k, uint64_t nprobe, const VectorsData& vectors, - ResultIds& result_ids, ResultDistances& result_distances) { + const std::vector& partition_tags, uint64_t k, const milvus::json& extra_params, + const VectorsData& vectors, ResultIds& result_ids, ResultDistances& result_distances) { auto query_ctx = context->Child("Query"); if (!initialized_.load(std::memory_order_acquire)) { @@ -1169,7 +1172,7 @@ DBImpl::Query(const std::shared_ptr& context, const std::string } cache::CpuCacheMgr::GetInstance()->PrintInfo(); // print cache info before query - status = QueryAsync(query_ctx, table_id, files_array, k, nprobe, vectors, result_ids, result_distances); + status = QueryAsync(query_ctx, table_id, files_array, k, extra_params, vectors, result_ids, result_distances); cache::CpuCacheMgr::GetInstance()->PrintInfo(); // print cache info after query query_ctx->GetTraceContext()->GetSpan()->Finish(); @@ -1179,8 +1182,8 @@ DBImpl::Query(const std::shared_ptr& context, const std::string Status DBImpl::QueryByFileID(const std::shared_ptr& context, const std::string& table_id, - const std::vector& file_ids, uint64_t k, uint64_t nprobe, const VectorsData& vectors, - ResultIds& result_ids, ResultDistances& result_distances) { + const std::vector& file_ids, uint64_t k, const milvus::json& extra_params, + const VectorsData& vectors, ResultIds& result_ids, ResultDistances& result_distances) { auto query_ctx = context->Child("Query by file id"); if (!initialized_.load(std::memory_order_acquire)) { @@ -1208,7 +1211,7 @@ DBImpl::QueryByFileID(const std::shared_ptr& context, const std } cache::CpuCacheMgr::GetInstance()->PrintInfo(); // print cache info before query - status = QueryAsync(query_ctx, table_id, files_array, k, nprobe, vectors, result_ids, result_distances); + status = QueryAsync(query_ctx, table_id, files_array, k, extra_params, vectors, result_ids, result_distances); cache::CpuCacheMgr::GetInstance()->PrintInfo(); // print cache info after query query_ctx->GetTraceContext()->GetSpan()->Finish(); @@ -1230,8 +1233,8 @@ DBImpl::Size(uint64_t& result) { /////////////////////////////////////////////////////////////////////////////////////////////////////////////////// Status DBImpl::QueryAsync(const std::shared_ptr& context, const std::string& table_id, - const meta::TableFilesSchema& files, uint64_t k, uint64_t nprobe, const VectorsData& vectors, - ResultIds& result_ids, ResultDistances& result_distances) { + const meta::TableFilesSchema& files, uint64_t k, const milvus::json& extra_params, + const VectorsData& vectors, ResultIds& result_ids, ResultDistances& result_distances) { auto query_async_ctx = context->Child("Query Async"); server::CollectQueryMetrics metrics(vectors.vector_count_); @@ -1242,7 +1245,7 @@ DBImpl::QueryAsync(const std::shared_ptr& context, const std::s auto status = OngoingFileChecker::GetInstance().MarkOngoingFiles(files); ENGINE_LOG_DEBUG << "Engine query begin, index file count: " << files.size(); - scheduler::SearchJobPtr job = std::make_shared(query_async_ctx, k, nprobe, vectors); + scheduler::SearchJobPtr job = std::make_shared(query_async_ctx, k, extra_params, vectors); for (auto& file : files) { scheduler::TableFileSchemaPtr file_ptr = std::make_shared(file); job->AddIndexFile(file_ptr); @@ -1465,7 +1468,7 @@ DBImpl::MergeFiles(const std::string& table_id, const meta::TableFilesSchema& fi // step 4: update table files state // if index type isn't IDMAP, set file type to TO_INDEX if file size exceed index_file_size // else set file type to RAW, no need to build index - if (table_file.engine_type_ != (int)EngineType::FAISS_IDMAP) { + if (!utils::IsRawIndexType(table_file.engine_type_)) { table_file.file_type_ = (segment_writer_ptr->Size() >= table_file.index_file_size_) ? meta::TableFileSchema::TO_INDEX : meta::TableFileSchema::RAW; @@ -1767,7 +1770,7 @@ DBImpl::BuildTableIndexRecursively(const std::string& table_id, const TableIndex // for IDMAP type, only wait all NEW file converted to RAW file // for other type, wait NEW/RAW/NEW_MERGE/NEW_INDEX/TO_INDEX files converted to INDEX files std::vector file_types; - if (index.engine_type_ == static_cast(EngineType::FAISS_IDMAP)) { + if (utils::IsRawIndexType(index.engine_type_)) { file_types = { static_cast(meta::TableFileSchema::NEW), static_cast(meta::TableFileSchema::NEW_MERGE), @@ -1789,7 +1792,7 @@ DBImpl::BuildTableIndexRecursively(const std::string& table_id, const TableIndex while (!table_files.empty()) { ENGINE_LOG_DEBUG << "Non index files detected! Will build index " << times; - if (index.engine_type_ != (int)EngineType::FAISS_IDMAP) { + if (!utils::IsRawIndexType(index.engine_type_)) { status = meta_ptr_->UpdateTableFilesToIndex(table_id); } diff --git a/core/src/db/DBImpl.h b/core/src/db/DBImpl.h index bedfa02536..79c7e75a8c 100644 --- a/core/src/db/DBImpl.h +++ b/core/src/db/DBImpl.h @@ -131,18 +131,18 @@ class DBImpl : public DB, public server::CacheConfigHandler { Status QueryByID(const std::shared_ptr& context, const std::string& table_id, - const std::vector& partition_tags, uint64_t k, uint64_t nprobe, IDNumber vector_id, - ResultIds& result_ids, ResultDistances& result_distances) override; + const std::vector& partition_tags, uint64_t k, const milvus::json& extra_params, + IDNumber vector_id, ResultIds& result_ids, ResultDistances& result_distances) override; Status Query(const std::shared_ptr& context, const std::string& table_id, - const std::vector& partition_tags, uint64_t k, uint64_t nprobe, const VectorsData& vectors, - ResultIds& result_ids, ResultDistances& result_distances) override; + const std::vector& partition_tags, uint64_t k, const milvus::json& extra_params, + const VectorsData& vectors, ResultIds& result_ids, ResultDistances& result_distances) override; Status QueryByFileID(const std::shared_ptr& context, const std::string& table_id, - const std::vector& file_ids, uint64_t k, uint64_t nprobe, const VectorsData& vectors, - ResultIds& result_ids, ResultDistances& result_distances) override; + const std::vector& file_ids, uint64_t k, const milvus::json& extra_params, + const VectorsData& vectors, ResultIds& result_ids, ResultDistances& result_distances) override; Status Size(uint64_t& result) override; @@ -154,8 +154,8 @@ class DBImpl : public DB, public server::CacheConfigHandler { private: Status QueryAsync(const std::shared_ptr& context, const std::string& table_id, - const meta::TableFilesSchema& files, uint64_t k, uint64_t nprobe, const VectorsData& vectors, - ResultIds& result_ids, ResultDistances& result_distances); + const meta::TableFilesSchema& files, uint64_t k, const milvus::json& extra_params, + const VectorsData& vectors, ResultIds& result_ids, ResultDistances& result_distances); Status GetVectorByIdHelper(const std::string& table_id, IDNumber vector_id, VectorsData& vector, diff --git a/core/src/db/Types.h b/core/src/db/Types.h index c5378affe2..7bffb3c02b 100644 --- a/core/src/db/Types.h +++ b/core/src/db/Types.h @@ -22,6 +22,7 @@ #include "db/engine/ExecutionEngine.h" #include "segment/Types.h" +#include "utils/Json.h" namespace milvus { namespace engine { @@ -35,8 +36,8 @@ typedef std::vector ResultDistances; struct TableIndex { int32_t engine_type_ = (int)EngineType::FAISS_IDMAP; - int32_t nlist_ = 16384; int32_t metric_type_ = (int)MetricType::L2; + milvus::json extra_params_ = {{"nlist", 16384}}; }; struct VectorsData { diff --git a/core/src/db/Utils.cpp b/core/src/db/Utils.cpp index 4fb26f222b..f9ac5dfe40 100644 --- a/core/src/db/Utils.cpp +++ b/core/src/db/Utils.cpp @@ -211,10 +211,15 @@ GetParentPath(const std::string& path, std::string& parent_path) { bool IsSameIndex(const TableIndex& index1, const TableIndex& index2) { - return index1.engine_type_ == index2.engine_type_ && index1.nlist_ == index2.nlist_ && + return index1.engine_type_ == index2.engine_type_ && index1.extra_params_ == index2.extra_params_ && index1.metric_type_ == index2.metric_type_; } +bool +IsRawIndexType(int32_t type) { + return (type == (int32_t)EngineType::FAISS_IDMAP) || (type == (int32_t)EngineType::FAISS_BIN_IDMAP); +} + meta::DateT GetDate(const std::time_t& t, int day_delta) { struct tm ltm; diff --git a/core/src/db/Utils.h b/core/src/db/Utils.h index 810a0b5292..88197d7d5c 100644 --- a/core/src/db/Utils.h +++ b/core/src/db/Utils.h @@ -45,6 +45,9 @@ GetParentPath(const std::string& path, std::string& parent_path); bool IsSameIndex(const TableIndex& index1, const TableIndex& index2); +bool +IsRawIndexType(int32_t type); + meta::DateT GetDate(const std::time_t& t, int day_delta = 0); meta::DateT diff --git a/core/src/db/engine/EngineFactory.cpp b/core/src/db/engine/EngineFactory.cpp index 964c6538e2..285c028d16 100644 --- a/core/src/db/engine/EngineFactory.cpp +++ b/core/src/db/engine/EngineFactory.cpp @@ -20,7 +20,7 @@ namespace engine { ExecutionEnginePtr EngineFactory::Build(uint16_t dimension, const std::string& location, EngineType index_type, MetricType metric_type, - int32_t nlist) { + const milvus::json& index_params) { if (index_type == EngineType::INVALID) { ENGINE_LOG_ERROR << "Unsupported engine type"; return nullptr; @@ -28,7 +28,7 @@ EngineFactory::Build(uint16_t dimension, const std::string& location, EngineType ENGINE_LOG_DEBUG << "EngineFactory index type: " << (int)index_type; ExecutionEnginePtr execution_engine_ptr = - std::make_shared(dimension, location, index_type, metric_type, nlist); + std::make_shared(dimension, location, index_type, metric_type, index_params); execution_engine_ptr->Init(); return execution_engine_ptr; diff --git a/core/src/db/engine/EngineFactory.h b/core/src/db/engine/EngineFactory.h index ed9894f616..46d37b86a7 100644 --- a/core/src/db/engine/EngineFactory.h +++ b/core/src/db/engine/EngineFactory.h @@ -12,6 +12,7 @@ #pragma once #include "ExecutionEngine.h" +#include "utils/Json.h" #include "utils/Status.h" #include @@ -23,7 +24,7 @@ class EngineFactory { public: static ExecutionEnginePtr Build(uint16_t dimension, const std::string& location, EngineType index_type, MetricType metric_type, - int32_t nlist); + const milvus::json& index_params); }; } // namespace engine diff --git a/core/src/db/engine/ExecutionEngine.h b/core/src/db/engine/ExecutionEngine.h index f027ad7f02..42edf56171 100644 --- a/core/src/db/engine/ExecutionEngine.h +++ b/core/src/db/engine/ExecutionEngine.h @@ -15,6 +15,7 @@ #include #include +#include "utils/Json.h" #include "utils/Status.h" namespace milvus { @@ -94,15 +95,16 @@ class ExecutionEngine { GetVectorByID(const int64_t& id, uint8_t* vector, bool hybrid) = 0; virtual Status - Search(int64_t n, const float* data, int64_t k, int64_t nprobe, float* distances, int64_t* labels, bool hybrid) = 0; - - virtual Status - Search(int64_t n, const uint8_t* data, int64_t k, int64_t nprobe, float* distances, int64_t* labels, + Search(int64_t n, const float* data, int64_t k, const milvus::json& extra_params, float* distances, int64_t* labels, bool hybrid) = 0; virtual Status - Search(int64_t n, const std::vector& ids, int64_t k, int64_t nprobe, float* distances, int64_t* labels, - bool hybrid) = 0; + Search(int64_t n, const uint8_t* data, int64_t k, const milvus::json& extra_params, float* distances, + int64_t* labels, bool hybrid) = 0; + + virtual Status + Search(int64_t n, const std::vector& ids, int64_t k, const milvus::json& extra_params, float* distances, + int64_t* labels, bool hybrid) = 0; virtual std::shared_ptr BuildIndex(const std::string& location, EngineType engine_type) = 0; diff --git a/core/src/db/engine/ExecutionEngineImpl.cpp b/core/src/db/engine/ExecutionEngineImpl.cpp index 6992fc6539..1fe820aa7a 100644 --- a/core/src/db/engine/ExecutionEngineImpl.cpp +++ b/core/src/db/engine/ExecutionEngineImpl.cpp @@ -43,22 +43,22 @@ namespace engine { namespace { Status -MappingMetricType(MetricType metric_type, knowhere::METRICTYPE& kw_type) { +MappingMetricType(MetricType metric_type, milvus::json& conf) { switch (metric_type) { case MetricType::IP: - kw_type = knowhere::METRICTYPE::IP; + conf[knowhere::Metric::TYPE] = knowhere::Metric::IP; break; case MetricType::L2: - kw_type = knowhere::METRICTYPE::L2; + conf[knowhere::Metric::TYPE] = knowhere::Metric::L2; break; case MetricType::HAMMING: - kw_type = knowhere::METRICTYPE::HAMMING; + conf[knowhere::Metric::TYPE] = knowhere::Metric::HAMMING; break; case MetricType::JACCARD: - kw_type = knowhere::METRICTYPE::JACCARD; + conf[knowhere::Metric::TYPE] = knowhere::Metric::JACCARD; break; case MetricType::TANIMOTO: - kw_type = knowhere::METRICTYPE::TANIMOTO; + conf[knowhere::Metric::TYPE] = knowhere::Metric::TANIMOTO; break; default: return Status(DB_ERROR, "Unsupported metric type"); @@ -94,8 +94,12 @@ class CachedQuantizer : public cache::DataObj { }; ExecutionEngineImpl::ExecutionEngineImpl(uint16_t dimension, const std::string& location, EngineType index_type, - MetricType metric_type, int32_t nlist) - : location_(location), dim_(dimension), index_type_(index_type), metric_type_(metric_type), nlist_(nlist) { + MetricType metric_type, const milvus::json& index_params) + : location_(location), + dim_(dimension), + index_type_(index_type), + metric_type_(metric_type), + index_params_(index_params) { EngineType tmp_index_type = server::ValidationUtil::IsBinaryMetricType((int32_t)metric_type) ? EngineType::FAISS_BIN_IDMAP : EngineType::FAISS_IDMAP; @@ -104,16 +108,15 @@ ExecutionEngineImpl::ExecutionEngineImpl(uint16_t dimension, const std::string& throw Exception(DB_ERROR, "Unsupported index type"); } - TempMetaConf temp_conf; - temp_conf.gpu_id = gpu_num_; - temp_conf.dim = dimension; - auto status = MappingMetricType(metric_type, temp_conf.metric_type); - if (!status.ok()) { - throw Exception(DB_ERROR, status.message()); - } - + milvus::json conf = index_params; + conf[knowhere::meta::DEVICEID] = gpu_num_; + conf[knowhere::meta::DIM] = dimension; + MappingMetricType(metric_type, conf); + ENGINE_LOG_DEBUG << "Index params: " << conf.dump(); auto adapter = AdapterMgr::GetInstance().GetAdapter(index_->GetType()); - auto conf = adapter->Match(temp_conf); + if (!adapter->CheckTrain(conf)) { + throw Exception(DB_ERROR, "Illegal index params"); + } ErrorCode ec = KNOWHERE_UNEXPECTED_ERROR; if (auto bf_index = std::dynamic_pointer_cast(index_)) { @@ -127,8 +130,12 @@ ExecutionEngineImpl::ExecutionEngineImpl(uint16_t dimension, const std::string& } ExecutionEngineImpl::ExecutionEngineImpl(VecIndexPtr index, const std::string& location, EngineType index_type, - MetricType metric_type, int32_t nlist) - : index_(std::move(index)), location_(location), index_type_(index_type), metric_type_(metric_type), nlist_(nlist) { + MetricType metric_type, const milvus::json& index_params) + : index_(std::move(index)), + location_(location), + index_type_(index_type), + metric_type_(metric_type), + index_params_(index_params) { } VecIndexPtr @@ -273,10 +280,9 @@ ExecutionEngineImpl::HybridLoad() const { auto best_index = std::distance(all_free_mem.begin(), max_e); auto best_device_id = gpus[best_index]; - auto quantizer_conf = std::make_shared(); - quantizer_conf->mode = 1; - quantizer_conf->gpu_id = best_device_id; + milvus::json quantizer_conf{{knowhere::meta::DEVICEID, best_device_id}, {"mode", 1}}; auto quantizer = index_->LoadQuantizer(quantizer_conf); + ENGINE_LOG_DEBUG << "Quantizer params: " << quantizer_conf.dump(); if (quantizer == nullptr) { ENGINE_LOG_ERROR << "quantizer is nullptr"; } @@ -400,22 +406,18 @@ ExecutionEngineImpl::Load(bool to_cache) { utils::GetParentPath(location_, segment_dir); auto segment_reader_ptr = std::make_shared(segment_dir); - if (index_type_ == EngineType::FAISS_IDMAP || index_type_ == EngineType::FAISS_BIN_IDMAP) { + if (utils::IsRawIndexType((int32_t)index_type_)) { index_ = index_type_ == EngineType::FAISS_IDMAP ? GetVecIndexFactory(IndexType::FAISS_IDMAP) : GetVecIndexFactory(IndexType::FAISS_BIN_IDMAP); - - TempMetaConf temp_conf; - temp_conf.gpu_id = gpu_num_; - temp_conf.dim = dim_; - auto status = MappingMetricType(metric_type_, temp_conf.metric_type); - if (!status.ok()) { - return status; + milvus::json conf{{knowhere::meta::DEVICEID, gpu_num_}, {knowhere::meta::DIM, dim_}}; + MappingMetricType(metric_type_, conf); + auto adapter = AdapterMgr::GetInstance().GetAdapter(index_->GetType()); + ENGINE_LOG_DEBUG << "Index params: " << conf.dump(); + if (!adapter->CheckTrain(conf)) { + throw Exception(DB_ERROR, "Illegal index params"); } - auto adapter = AdapterMgr::GetInstance().GetAdapter(index_->GetType()); - auto conf = adapter->Match(temp_conf); - - status = segment_reader_ptr->Load(); + auto status = segment_reader_ptr->Load(); if (!status.ok()) { std::string msg = "Failed to load segment from " + location_; ENGINE_LOG_ERROR << msg; @@ -429,6 +431,7 @@ ExecutionEngineImpl::Load(bool to_cache) { auto vectors_uids = vectors->GetUids(); index_->SetUids(vectors_uids); + ENGINE_LOG_DEBUG << "set uids " << index_->GetUids().size() << " for index " << location_; auto vectors_data = vectors->GetData(); @@ -453,7 +456,7 @@ ExecutionEngineImpl::Load(bool to_cache) { float_vectors.data(), Config()); status = std::static_pointer_cast(index_)->SetBlacklist(concurrent_bitset_ptr); - int64_t index_size = vectors->GetCount() * conf->d * sizeof(float); + int64_t index_size = vectors->GetCount() * dim_ * sizeof(float); int64_t bitset_size = vectors->GetCount() / 8; index_->set_size(index_size + bitset_size); } else if (index_type_ == EngineType::FAISS_BIN_IDMAP) { @@ -465,7 +468,7 @@ ExecutionEngineImpl::Load(bool to_cache) { vectors_data.data(), Config()); status = std::static_pointer_cast(index_)->SetBlacklist(concurrent_bitset_ptr); - int64_t index_size = vectors->GetCount() * conf->d * sizeof(uint8_t); + int64_t index_size = vectors->GetCount() * dim_ * sizeof(uint8_t); int64_t bitset_size = vectors->GetCount() / 8; index_->set_size(index_size + bitset_size); } @@ -508,6 +511,7 @@ ExecutionEngineImpl::Load(bool to_cache) { std::vector uids; segment_reader_ptr->LoadUids(uids); index_->SetUids(uids); + ENGINE_LOG_DEBUG << "set uids " << index_->GetUids().size() << " for index " << location_; ENGINE_LOG_DEBUG << "Finished loading index file from segment " << segment_dir; } @@ -548,9 +552,7 @@ ExecutionEngineImpl::CopyToGpu(uint64_t device_id, bool hybrid) { if (device_id != NOT_FOUND) { // cache hit - auto config = std::make_shared(); - config->gpu_id = device_id; - config->mode = 2; + milvus::json quantizer_conf{{knowhere::meta::DEVICEID : device_id}, {"mode" : 2}}; auto new_index = index_->LoadData(quantizer, config); index_ = new_index; } @@ -723,30 +725,36 @@ ExecutionEngineImpl::BuildIndex(const std::string& location, EngineType engine_t throw Exception(DB_ERROR, "Unsupported index type"); } - TempMetaConf temp_conf; - temp_conf.gpu_id = gpu_num_; - temp_conf.dim = Dimension(); - temp_conf.nlist = nlist_; - temp_conf.size = Count(); - auto status = MappingMetricType(metric_type_, temp_conf.metric_type); - if (!status.ok()) { - throw Exception(DB_ERROR, status.message()); - } - + milvus::json conf = index_params_; + conf[knowhere::meta::DIM] = Dimension(); + conf[knowhere::meta::ROWS] = Count(); + conf[knowhere::meta::DEVICEID] = gpu_num_; + MappingMetricType(metric_type_, conf); + ENGINE_LOG_DEBUG << "Index params: " << conf.dump(); auto adapter = AdapterMgr::GetInstance().GetAdapter(to_index->GetType()); - auto conf = adapter->Match(temp_conf); + if (!adapter->CheckTrain(conf)) { + throw Exception(DB_ERROR, "Illegal index params"); + } + ENGINE_LOG_DEBUG << "Index config: " << conf.dump(); + auto status = Status::OK(); + std::vector uids; if (from_index) { status = to_index->BuildAll(Count(), from_index->GetRawVectors(), from_index->GetRawIds(), conf); + uids = from_index->GetUids(); } else if (bin_from_index) { status = to_index->BuildAll(Count(), bin_from_index->GetRawVectors(), bin_from_index->GetRawIds(), conf); + uids = bin_from_index->GetUids(); } + to_index->SetUids(uids); + ENGINE_LOG_DEBUG << "set uids " << to_index->GetUids().size() << " for " << location; + if (!status.ok()) { throw Exception(DB_ERROR, status.message()); } ENGINE_LOG_DEBUG << "Finish build index file: " << location << " size: " << to_index->Size(); - return std::make_shared(to_index, location, engine_type, metric_type_, nlist_); + return std::make_shared(to_index, location, engine_type, metric_type_, index_params_); } // map offsets to ids @@ -761,8 +769,8 @@ MapUids(const std::vector& uids, int64_t* labels, size_t num) } Status -ExecutionEngineImpl::Search(int64_t n, const float* data, int64_t k, int64_t nprobe, float* distances, int64_t* labels, - bool hybrid) { +ExecutionEngineImpl::Search(int64_t n, const float* data, int64_t k, const milvus::json& extra_params, float* distances, + int64_t* labels, bool hybrid) { #if 0 if (index_type_ == EngineType::FAISS_IVFSQ8H) { if (!hybrid) { @@ -786,9 +794,7 @@ ExecutionEngineImpl::Search(int64_t n, const float* data, int64_t k, int64_t npr if (device_id != NOT_FOUND) { // cache hit - auto config = std::make_shared(); - config->gpu_id = device_id; - config->mode = 2; + milvus::json quantizer_conf{{knowhere::meta::DEVICEID : device_id}, {"mode" : 2}}; auto new_index = index_->LoadData(quantizer, config); index_ = new_index; } @@ -824,15 +830,13 @@ ExecutionEngineImpl::Search(int64_t n, const float* data, int64_t k, int64_t npr return Status(DB_ERROR, "index is null"); } - ENGINE_LOG_DEBUG << "Search Params: [k] " << k << " [nprobe] " << nprobe; - - // TODO(linxj): remove here. Get conf from function - TempMetaConf temp_conf; - temp_conf.k = k; - temp_conf.nprobe = nprobe; - + milvus::json conf = extra_params; + conf[knowhere::meta::TOPK] = k; auto adapter = AdapterMgr::GetInstance().GetAdapter(index_->GetType()); - auto conf = adapter->MatchSearch(temp_conf, index_->GetType()); + ENGINE_LOG_DEBUG << "Search params: " << conf.dump(); + if (!adapter->CheckSearch(conf, index_->GetType())) { + throw Exception(DB_ERROR, "Illegal search params"); + } if (hybrid) { HybridLoad(); @@ -843,6 +847,7 @@ ExecutionEngineImpl::Search(int64_t n, const float* data, int64_t k, int64_t npr rc.RecordSection("search done"); // map offsets to ids + ENGINE_LOG_DEBUG << "get uids " << index_->GetUids().size() << " from index " << location_; MapUids(index_->GetUids(), labels, n * k); rc.RecordSection("map uids " + std::to_string(n * k)); @@ -858,8 +863,8 @@ ExecutionEngineImpl::Search(int64_t n, const float* data, int64_t k, int64_t npr } Status -ExecutionEngineImpl::Search(int64_t n, const uint8_t* data, int64_t k, int64_t nprobe, float* distances, - int64_t* labels, bool hybrid) { +ExecutionEngineImpl::Search(int64_t n, const uint8_t* data, int64_t k, const milvus::json& extra_params, + float* distances, int64_t* labels, bool hybrid) { TimeRecorder rc("ExecutionEngineImpl::Search uint8"); if (index_ == nullptr) { @@ -867,15 +872,13 @@ ExecutionEngineImpl::Search(int64_t n, const uint8_t* data, int64_t k, int64_t n return Status(DB_ERROR, "index is null"); } - ENGINE_LOG_DEBUG << "Search Params: [k] " << k << " [nprobe] " << nprobe; - - // TODO(linxj): remove here. Get conf from function - TempMetaConf temp_conf; - temp_conf.k = k; - temp_conf.nprobe = nprobe; - + milvus::json conf = extra_params; + conf[knowhere::meta::TOPK] = k; auto adapter = AdapterMgr::GetInstance().GetAdapter(index_->GetType()); - auto conf = adapter->MatchSearch(temp_conf, index_->GetType()); + ENGINE_LOG_DEBUG << "Search params: " << conf.dump(); + if (!adapter->CheckSearch(conf, index_->GetType())) { + throw Exception(DB_ERROR, "Illegal search params"); + } if (hybrid) { HybridLoad(); @@ -886,6 +889,7 @@ ExecutionEngineImpl::Search(int64_t n, const uint8_t* data, int64_t k, int64_t n rc.RecordSection("search done"); // map offsets to ids + ENGINE_LOG_DEBUG << "get uids " << index_->GetUids().size() << " from index " << location_; MapUids(index_->GetUids(), labels, n * k); rc.RecordSection("map uids " + std::to_string(n * k)); @@ -901,8 +905,8 @@ ExecutionEngineImpl::Search(int64_t n, const uint8_t* data, int64_t k, int64_t n } Status -ExecutionEngineImpl::Search(int64_t n, const std::vector& ids, int64_t k, int64_t nprobe, float* distances, - int64_t* labels, bool hybrid) { +ExecutionEngineImpl::Search(int64_t n, const std::vector& ids, int64_t k, const milvus::json& extra_params, + float* distances, int64_t* labels, bool hybrid) { TimeRecorder rc("ExecutionEngineImpl::Search vector of ids"); if (index_ == nullptr) { @@ -910,15 +914,13 @@ ExecutionEngineImpl::Search(int64_t n, const std::vector& ids, int64_t return Status(DB_ERROR, "index is null"); } - ENGINE_LOG_DEBUG << "Search by ids Params: [k] " << k << " [nprobe] " << nprobe; - - // TODO(linxj): remove here. Get conf from function - TempMetaConf temp_conf; - temp_conf.k = k; - temp_conf.nprobe = nprobe; - + milvus::json conf = extra_params; + conf[knowhere::meta::TOPK] = k; auto adapter = AdapterMgr::GetInstance().GetAdapter(index_->GetType()); - auto conf = adapter->MatchSearch(temp_conf, index_->GetType()); + ENGINE_LOG_DEBUG << "Search params: " << conf.dump(); + if (!adapter->CheckSearch(conf, index_->GetType())) { + throw Exception(DB_ERROR, "Illegal search params"); + } if (hybrid) { HybridLoad(); @@ -971,6 +973,7 @@ ExecutionEngineImpl::Search(int64_t n, const std::vector& ids, int64_t rc.RecordSection("search done"); // map offsets to ids + ENGINE_LOG_DEBUG << "get uids " << index_->GetUids().size() << " from index " << location_; MapUids(uids, labels, offsets.size() * k); rc.RecordSection("map uids " + std::to_string(offsets.size() * k)); @@ -993,19 +996,13 @@ ExecutionEngineImpl::GetVectorByID(const int64_t& id, float* vector, bool hybrid return Status(DB_ERROR, "index is null"); } - // TODO(linxj): remove here. Get conf from function - TempMetaConf temp_conf; - - auto adapter = AdapterMgr::GetInstance().GetAdapter(index_->GetType()); - auto conf = adapter->MatchSearch(temp_conf, index_->GetType()); - if (hybrid) { HybridLoad(); } // Only one id for now std::vector ids{id}; - auto status = index_->GetVectorById(1, ids.data(), vector, conf); + auto status = index_->GetVectorById(1, ids.data(), vector, milvus::json()); if (hybrid) { HybridUnset(); @@ -1026,19 +1023,13 @@ ExecutionEngineImpl::GetVectorByID(const int64_t& id, uint8_t* vector, bool hybr ENGINE_LOG_DEBUG << "Get binary vector by id: " << id; - // TODO(linxj): remove here. Get conf from function - TempMetaConf temp_conf; - - auto adapter = AdapterMgr::GetInstance().GetAdapter(index_->GetType()); - auto conf = adapter->MatchSearch(temp_conf, index_->GetType()); - if (hybrid) { HybridLoad(); } // Only one id for now std::vector ids{id}; - auto status = index_->GetVectorById(1, ids.data(), vector, conf); + auto status = index_->GetVectorById(1, ids.data(), vector, milvus::json()); if (hybrid) { HybridUnset(); @@ -1075,7 +1066,7 @@ ExecutionEngineImpl::Init() { std::vector gpu_ids; Status s = config.GetGpuResourceConfigBuildIndexResources(gpu_ids); if (!s.ok()) { - gpu_num_ = knowhere::INVALID_VALUE; + gpu_num_ = -1; return s; } for (auto id : gpu_ids) { diff --git a/core/src/db/engine/ExecutionEngineImpl.h b/core/src/db/engine/ExecutionEngineImpl.h index c002ef5561..d0d55b2f10 100644 --- a/core/src/db/engine/ExecutionEngineImpl.h +++ b/core/src/db/engine/ExecutionEngineImpl.h @@ -11,7 +11,8 @@ #pragma once -#include +#include "segment/SegmentReader.h" +#include "utils/Json.h" #include #include @@ -26,10 +27,10 @@ namespace engine { class ExecutionEngineImpl : public ExecutionEngine { public: ExecutionEngineImpl(uint16_t dimension, const std::string& location, EngineType index_type, MetricType metric_type, - int32_t nlist); + const milvus::json& index_params); ExecutionEngineImpl(VecIndexPtr index, const std::string& location, EngineType index_type, MetricType metric_type, - int32_t nlist); + const milvus::json& index_params); Status AddWithIds(int64_t n, const float* xdata, const int64_t* xids) override; @@ -77,16 +78,16 @@ class ExecutionEngineImpl : public ExecutionEngine { GetVectorByID(const int64_t& id, uint8_t* vector, bool hybrid) override; Status - Search(int64_t n, const float* data, int64_t k, int64_t nprobe, float* distances, int64_t* labels, + Search(int64_t n, const float* data, int64_t k, const milvus::json& extra_params, float* distances, int64_t* labels, bool hybrid = false) override; Status - Search(int64_t n, const uint8_t* data, int64_t k, int64_t nprobe, float* distances, int64_t* labels, - bool hybrid = false) override; + Search(int64_t n, const uint8_t* data, int64_t k, const milvus::json& extra_params, float* distances, + int64_t* labels, bool hybrid = false) override; Status - Search(int64_t n, const std::vector& ids, int64_t k, int64_t nprobe, float* distances, int64_t* labels, - bool hybrid) override; + Search(int64_t n, const std::vector& ids, int64_t k, const milvus::json& extra_params, float* distances, + int64_t* labels, bool hybrid) override; ExecutionEnginePtr BuildIndex(const std::string& location, EngineType engine_type) override; @@ -136,7 +137,7 @@ class ExecutionEngineImpl : public ExecutionEngine { int64_t dim_; std::string location_; - int64_t nlist_ = 0; + milvus::json index_params_; int64_t gpu_num_ = 0; }; diff --git a/core/src/db/meta/MetaTypes.h b/core/src/db/meta/MetaTypes.h index e53568145f..d8df8c2622 100644 --- a/core/src/db/meta/MetaTypes.h +++ b/core/src/db/meta/MetaTypes.h @@ -54,7 +54,7 @@ struct TableSchema { int64_t flag_ = 0; int64_t index_file_size_ = DEFAULT_INDEX_FILE_SIZE; int32_t engine_type_ = DEFAULT_ENGINE_TYPE; - int32_t nlist_ = DEFAULT_NLIST; + std::string index_params_ = "{ \"nlist\": 16384 }"; int32_t metric_type_ = DEFAULT_METRIC_TYPE; std::string owner_table_; std::string partition_tag_; @@ -89,7 +89,7 @@ struct TableFileSchema { int64_t created_on_ = 0; int64_t index_file_size_ = DEFAULT_INDEX_FILE_SIZE; // not persist to meta int32_t engine_type_ = DEFAULT_ENGINE_TYPE; - int32_t nlist_ = DEFAULT_NLIST; // not persist to meta + std::string index_params_; // not persist to meta int32_t metric_type_ = DEFAULT_METRIC_TYPE; // not persist to meta uint64_t flush_lsn_ = 0; }; // TableFileSchema diff --git a/core/src/db/meta/MySQLMetaImpl.cpp b/core/src/db/meta/MySQLMetaImpl.cpp index 0016c47ef4..b1d28c2d34 100644 --- a/core/src/db/meta/MySQLMetaImpl.cpp +++ b/core/src/db/meta/MySQLMetaImpl.cpp @@ -144,7 +144,7 @@ static const MetaSchema TABLES_SCHEMA(META_TABLES, { MetaField("flag", "BIGINT", "DEFAULT 0 NOT NULL"), MetaField("index_file_size", "BIGINT", "DEFAULT 1024 NOT NULL"), MetaField("engine_type", "INT", "DEFAULT 1 NOT NULL"), - MetaField("nlist", "INT", "DEFAULT 16384 NOT NULL"), + MetaField("index_params", "VARCHAR(512)", "NOT NULL"), MetaField("metric_type", "INT", "DEFAULT 1 NOT NULL"), MetaField("owner_table", "VARCHAR(255)", "NOT NULL"), MetaField("partition_tag", "VARCHAR(255)", "NOT NULL"), @@ -398,7 +398,7 @@ MySQLMetaImpl::CreateTable(TableSchema& table_schema) { std::string flag = std::to_string(table_schema.flag_); std::string index_file_size = std::to_string(table_schema.index_file_size_); std::string engine_type = std::to_string(table_schema.engine_type_); - std::string nlist = std::to_string(table_schema.nlist_); + std::string& index_params = table_schema.index_params_; std::string metric_type = std::to_string(table_schema.metric_type_); std::string& owner_table = table_schema.owner_table_; std::string& partition_tag = table_schema.partition_tag_; @@ -407,9 +407,9 @@ MySQLMetaImpl::CreateTable(TableSchema& table_schema) { createTableQuery << "INSERT INTO " << META_TABLES << " VALUES(" << id << ", " << mysqlpp::quote << table_id << ", " << state << ", " << dimension << ", " << created_on << ", " << flag << ", " - << index_file_size << ", " << engine_type << ", " << nlist << ", " << metric_type << ", " - << mysqlpp::quote << owner_table << ", " << mysqlpp::quote << partition_tag << ", " - << mysqlpp::quote << version << ", " << flush_lsn << ");"; + << index_file_size << ", " << engine_type << ", " << mysqlpp::quote << index_params << ", " + << metric_type << ", " << mysqlpp::quote << owner_table << ", " << mysqlpp::quote + << partition_tag << ", " << mysqlpp::quote << version << ", " << flush_lsn << ");"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::CreateTable: " << createTableQuery.str(); @@ -446,8 +446,8 @@ MySQLMetaImpl::DescribeTable(TableSchema& table_schema) { mysqlpp::Query describeTableQuery = connectionPtr->query(); describeTableQuery - << "SELECT id, state, dimension, created_on, flag, index_file_size, engine_type, nlist, metric_type" - << " ,owner_table, partition_tag, version, flush_lsn" + << "SELECT id, state, dimension, created_on, flag, index_file_size, engine_type, index_params" + << " , metric_type ,owner_table, partition_tag, version, flush_lsn" << " FROM " << META_TABLES << " WHERE table_id = " << mysqlpp::quote << table_schema.table_id_ << " AND state <> " << std::to_string(TableSchema::TO_DELETE) << ";"; @@ -465,7 +465,7 @@ MySQLMetaImpl::DescribeTable(TableSchema& table_schema) { table_schema.flag_ = resRow["flag"]; table_schema.index_file_size_ = resRow["index_file_size"]; table_schema.engine_type_ = resRow["engine_type"]; - table_schema.nlist_ = resRow["nlist"]; + resRow["index_params"].to_string(table_schema.index_params_); table_schema.metric_type_ = resRow["metric_type"]; resRow["owner_table"].to_string(table_schema.owner_table_); resRow["partition_tag"].to_string(table_schema.partition_tag_); @@ -534,7 +534,7 @@ MySQLMetaImpl::AllTables(std::vector& table_schema_array) { } mysqlpp::Query allTablesQuery = connectionPtr->query(); - allTablesQuery << "SELECT id, table_id, dimension, engine_type, nlist, index_file_size, metric_type" + allTablesQuery << "SELECT id, table_id, dimension, engine_type, index_params, index_file_size, metric_type" << " ,owner_table, partition_tag, version, flush_lsn" << " FROM " << META_TABLES << " WHERE state <> " << std::to_string(TableSchema::TO_DELETE) << " AND owner_table = \"\";"; @@ -551,7 +551,7 @@ MySQLMetaImpl::AllTables(std::vector& table_schema_array) { table_schema.dimension_ = resRow["dimension"]; table_schema.index_file_size_ = resRow["index_file_size"]; table_schema.engine_type_ = resRow["engine_type"]; - table_schema.nlist_ = resRow["nlist"]; + resRow["index_params"].to_string(table_schema.index_params_); table_schema.metric_type_ = resRow["metric_type"]; resRow["owner_table"].to_string(table_schema.owner_table_); resRow["partition_tag"].to_string(table_schema.partition_tag_); @@ -673,17 +673,8 @@ MySQLMetaImpl::CreateTableFile(TableFileSchema& file_schema) { file_schema.created_on_ = utils::GetMicroSecTimeStamp(); file_schema.updated_time_ = file_schema.created_on_; file_schema.index_file_size_ = table_schema.index_file_size_; - - if (file_schema.file_type_ == TableFileSchema::FILE_TYPE::NEW || - file_schema.file_type_ == TableFileSchema::FILE_TYPE::NEW_MERGE) { - file_schema.engine_type_ = server::ValidationUtil::IsBinaryMetricType(table_schema.metric_type_) - ? (int32_t)EngineType::FAISS_BIN_IDMAP - : (int32_t)EngineType::FAISS_IDMAP; - } else { - file_schema.engine_type_ = table_schema.engine_type_; - } - - file_schema.nlist_ = table_schema.nlist_; + file_schema.index_params_ = table_schema.index_params_; + file_schema.engine_type_ = table_schema.engine_type_; file_schema.metric_type_ = table_schema.metric_type_; std::string id = "NULL"; // auto-increment @@ -785,7 +776,7 @@ MySQLMetaImpl::GetTableFiles(const std::string& table_id, const std::vectorquery(); - describeTableIndexQuery << "SELECT engine_type, nlist, index_file_size, metric_type" + describeTableIndexQuery << "SELECT engine_type, index_params, index_file_size, metric_type" << " FROM " << META_TABLES << " WHERE table_id = " << mysqlpp::quote << table_id << " AND state <> " << std::to_string(TableSchema::TO_DELETE) << ";"; @@ -1275,7 +1267,9 @@ MySQLMetaImpl::DescribeTableIndex(const std::string& table_id, TableIndex& index const mysqlpp::Row& resRow = res[0]; index.engine_type_ = resRow["engine_type"]; - index.nlist_ = resRow["nlist"]; + std::string str_index_params; + resRow["index_params"].to_string(str_index_params); + index.extra_params_ = milvus::json::parse(str_index_params); index.metric_type_ = resRow["metric_type"]; } else { return Status(DB_NOT_FOUND, "Table " + table_id + " not found"); @@ -1334,7 +1328,7 @@ MySQLMetaImpl::DropTableIndex(const std::string& table_id) { // set table index type to raw dropTableIndexQuery << "UPDATE " << META_TABLES << " SET engine_type = " << std::to_string(DEFAULT_ENGINE_TYPE) - << " ,nlist = " << std::to_string(DEFAULT_NLIST) + << " , index_params = '{}'" << " WHERE table_id = " << mysqlpp::quote << table_id << ";"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::DropTableIndex: " << dropTableIndexQuery.str(); @@ -1426,7 +1420,7 @@ MySQLMetaImpl::ShowPartitions(const std::string& table_id, std::vectorquery(); allPartitionsQuery << "SELECT table_id, id, state, dimension, created_on, flag, index_file_size," - << " engine_type, nlist, metric_type, partition_tag, version FROM " << META_TABLES + << " engine_type, index_params, metric_type, partition_tag, version FROM " << META_TABLES << " WHERE owner_table = " << mysqlpp::quote << table_id << " AND state <> " << std::to_string(TableSchema::TO_DELETE) << ";"; @@ -1445,7 +1439,7 @@ MySQLMetaImpl::ShowPartitions(const std::string& table_id, std::vector& file_schema.created_on_ = resRow["created_on"]; file_schema.index_file_size_ = table_schema.index_file_size_; - file_schema.nlist_ = table_schema.nlist_; + file_schema.index_params_ = table_schema.index_params_; file_schema.metric_type_ = table_schema.metric_type_; file_schema.dimension_ = table_schema.dimension_; @@ -2083,8 +2077,7 @@ MySQLMetaImpl::CleanUpFilesWithTTL(uint64_t seconds /*, CleanUpFilter* filter*/) // If we are deleting a raw table file, it means it's okay to delete the entire segment directory. // Else, we can only delete the single file // TODO(zhiru): We determine whether a table file is raw by its engine type. This is a bit hacky - if (table_file.engine_type_ == (int32_t)EngineType::FAISS_IDMAP || - table_file.engine_type_ == (int32_t)EngineType::FAISS_BIN_IDMAP) { + if (utils::IsRawIndexType(table_file.engine_type_)) { utils::DeleteSegment(options_, table_file); std::string segment_dir; utils::GetParentPath(table_file.location_, segment_dir); diff --git a/core/src/db/meta/SqliteMetaImpl.cpp b/core/src/db/meta/SqliteMetaImpl.cpp index 748cbb4944..ca37f2c258 100644 --- a/core/src/db/meta/SqliteMetaImpl.cpp +++ b/core/src/db/meta/SqliteMetaImpl.cpp @@ -68,7 +68,8 @@ StoragePrototype(const std::string& path) { make_column("created_on", &TableSchema::created_on_), make_column("flag", &TableSchema::flag_, default_value(0)), make_column("index_file_size", &TableSchema::index_file_size_), - make_column("engine_type", &TableSchema::engine_type_), make_column("nlist", &TableSchema::nlist_), + make_column("engine_type", &TableSchema::engine_type_), + make_column("index_params", &TableSchema::index_params_), make_column("metric_type", &TableSchema::metric_type_), make_column("owner_table", &TableSchema::owner_table_, default_value("")), make_column("partition_tag", &TableSchema::partition_tag_, default_value("")), @@ -213,7 +214,7 @@ SqliteMetaImpl::DescribeTable(TableSchema& table_schema) { auto groups = ConnectorPtr->select( columns(&TableSchema::id_, &TableSchema::state_, &TableSchema::dimension_, &TableSchema::created_on_, &TableSchema::flag_, &TableSchema::index_file_size_, &TableSchema::engine_type_, - &TableSchema::nlist_, &TableSchema::metric_type_, &TableSchema::owner_table_, + &TableSchema::index_params_, &TableSchema::metric_type_, &TableSchema::owner_table_, &TableSchema::partition_tag_, &TableSchema::version_, &TableSchema::flush_lsn_), where(c(&TableSchema::table_id_) == table_schema.table_id_ and c(&TableSchema::state_) != (int)TableSchema::TO_DELETE)); @@ -226,7 +227,7 @@ SqliteMetaImpl::DescribeTable(TableSchema& table_schema) { table_schema.flag_ = std::get<4>(groups[0]); table_schema.index_file_size_ = std::get<5>(groups[0]); table_schema.engine_type_ = std::get<6>(groups[0]); - table_schema.nlist_ = std::get<7>(groups[0]); + table_schema.index_params_ = std::get<7>(groups[0]); table_schema.metric_type_ = std::get<8>(groups[0]); table_schema.owner_table_ = std::get<9>(groups[0]); table_schema.partition_tag_ = std::get<10>(groups[0]); @@ -272,7 +273,7 @@ SqliteMetaImpl::AllTables(std::vector& table_schema_array) { auto selected = ConnectorPtr->select( columns(&TableSchema::id_, &TableSchema::table_id_, &TableSchema::dimension_, &TableSchema::created_on_, &TableSchema::flag_, &TableSchema::index_file_size_, &TableSchema::engine_type_, - &TableSchema::nlist_, &TableSchema::metric_type_, &TableSchema::owner_table_, + &TableSchema::index_params_, &TableSchema::metric_type_, &TableSchema::owner_table_, &TableSchema::partition_tag_, &TableSchema::version_, &TableSchema::flush_lsn_), where(c(&TableSchema::state_) != (int)TableSchema::TO_DELETE and c(&TableSchema::owner_table_) == "")); for (auto& table : selected) { @@ -284,7 +285,7 @@ SqliteMetaImpl::AllTables(std::vector& table_schema_array) { schema.flag_ = std::get<4>(table); schema.index_file_size_ = std::get<5>(table); schema.engine_type_ = std::get<6>(table); - schema.nlist_ = std::get<7>(table); + schema.index_params_ = std::get<7>(table); schema.metric_type_ = std::get<8>(table); schema.owner_table_ = std::get<9>(table); schema.partition_tag_ = std::get<10>(table); @@ -373,17 +374,8 @@ SqliteMetaImpl::CreateTableFile(TableFileSchema& file_schema) { file_schema.created_on_ = utils::GetMicroSecTimeStamp(); file_schema.updated_time_ = file_schema.created_on_; file_schema.index_file_size_ = table_schema.index_file_size_; - - if (file_schema.file_type_ == TableFileSchema::FILE_TYPE::NEW || - file_schema.file_type_ == TableFileSchema::FILE_TYPE::NEW_MERGE) { - file_schema.engine_type_ = server::ValidationUtil::IsBinaryMetricType(table_schema.metric_type_) - ? (int32_t)EngineType::FAISS_BIN_IDMAP - : (int32_t)EngineType::FAISS_IDMAP; - } else { - file_schema.engine_type_ = table_schema.engine_type_; - } - - file_schema.nlist_ = table_schema.nlist_; + file_schema.index_params_ = table_schema.index_params_; + file_schema.engine_type_ = table_schema.engine_type_; file_schema.metric_type_ = table_schema.metric_type_; // multi-threads call sqlite update may get exception('bad logic', etc), so we add a lock here @@ -436,7 +428,7 @@ SqliteMetaImpl::GetTableFiles(const std::string& table_id, const std::vector(file); file_schema.dimension_ = table_schema.dimension_; file_schema.index_file_size_ = table_schema.index_file_size_; - file_schema.nlist_ = table_schema.nlist_; + file_schema.index_params_ = table_schema.index_params_; file_schema.metric_type_ = table_schema.metric_type_; utils::GetTableFilePath(options_, file_schema); @@ -486,7 +478,7 @@ SqliteMetaImpl::GetTableFilesBySegmentId(const std::string& segment_id, file_schema.created_on_ = std::get<9>(file); file_schema.dimension_ = table_schema.dimension_; file_schema.index_file_size_ = table_schema.index_file_size_; - file_schema.nlist_ = table_schema.nlist_; + file_schema.index_params_ = table_schema.index_params_; file_schema.metric_type_ = table_schema.metric_type_; utils::GetTableFilePath(options_, file_schema); @@ -601,7 +593,7 @@ SqliteMetaImpl::GetTableFilesByFlushLSN(uint64_t flush_lsn, TableFilesSchema& ta } table_file.dimension_ = groups[table_file.table_id_].dimension_; table_file.index_file_size_ = groups[table_file.table_id_].index_file_size_; - table_file.nlist_ = groups[table_file.table_id_].nlist_; + table_file.index_params_ = groups[table_file.table_id_].index_params_; table_file.metric_type_ = groups[table_file.table_id_].metric_type_; table_files.push_back(table_file); } @@ -721,7 +713,7 @@ SqliteMetaImpl::UpdateTableIndex(const std::string& table_id, const TableIndex& table_schema.partition_tag_ = std::get<7>(tables[0]); table_schema.version_ = std::get<8>(tables[0]); table_schema.engine_type_ = index.engine_type_; - table_schema.nlist_ = index.nlist_; + table_schema.index_params_ = index.extra_params_.dump(); table_schema.metric_type_ = index.metric_type_; ConnectorPtr->update(table_schema); @@ -773,12 +765,12 @@ SqliteMetaImpl::DescribeTableIndex(const std::string& table_id, TableIndex& inde fiu_do_on("SqliteMetaImpl.DescribeTableIndex.throw_exception", throw std::exception()); auto groups = ConnectorPtr->select( - columns(&TableSchema::engine_type_, &TableSchema::nlist_, &TableSchema::metric_type_), + columns(&TableSchema::engine_type_, &TableSchema::index_params_, &TableSchema::metric_type_), where(c(&TableSchema::table_id_) == table_id and c(&TableSchema::state_) != (int)TableSchema::TO_DELETE)); if (groups.size() == 1) { index.engine_type_ = std::get<0>(groups[0]); - index.nlist_ = std::get<1>(groups[0]); + index.extra_params_ = milvus::json::parse(std::get<1>(groups[0])); index.metric_type_ = std::get<2>(groups[0]); } else { return Status(DB_NOT_FOUND, "Table " + table_id + " not found"); @@ -813,7 +805,7 @@ SqliteMetaImpl::DropTableIndex(const std::string& table_id) { // set table index type to raw ConnectorPtr->update_all( - set(c(&TableSchema::engine_type_) = DEFAULT_ENGINE_TYPE, c(&TableSchema::nlist_) = DEFAULT_NLIST), + set(c(&TableSchema::engine_type_) = DEFAULT_ENGINE_TYPE, c(&TableSchema::index_params_) = "{}"), where(c(&TableSchema::table_id_) == table_id)); ENGINE_LOG_DEBUG << "Successfully drop table index, table id = " << table_id; @@ -886,13 +878,14 @@ SqliteMetaImpl::ShowPartitions(const std::string& table_id, std::vectorselect(columns(&TableSchema::id_, &TableSchema::state_, &TableSchema::dimension_, - &TableSchema::created_on_, &TableSchema::flag_, &TableSchema::index_file_size_, - &TableSchema::engine_type_, &TableSchema::nlist_, &TableSchema::metric_type_, - &TableSchema::partition_tag_, &TableSchema::version_, &TableSchema::table_id_), - where(c(&TableSchema::owner_table_) == table_id and - c(&TableSchema::state_) != (int)TableSchema::TO_DELETE)); + auto partitions = ConnectorPtr->select( + columns(&TableSchema::id_, &TableSchema::state_, &TableSchema::dimension_, &TableSchema::created_on_, + &TableSchema::flag_, &TableSchema::index_file_size_, &TableSchema::engine_type_, + &TableSchema::index_params_, &TableSchema::metric_type_, &TableSchema::partition_tag_, + &TableSchema::version_, &TableSchema::table_id_), + where(c(&TableSchema::owner_table_) == table_id and + c(&TableSchema::state_) != (int)TableSchema::TO_DELETE)); + for (size_t i = 0; i < partitions.size(); i++) { meta::TableSchema partition_schema; partition_schema.id_ = std::get<0>(partitions[i]); @@ -902,7 +895,7 @@ SqliteMetaImpl::ShowPartitions(const std::string& table_id, std::vector(partitions[i]); partition_schema.index_file_size_ = std::get<5>(partitions[i]); partition_schema.engine_type_ = std::get<6>(partitions[i]); - partition_schema.nlist_ = std::get<7>(partitions[i]); + partition_schema.index_params_ = std::get<7>(partitions[i]); partition_schema.metric_type_ = std::get<8>(partitions[i]); partition_schema.owner_table_ = table_id; partition_schema.partition_tag_ = std::get<9>(partitions[i]); @@ -995,7 +988,7 @@ SqliteMetaImpl::FilesToSearch(const std::string& table_id, const std::vector(file); table_file.dimension_ = table_schema.dimension_; table_file.index_file_size_ = table_schema.index_file_size_; - table_file.nlist_ = table_schema.nlist_; + table_file.index_params_ = table_schema.index_params_; table_file.metric_type_ = table_schema.metric_type_; auto status = utils::GetTableFilePath(options_, table_file); @@ -1063,7 +1056,7 @@ SqliteMetaImpl::FilesToMerge(const std::string& table_id, TableFilesSchema& file table_file.created_on_ = std::get<8>(file); table_file.dimension_ = table_schema.dimension_; table_file.index_file_size_ = table_schema.index_file_size_; - table_file.nlist_ = table_schema.nlist_; + table_file.index_params_ = table_schema.index_params_; table_file.metric_type_ = table_schema.metric_type_; auto status = utils::GetTableFilePath(options_, table_file); @@ -1134,7 +1127,7 @@ SqliteMetaImpl::FilesToIndex(TableFilesSchema& files) { } table_file.dimension_ = groups[table_file.table_id_].dimension_; table_file.index_file_size_ = groups[table_file.table_id_].index_file_size_; - table_file.nlist_ = groups[table_file.table_id_].nlist_; + table_file.index_params_ = groups[table_file.table_id_].index_params_; table_file.metric_type_ = groups[table_file.table_id_].metric_type_; files.push_back(table_file); } @@ -1192,7 +1185,7 @@ SqliteMetaImpl::FilesByType(const std::string& table_id, const std::vector& file_schema.dimension_ = table_schema.dimension_; file_schema.index_file_size_ = table_schema.index_file_size_; - file_schema.nlist_ = table_schema.nlist_; + file_schema.index_params_ = table_schema.index_params_; file_schema.metric_type_ = table_schema.metric_type_; switch (file_schema.file_type_) { @@ -1423,8 +1416,7 @@ SqliteMetaImpl::CleanUpFilesWithTTL(uint64_t seconds /*, CleanUpFilter* filter*/ // If we are deleting a raw table file, it means it's okay to delete the entire segment directory. // Else, we can only delete the single file // TODO(zhiru): We determine whether a table file is raw by its engine type. This is a bit hacky - if (table_file.engine_type_ == (int32_t)EngineType::FAISS_IDMAP || - table_file.engine_type_ == (int32_t)EngineType::FAISS_BIN_IDMAP) { + if (utils::IsRawIndexType(table_file.engine_type_)) { utils::DeleteSegment(options_, table_file); std::string segment_dir; utils::GetParentPath(table_file.location_, segment_dir); diff --git a/core/src/grpc/gen-milvus/milvus.pb.cc b/core/src/grpc/gen-milvus/milvus.pb.cc index 13ebbffb8a..3886e57da1 100644 --- a/core/src/grpc/gen-milvus/milvus.pb.cc +++ b/core/src/grpc/gen-milvus/milvus.pb.cc @@ -15,14 +15,18 @@ #include // @@protoc_insertion_point(includes) #include -extern PROTOBUF_INTERNAL_EXPORT_milvus_2eproto ::PROTOBUF_NAMESPACE_ID::internal::SCCInfo<0> scc_info_Index_milvus_2eproto; +extern PROTOBUF_INTERNAL_EXPORT_milvus_2eproto ::PROTOBUF_NAMESPACE_ID::internal::SCCInfo<0> scc_info_KeyValuePair_milvus_2eproto; extern PROTOBUF_INTERNAL_EXPORT_milvus_2eproto ::PROTOBUF_NAMESPACE_ID::internal::SCCInfo<1> scc_info_PartitionStat_milvus_2eproto; extern PROTOBUF_INTERNAL_EXPORT_milvus_2eproto ::PROTOBUF_NAMESPACE_ID::internal::SCCInfo<0> scc_info_RowRecord_milvus_2eproto; -extern PROTOBUF_INTERNAL_EXPORT_milvus_2eproto ::PROTOBUF_NAMESPACE_ID::internal::SCCInfo<1> scc_info_SearchParam_milvus_2eproto; +extern PROTOBUF_INTERNAL_EXPORT_milvus_2eproto ::PROTOBUF_NAMESPACE_ID::internal::SCCInfo<2> scc_info_SearchParam_milvus_2eproto; extern PROTOBUF_INTERNAL_EXPORT_milvus_2eproto ::PROTOBUF_NAMESPACE_ID::internal::SCCInfo<0> scc_info_SegmentStat_milvus_2eproto; extern PROTOBUF_INTERNAL_EXPORT_status_2eproto ::PROTOBUF_NAMESPACE_ID::internal::SCCInfo<0> scc_info_Status_status_2eproto; namespace milvus { namespace grpc { +class KeyValuePairDefaultTypeInternal { + public: + ::PROTOBUF_NAMESPACE_ID::internal::ExplicitlyConstructed _instance; +} _KeyValuePair_default_instance_; class TableNameDefaultTypeInternal { public: ::PROTOBUF_NAMESPACE_ID::internal::ExplicitlyConstructed _instance; @@ -87,10 +91,6 @@ class CommandDefaultTypeInternal { public: ::PROTOBUF_NAMESPACE_ID::internal::ExplicitlyConstructed _instance; } _Command_default_instance_; -class IndexDefaultTypeInternal { - public: - ::PROTOBUF_NAMESPACE_ID::internal::ExplicitlyConstructed _instance; -} _Index_default_instance_; class IndexParamDefaultTypeInternal { public: ::PROTOBUF_NAMESPACE_ID::internal::ExplicitlyConstructed _instance; @@ -200,20 +200,6 @@ static void InitDefaultsscc_info_GetVectorIDsParam_milvus_2eproto() { ::PROTOBUF_NAMESPACE_ID::internal::SCCInfo<0> scc_info_GetVectorIDsParam_milvus_2eproto = {{ATOMIC_VAR_INIT(::PROTOBUF_NAMESPACE_ID::internal::SCCInfoBase::kUninitialized), 0, InitDefaultsscc_info_GetVectorIDsParam_milvus_2eproto}, {}}; -static void InitDefaultsscc_info_Index_milvus_2eproto() { - GOOGLE_PROTOBUF_VERIFY_VERSION; - - { - void* ptr = &::milvus::grpc::_Index_default_instance_; - new (ptr) ::milvus::grpc::Index(); - ::PROTOBUF_NAMESPACE_ID::internal::OnShutdownDestroyMessage(ptr); - } - ::milvus::grpc::Index::InitAsDefaultInstance(); -} - -::PROTOBUF_NAMESPACE_ID::internal::SCCInfo<0> scc_info_Index_milvus_2eproto = - {{ATOMIC_VAR_INIT(::PROTOBUF_NAMESPACE_ID::internal::SCCInfoBase::kUninitialized), 0, InitDefaultsscc_info_Index_milvus_2eproto}, {}}; - static void InitDefaultsscc_info_IndexParam_milvus_2eproto() { GOOGLE_PROTOBUF_VERIFY_VERSION; @@ -228,7 +214,7 @@ static void InitDefaultsscc_info_IndexParam_milvus_2eproto() { ::PROTOBUF_NAMESPACE_ID::internal::SCCInfo<2> scc_info_IndexParam_milvus_2eproto = {{ATOMIC_VAR_INIT(::PROTOBUF_NAMESPACE_ID::internal::SCCInfoBase::kUninitialized), 2, InitDefaultsscc_info_IndexParam_milvus_2eproto}, { &scc_info_Status_status_2eproto.base, - &scc_info_Index_milvus_2eproto.base,}}; + &scc_info_KeyValuePair_milvus_2eproto.base,}}; static void InitDefaultsscc_info_InsertParam_milvus_2eproto() { GOOGLE_PROTOBUF_VERIFY_VERSION; @@ -241,9 +227,24 @@ static void InitDefaultsscc_info_InsertParam_milvus_2eproto() { ::milvus::grpc::InsertParam::InitAsDefaultInstance(); } -::PROTOBUF_NAMESPACE_ID::internal::SCCInfo<1> scc_info_InsertParam_milvus_2eproto = - {{ATOMIC_VAR_INIT(::PROTOBUF_NAMESPACE_ID::internal::SCCInfoBase::kUninitialized), 1, InitDefaultsscc_info_InsertParam_milvus_2eproto}, { - &scc_info_RowRecord_milvus_2eproto.base,}}; +::PROTOBUF_NAMESPACE_ID::internal::SCCInfo<2> scc_info_InsertParam_milvus_2eproto = + {{ATOMIC_VAR_INIT(::PROTOBUF_NAMESPACE_ID::internal::SCCInfoBase::kUninitialized), 2, InitDefaultsscc_info_InsertParam_milvus_2eproto}, { + &scc_info_RowRecord_milvus_2eproto.base, + &scc_info_KeyValuePair_milvus_2eproto.base,}}; + +static void InitDefaultsscc_info_KeyValuePair_milvus_2eproto() { + GOOGLE_PROTOBUF_VERIFY_VERSION; + + { + void* ptr = &::milvus::grpc::_KeyValuePair_default_instance_; + new (ptr) ::milvus::grpc::KeyValuePair(); + ::PROTOBUF_NAMESPACE_ID::internal::OnShutdownDestroyMessage(ptr); + } + ::milvus::grpc::KeyValuePair::InitAsDefaultInstance(); +} + +::PROTOBUF_NAMESPACE_ID::internal::SCCInfo<0> scc_info_KeyValuePair_milvus_2eproto = + {{ATOMIC_VAR_INIT(::PROTOBUF_NAMESPACE_ID::internal::SCCInfoBase::kUninitialized), 0, InitDefaultsscc_info_KeyValuePair_milvus_2eproto}, {}}; static void InitDefaultsscc_info_PartitionList_milvus_2eproto() { GOOGLE_PROTOBUF_VERIFY_VERSION; @@ -314,8 +315,9 @@ static void InitDefaultsscc_info_SearchByIDParam_milvus_2eproto() { ::milvus::grpc::SearchByIDParam::InitAsDefaultInstance(); } -::PROTOBUF_NAMESPACE_ID::internal::SCCInfo<0> scc_info_SearchByIDParam_milvus_2eproto = - {{ATOMIC_VAR_INIT(::PROTOBUF_NAMESPACE_ID::internal::SCCInfoBase::kUninitialized), 0, InitDefaultsscc_info_SearchByIDParam_milvus_2eproto}, {}}; +::PROTOBUF_NAMESPACE_ID::internal::SCCInfo<1> scc_info_SearchByIDParam_milvus_2eproto = + {{ATOMIC_VAR_INIT(::PROTOBUF_NAMESPACE_ID::internal::SCCInfoBase::kUninitialized), 1, InitDefaultsscc_info_SearchByIDParam_milvus_2eproto}, { + &scc_info_KeyValuePair_milvus_2eproto.base,}}; static void InitDefaultsscc_info_SearchInFilesParam_milvus_2eproto() { GOOGLE_PROTOBUF_VERIFY_VERSION; @@ -343,9 +345,10 @@ static void InitDefaultsscc_info_SearchParam_milvus_2eproto() { ::milvus::grpc::SearchParam::InitAsDefaultInstance(); } -::PROTOBUF_NAMESPACE_ID::internal::SCCInfo<1> scc_info_SearchParam_milvus_2eproto = - {{ATOMIC_VAR_INIT(::PROTOBUF_NAMESPACE_ID::internal::SCCInfoBase::kUninitialized), 1, InitDefaultsscc_info_SearchParam_milvus_2eproto}, { - &scc_info_RowRecord_milvus_2eproto.base,}}; +::PROTOBUF_NAMESPACE_ID::internal::SCCInfo<2> scc_info_SearchParam_milvus_2eproto = + {{ATOMIC_VAR_INIT(::PROTOBUF_NAMESPACE_ID::internal::SCCInfoBase::kUninitialized), 2, InitDefaultsscc_info_SearchParam_milvus_2eproto}, { + &scc_info_RowRecord_milvus_2eproto.base, + &scc_info_KeyValuePair_milvus_2eproto.base,}}; static void InitDefaultsscc_info_SegmentStat_milvus_2eproto() { GOOGLE_PROTOBUF_VERIFY_VERSION; @@ -447,9 +450,10 @@ static void InitDefaultsscc_info_TableSchema_milvus_2eproto() { ::milvus::grpc::TableSchema::InitAsDefaultInstance(); } -::PROTOBUF_NAMESPACE_ID::internal::SCCInfo<1> scc_info_TableSchema_milvus_2eproto = - {{ATOMIC_VAR_INIT(::PROTOBUF_NAMESPACE_ID::internal::SCCInfoBase::kUninitialized), 1, InitDefaultsscc_info_TableSchema_milvus_2eproto}, { - &scc_info_Status_status_2eproto.base,}}; +::PROTOBUF_NAMESPACE_ID::internal::SCCInfo<2> scc_info_TableSchema_milvus_2eproto = + {{ATOMIC_VAR_INIT(::PROTOBUF_NAMESPACE_ID::internal::SCCInfoBase::kUninitialized), 2, InitDefaultsscc_info_TableSchema_milvus_2eproto}, { + &scc_info_Status_status_2eproto.base, + &scc_info_KeyValuePair_milvus_2eproto.base,}}; static void InitDefaultsscc_info_TopKQueryResult_milvus_2eproto() { GOOGLE_PROTOBUF_VERIFY_VERSION; @@ -516,6 +520,13 @@ static constexpr ::PROTOBUF_NAMESPACE_ID::EnumDescriptor const** file_level_enum static constexpr ::PROTOBUF_NAMESPACE_ID::ServiceDescriptor const** file_level_service_descriptors_milvus_2eproto = nullptr; const ::PROTOBUF_NAMESPACE_ID::uint32 TableStruct_milvus_2eproto::offsets[] PROTOBUF_SECTION_VARIABLE(protodesc_cold) = { + ~0u, // no _has_bits_ + PROTOBUF_FIELD_OFFSET(::milvus::grpc::KeyValuePair, _internal_metadata_), + ~0u, // no _extensions_ + ~0u, // no _oneof_case_ + ~0u, // no _weak_field_map_ + PROTOBUF_FIELD_OFFSET(::milvus::grpc::KeyValuePair, key_), + PROTOBUF_FIELD_OFFSET(::milvus::grpc::KeyValuePair, value_), ~0u, // no _has_bits_ PROTOBUF_FIELD_OFFSET(::milvus::grpc::TableName, _internal_metadata_), ~0u, // no _extensions_ @@ -539,6 +550,7 @@ const ::PROTOBUF_NAMESPACE_ID::uint32 TableStruct_milvus_2eproto::offsets[] PROT PROTOBUF_FIELD_OFFSET(::milvus::grpc::TableSchema, dimension_), PROTOBUF_FIELD_OFFSET(::milvus::grpc::TableSchema, index_file_size_), PROTOBUF_FIELD_OFFSET(::milvus::grpc::TableSchema, metric_type_), + PROTOBUF_FIELD_OFFSET(::milvus::grpc::TableSchema, extra_params_), ~0u, // no _has_bits_ PROTOBUF_FIELD_OFFSET(::milvus::grpc::PartitionParam, _internal_metadata_), ~0u, // no _extensions_ @@ -569,6 +581,7 @@ const ::PROTOBUF_NAMESPACE_ID::uint32 TableStruct_milvus_2eproto::offsets[] PROT PROTOBUF_FIELD_OFFSET(::milvus::grpc::InsertParam, row_record_array_), PROTOBUF_FIELD_OFFSET(::milvus::grpc::InsertParam, row_id_array_), PROTOBUF_FIELD_OFFSET(::milvus::grpc::InsertParam, partition_tag_), + PROTOBUF_FIELD_OFFSET(::milvus::grpc::InsertParam, extra_params_), ~0u, // no _has_bits_ PROTOBUF_FIELD_OFFSET(::milvus::grpc::VectorIds, _internal_metadata_), ~0u, // no _extensions_ @@ -582,10 +595,10 @@ const ::PROTOBUF_NAMESPACE_ID::uint32 TableStruct_milvus_2eproto::offsets[] PROT ~0u, // no _oneof_case_ ~0u, // no _weak_field_map_ PROTOBUF_FIELD_OFFSET(::milvus::grpc::SearchParam, table_name_), + PROTOBUF_FIELD_OFFSET(::milvus::grpc::SearchParam, partition_tag_array_), PROTOBUF_FIELD_OFFSET(::milvus::grpc::SearchParam, query_record_array_), PROTOBUF_FIELD_OFFSET(::milvus::grpc::SearchParam, topk_), - PROTOBUF_FIELD_OFFSET(::milvus::grpc::SearchParam, nprobe_), - PROTOBUF_FIELD_OFFSET(::milvus::grpc::SearchParam, partition_tag_array_), + PROTOBUF_FIELD_OFFSET(::milvus::grpc::SearchParam, extra_params_), ~0u, // no _has_bits_ PROTOBUF_FIELD_OFFSET(::milvus::grpc::SearchInFilesParam, _internal_metadata_), ~0u, // no _extensions_ @@ -599,10 +612,10 @@ const ::PROTOBUF_NAMESPACE_ID::uint32 TableStruct_milvus_2eproto::offsets[] PROT ~0u, // no _oneof_case_ ~0u, // no _weak_field_map_ PROTOBUF_FIELD_OFFSET(::milvus::grpc::SearchByIDParam, table_name_), + PROTOBUF_FIELD_OFFSET(::milvus::grpc::SearchByIDParam, partition_tag_array_), PROTOBUF_FIELD_OFFSET(::milvus::grpc::SearchByIDParam, id_), PROTOBUF_FIELD_OFFSET(::milvus::grpc::SearchByIDParam, topk_), - PROTOBUF_FIELD_OFFSET(::milvus::grpc::SearchByIDParam, nprobe_), - PROTOBUF_FIELD_OFFSET(::milvus::grpc::SearchByIDParam, partition_tag_array_), + PROTOBUF_FIELD_OFFSET(::milvus::grpc::SearchByIDParam, extra_params_), ~0u, // no _has_bits_ PROTOBUF_FIELD_OFFSET(::milvus::grpc::TopKQueryResult, _internal_metadata_), ~0u, // no _extensions_ @@ -640,20 +653,14 @@ const ::PROTOBUF_NAMESPACE_ID::uint32 TableStruct_milvus_2eproto::offsets[] PROT ~0u, // no _weak_field_map_ PROTOBUF_FIELD_OFFSET(::milvus::grpc::Command, cmd_), ~0u, // no _has_bits_ - PROTOBUF_FIELD_OFFSET(::milvus::grpc::Index, _internal_metadata_), - ~0u, // no _extensions_ - ~0u, // no _oneof_case_ - ~0u, // no _weak_field_map_ - PROTOBUF_FIELD_OFFSET(::milvus::grpc::Index, index_type_), - PROTOBUF_FIELD_OFFSET(::milvus::grpc::Index, nlist_), - ~0u, // no _has_bits_ PROTOBUF_FIELD_OFFSET(::milvus::grpc::IndexParam, _internal_metadata_), ~0u, // no _extensions_ ~0u, // no _oneof_case_ ~0u, // no _weak_field_map_ PROTOBUF_FIELD_OFFSET(::milvus::grpc::IndexParam, status_), PROTOBUF_FIELD_OFFSET(::milvus::grpc::IndexParam, table_name_), - PROTOBUF_FIELD_OFFSET(::milvus::grpc::IndexParam, index_), + PROTOBUF_FIELD_OFFSET(::milvus::grpc::IndexParam, index_type_), + PROTOBUF_FIELD_OFFSET(::milvus::grpc::IndexParam, extra_params_), ~0u, // no _has_bits_ PROTOBUF_FIELD_OFFSET(::milvus::grpc::FlushParam, _internal_metadata_), ~0u, // no _extensions_ @@ -715,35 +722,36 @@ const ::PROTOBUF_NAMESPACE_ID::uint32 TableStruct_milvus_2eproto::offsets[] PROT PROTOBUF_FIELD_OFFSET(::milvus::grpc::GetVectorIDsParam, segment_name_), }; static const ::PROTOBUF_NAMESPACE_ID::internal::MigrationSchema schemas[] PROTOBUF_SECTION_VARIABLE(protodesc_cold) = { - { 0, -1, sizeof(::milvus::grpc::TableName)}, - { 6, -1, sizeof(::milvus::grpc::TableNameList)}, - { 13, -1, sizeof(::milvus::grpc::TableSchema)}, - { 23, -1, sizeof(::milvus::grpc::PartitionParam)}, - { 30, -1, sizeof(::milvus::grpc::PartitionList)}, - { 37, -1, sizeof(::milvus::grpc::RowRecord)}, - { 44, -1, sizeof(::milvus::grpc::InsertParam)}, - { 53, -1, sizeof(::milvus::grpc::VectorIds)}, - { 60, -1, sizeof(::milvus::grpc::SearchParam)}, - { 70, -1, sizeof(::milvus::grpc::SearchInFilesParam)}, - { 77, -1, sizeof(::milvus::grpc::SearchByIDParam)}, - { 87, -1, sizeof(::milvus::grpc::TopKQueryResult)}, - { 96, -1, sizeof(::milvus::grpc::StringReply)}, - { 103, -1, sizeof(::milvus::grpc::BoolReply)}, - { 110, -1, sizeof(::milvus::grpc::TableRowCount)}, - { 117, -1, sizeof(::milvus::grpc::Command)}, - { 123, -1, sizeof(::milvus::grpc::Index)}, - { 130, -1, sizeof(::milvus::grpc::IndexParam)}, - { 138, -1, sizeof(::milvus::grpc::FlushParam)}, - { 144, -1, sizeof(::milvus::grpc::DeleteByIDParam)}, - { 151, -1, sizeof(::milvus::grpc::SegmentStat)}, - { 160, -1, sizeof(::milvus::grpc::PartitionStat)}, - { 168, -1, sizeof(::milvus::grpc::TableInfo)}, - { 176, -1, sizeof(::milvus::grpc::VectorIdentity)}, - { 183, -1, sizeof(::milvus::grpc::VectorData)}, - { 190, -1, sizeof(::milvus::grpc::GetVectorIDsParam)}, + { 0, -1, sizeof(::milvus::grpc::KeyValuePair)}, + { 7, -1, sizeof(::milvus::grpc::TableName)}, + { 13, -1, sizeof(::milvus::grpc::TableNameList)}, + { 20, -1, sizeof(::milvus::grpc::TableSchema)}, + { 31, -1, sizeof(::milvus::grpc::PartitionParam)}, + { 38, -1, sizeof(::milvus::grpc::PartitionList)}, + { 45, -1, sizeof(::milvus::grpc::RowRecord)}, + { 52, -1, sizeof(::milvus::grpc::InsertParam)}, + { 62, -1, sizeof(::milvus::grpc::VectorIds)}, + { 69, -1, sizeof(::milvus::grpc::SearchParam)}, + { 79, -1, sizeof(::milvus::grpc::SearchInFilesParam)}, + { 86, -1, sizeof(::milvus::grpc::SearchByIDParam)}, + { 96, -1, sizeof(::milvus::grpc::TopKQueryResult)}, + { 105, -1, sizeof(::milvus::grpc::StringReply)}, + { 112, -1, sizeof(::milvus::grpc::BoolReply)}, + { 119, -1, sizeof(::milvus::grpc::TableRowCount)}, + { 126, -1, sizeof(::milvus::grpc::Command)}, + { 132, -1, sizeof(::milvus::grpc::IndexParam)}, + { 141, -1, sizeof(::milvus::grpc::FlushParam)}, + { 147, -1, sizeof(::milvus::grpc::DeleteByIDParam)}, + { 154, -1, sizeof(::milvus::grpc::SegmentStat)}, + { 163, -1, sizeof(::milvus::grpc::PartitionStat)}, + { 171, -1, sizeof(::milvus::grpc::TableInfo)}, + { 179, -1, sizeof(::milvus::grpc::VectorIdentity)}, + { 186, -1, sizeof(::milvus::grpc::VectorData)}, + { 193, -1, sizeof(::milvus::grpc::GetVectorIDsParam)}, }; static ::PROTOBUF_NAMESPACE_ID::Message const * const file_default_instances[] = { + reinterpret_cast(&::milvus::grpc::_KeyValuePair_default_instance_), reinterpret_cast(&::milvus::grpc::_TableName_default_instance_), reinterpret_cast(&::milvus::grpc::_TableNameList_default_instance_), reinterpret_cast(&::milvus::grpc::_TableSchema_default_instance_), @@ -760,7 +768,6 @@ static ::PROTOBUF_NAMESPACE_ID::Message const * const file_default_instances[] = reinterpret_cast(&::milvus::grpc::_BoolReply_default_instance_), reinterpret_cast(&::milvus::grpc::_TableRowCount_default_instance_), reinterpret_cast(&::milvus::grpc::_Command_default_instance_), - reinterpret_cast(&::milvus::grpc::_Index_default_instance_), reinterpret_cast(&::milvus::grpc::_IndexParam_default_instance_), reinterpret_cast(&::milvus::grpc::_FlushParam_default_instance_), reinterpret_cast(&::milvus::grpc::_DeleteByIDParam_default_instance_), @@ -774,44 +781,49 @@ static ::PROTOBUF_NAMESPACE_ID::Message const * const file_default_instances[] = const char descriptor_table_protodef_milvus_2eproto[] PROTOBUF_SECTION_VARIABLE(protodesc_cold) = "\n\014milvus.proto\022\013milvus.grpc\032\014status.prot" - "o\"\037\n\tTableName\022\022\n\ntable_name\030\001 \001(\t\"I\n\rTa" - "bleNameList\022#\n\006status\030\001 \001(\0132\023.milvus.grp" - "c.Status\022\023\n\013table_names\030\002 \003(\t\"\207\001\n\013TableS" - "chema\022#\n\006status\030\001 \001(\0132\023.milvus.grpc.Stat" - "us\022\022\n\ntable_name\030\002 \001(\t\022\021\n\tdimension\030\003 \001(" - "\003\022\027\n\017index_file_size\030\004 \001(\003\022\023\n\013metric_typ" - "e\030\005 \001(\005\"1\n\016PartitionParam\022\022\n\ntable_name\030" - "\001 \001(\t\022\013\n\003tag\030\002 \001(\t\"Q\n\rPartitionList\022#\n\006s" - "tatus\030\001 \001(\0132\023.milvus.grpc.Status\022\033\n\023part" - "ition_tag_array\030\002 \003(\t\"4\n\tRowRecord\022\022\n\nfl" - "oat_data\030\001 \003(\002\022\023\n\013binary_data\030\002 \001(\014\"\200\001\n\013" - "InsertParam\022\022\n\ntable_name\030\001 \001(\t\0220\n\020row_r" - "ecord_array\030\002 \003(\0132\026.milvus.grpc.RowRecor" - "d\022\024\n\014row_id_array\030\003 \003(\003\022\025\n\rpartition_tag" - "\030\004 \001(\t\"I\n\tVectorIds\022#\n\006status\030\001 \001(\0132\023.mi" - "lvus.grpc.Status\022\027\n\017vector_id_array\030\002 \003(" - "\003\"\220\001\n\013SearchParam\022\022\n\ntable_name\030\001 \001(\t\0222\n" - "\022query_record_array\030\002 \003(\0132\026.milvus.grpc." - "RowRecord\022\014\n\004topk\030\003 \001(\003\022\016\n\006nprobe\030\004 \001(\003\022" - "\033\n\023partition_tag_array\030\005 \003(\t\"[\n\022SearchIn" - "FilesParam\022\025\n\rfile_id_array\030\001 \003(\t\022.\n\014sea" - "rch_param\030\002 \001(\0132\030.milvus.grpc.SearchPara" - "m\"l\n\017SearchByIDParam\022\022\n\ntable_name\030\001 \001(\t" - "\022\n\n\002id\030\002 \001(\003\022\014\n\004topk\030\003 \001(\003\022\016\n\006nprobe\030\004 \001" - "(\003\022\033\n\023partition_tag_array\030\005 \003(\t\"g\n\017TopKQ" - "ueryResult\022#\n\006status\030\001 \001(\0132\023.milvus.grpc" - ".Status\022\017\n\007row_num\030\002 \001(\003\022\013\n\003ids\030\003 \003(\003\022\021\n" - "\tdistances\030\004 \003(\002\"H\n\013StringReply\022#\n\006statu" - "s\030\001 \001(\0132\023.milvus.grpc.Status\022\024\n\014string_r" - "eply\030\002 \001(\t\"D\n\tBoolReply\022#\n\006status\030\001 \001(\0132" - "\023.milvus.grpc.Status\022\022\n\nbool_reply\030\002 \001(\010" - "\"M\n\rTableRowCount\022#\n\006status\030\001 \001(\0132\023.milv" - "us.grpc.Status\022\027\n\017table_row_count\030\002 \001(\003\"" - "\026\n\007Command\022\013\n\003cmd\030\001 \001(\t\"*\n\005Index\022\022\n\ninde" - "x_type\030\001 \001(\005\022\r\n\005nlist\030\002 \001(\005\"h\n\nIndexPara" - "m\022#\n\006status\030\001 \001(\0132\023.milvus.grpc.Status\022\022" - "\n\ntable_name\030\002 \001(\t\022!\n\005index\030\003 \001(\0132\022.milv" - "us.grpc.Index\"&\n\nFlushParam\022\030\n\020table_nam" + "o\"*\n\014KeyValuePair\022\013\n\003key\030\001 \001(\t\022\r\n\005value\030" + "\002 \001(\t\"\037\n\tTableName\022\022\n\ntable_name\030\001 \001(\t\"I" + "\n\rTableNameList\022#\n\006status\030\001 \001(\0132\023.milvus" + ".grpc.Status\022\023\n\013table_names\030\002 \003(\t\"\270\001\n\013Ta" + "bleSchema\022#\n\006status\030\001 \001(\0132\023.milvus.grpc." + "Status\022\022\n\ntable_name\030\002 \001(\t\022\021\n\tdimension\030" + "\003 \001(\003\022\027\n\017index_file_size\030\004 \001(\003\022\023\n\013metric" + "_type\030\005 \001(\005\022/\n\014extra_params\030\006 \003(\0132\031.milv" + "us.grpc.KeyValuePair\"1\n\016PartitionParam\022\022" + "\n\ntable_name\030\001 \001(\t\022\013\n\003tag\030\002 \001(\t\"Q\n\rParti" + "tionList\022#\n\006status\030\001 \001(\0132\023.milvus.grpc.S" + "tatus\022\033\n\023partition_tag_array\030\002 \003(\t\"4\n\tRo" + "wRecord\022\022\n\nfloat_data\030\001 \003(\002\022\023\n\013binary_da" + "ta\030\002 \001(\014\"\261\001\n\013InsertParam\022\022\n\ntable_name\030\001" + " \001(\t\0220\n\020row_record_array\030\002 \003(\0132\026.milvus." + "grpc.RowRecord\022\024\n\014row_id_array\030\003 \003(\003\022\025\n\r" + "partition_tag\030\004 \001(\t\022/\n\014extra_params\030\005 \003(" + "\0132\031.milvus.grpc.KeyValuePair\"I\n\tVectorId" + "s\022#\n\006status\030\001 \001(\0132\023.milvus.grpc.Status\022\027" + "\n\017vector_id_array\030\002 \003(\003\"\261\001\n\013SearchParam\022" + "\022\n\ntable_name\030\001 \001(\t\022\033\n\023partition_tag_arr" + "ay\030\002 \003(\t\0222\n\022query_record_array\030\003 \003(\0132\026.m" + "ilvus.grpc.RowRecord\022\014\n\004topk\030\004 \001(\003\022/\n\014ex" + "tra_params\030\005 \003(\0132\031.milvus.grpc.KeyValueP" + "air\"[\n\022SearchInFilesParam\022\025\n\rfile_id_arr" + "ay\030\001 \003(\t\022.\n\014search_param\030\002 \001(\0132\030.milvus." + "grpc.SearchParam\"\215\001\n\017SearchByIDParam\022\022\n\n" + "table_name\030\001 \001(\t\022\033\n\023partition_tag_array\030" + "\002 \003(\t\022\n\n\002id\030\003 \001(\003\022\014\n\004topk\030\004 \001(\003\022/\n\014extra" + "_params\030\005 \003(\0132\031.milvus.grpc.KeyValuePair" + "\"g\n\017TopKQueryResult\022#\n\006status\030\001 \001(\0132\023.mi" + "lvus.grpc.Status\022\017\n\007row_num\030\002 \001(\003\022\013\n\003ids" + "\030\003 \003(\003\022\021\n\tdistances\030\004 \003(\002\"H\n\013StringReply" + "\022#\n\006status\030\001 \001(\0132\023.milvus.grpc.Status\022\024\n" + "\014string_reply\030\002 \001(\t\"D\n\tBoolReply\022#\n\006stat" + "us\030\001 \001(\0132\023.milvus.grpc.Status\022\022\n\nbool_re" + "ply\030\002 \001(\010\"M\n\rTableRowCount\022#\n\006status\030\001 \001" + "(\0132\023.milvus.grpc.Status\022\027\n\017table_row_cou" + "nt\030\002 \001(\003\"\026\n\007Command\022\013\n\003cmd\030\001 \001(\t\"\212\001\n\nInd" + "exParam\022#\n\006status\030\001 \001(\0132\023.milvus.grpc.St" + "atus\022\022\n\ntable_name\030\002 \001(\t\022\022\n\nindex_type\030\003" + " \001(\005\022/\n\014extra_params\030\004 \003(\0132\031.milvus.grpc" + ".KeyValuePair\"&\n\nFlushParam\022\030\n\020table_nam" "e_array\030\001 \003(\t\"7\n\017DeleteByIDParam\022\022\n\ntabl" "e_name\030\001 \001(\t\022\020\n\010id_array\030\002 \003(\003\"]\n\013Segmen" "tStat\022\024\n\014segment_name\030\001 \001(\t\022\021\n\trow_count" @@ -878,9 +890,9 @@ static ::PROTOBUF_NAMESPACE_ID::internal::SCCInfoBase*const descriptor_table_mil &scc_info_DeleteByIDParam_milvus_2eproto.base, &scc_info_FlushParam_milvus_2eproto.base, &scc_info_GetVectorIDsParam_milvus_2eproto.base, - &scc_info_Index_milvus_2eproto.base, &scc_info_IndexParam_milvus_2eproto.base, &scc_info_InsertParam_milvus_2eproto.base, + &scc_info_KeyValuePair_milvus_2eproto.base, &scc_info_PartitionList_milvus_2eproto.base, &scc_info_PartitionParam_milvus_2eproto.base, &scc_info_PartitionStat_milvus_2eproto.base, @@ -903,7 +915,7 @@ static ::PROTOBUF_NAMESPACE_ID::internal::SCCInfoBase*const descriptor_table_mil static ::PROTOBUF_NAMESPACE_ID::internal::once_flag descriptor_table_milvus_2eproto_once; static bool descriptor_table_milvus_2eproto_initialized = false; const ::PROTOBUF_NAMESPACE_ID::internal::DescriptorTable descriptor_table_milvus_2eproto = { - &descriptor_table_milvus_2eproto_initialized, descriptor_table_protodef_milvus_2eproto, "milvus.proto", 3788, + &descriptor_table_milvus_2eproto_initialized, descriptor_table_protodef_milvus_2eproto, "milvus.proto", 3988, &descriptor_table_milvus_2eproto_once, descriptor_table_milvus_2eproto_sccs, descriptor_table_milvus_2eproto_deps, 26, 1, schemas, file_default_instances, TableStruct_milvus_2eproto::offsets, file_level_metadata_milvus_2eproto, 26, file_level_enum_descriptors_milvus_2eproto, file_level_service_descriptors_milvus_2eproto, @@ -914,6 +926,338 @@ static bool dynamic_init_dummy_milvus_2eproto = ( ::PROTOBUF_NAMESPACE_ID::inte namespace milvus { namespace grpc { +// =================================================================== + +void KeyValuePair::InitAsDefaultInstance() { +} +class KeyValuePair::_Internal { + public: +}; + +KeyValuePair::KeyValuePair() + : ::PROTOBUF_NAMESPACE_ID::Message(), _internal_metadata_(nullptr) { + SharedCtor(); + // @@protoc_insertion_point(constructor:milvus.grpc.KeyValuePair) +} +KeyValuePair::KeyValuePair(const KeyValuePair& from) + : ::PROTOBUF_NAMESPACE_ID::Message(), + _internal_metadata_(nullptr) { + _internal_metadata_.MergeFrom(from._internal_metadata_); + key_.UnsafeSetDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); + if (!from.key().empty()) { + key_.AssignWithDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), from.key_); + } + value_.UnsafeSetDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); + if (!from.value().empty()) { + value_.AssignWithDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), from.value_); + } + // @@protoc_insertion_point(copy_constructor:milvus.grpc.KeyValuePair) +} + +void KeyValuePair::SharedCtor() { + ::PROTOBUF_NAMESPACE_ID::internal::InitSCC(&scc_info_KeyValuePair_milvus_2eproto.base); + key_.UnsafeSetDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); + value_.UnsafeSetDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); +} + +KeyValuePair::~KeyValuePair() { + // @@protoc_insertion_point(destructor:milvus.grpc.KeyValuePair) + SharedDtor(); +} + +void KeyValuePair::SharedDtor() { + key_.DestroyNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); + value_.DestroyNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); +} + +void KeyValuePair::SetCachedSize(int size) const { + _cached_size_.Set(size); +} +const KeyValuePair& KeyValuePair::default_instance() { + ::PROTOBUF_NAMESPACE_ID::internal::InitSCC(&::scc_info_KeyValuePair_milvus_2eproto.base); + return *internal_default_instance(); +} + + +void KeyValuePair::Clear() { +// @@protoc_insertion_point(message_clear_start:milvus.grpc.KeyValuePair) + ::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0; + // Prevent compiler warnings about cached_has_bits being unused + (void) cached_has_bits; + + key_.ClearToEmptyNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); + value_.ClearToEmptyNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); + _internal_metadata_.Clear(); +} + +#if GOOGLE_PROTOBUF_ENABLE_EXPERIMENTAL_PARSER +const char* KeyValuePair::_InternalParse(const char* ptr, ::PROTOBUF_NAMESPACE_ID::internal::ParseContext* ctx) { +#define CHK_(x) if (PROTOBUF_PREDICT_FALSE(!(x))) goto failure + while (!ctx->Done(&ptr)) { + ::PROTOBUF_NAMESPACE_ID::uint32 tag; + ptr = ::PROTOBUF_NAMESPACE_ID::internal::ReadTag(ptr, &tag); + CHK_(ptr); + switch (tag >> 3) { + // string key = 1; + case 1: + if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 10)) { + ptr = ::PROTOBUF_NAMESPACE_ID::internal::InlineGreedyStringParserUTF8(mutable_key(), ptr, ctx, "milvus.grpc.KeyValuePair.key"); + CHK_(ptr); + } else goto handle_unusual; + continue; + // string value = 2; + case 2: + if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 18)) { + ptr = ::PROTOBUF_NAMESPACE_ID::internal::InlineGreedyStringParserUTF8(mutable_value(), ptr, ctx, "milvus.grpc.KeyValuePair.value"); + CHK_(ptr); + } else goto handle_unusual; + continue; + default: { + handle_unusual: + if ((tag & 7) == 4 || tag == 0) { + ctx->SetLastTag(tag); + goto success; + } + ptr = UnknownFieldParse(tag, &_internal_metadata_, ptr, ctx); + CHK_(ptr != nullptr); + continue; + } + } // switch + } // while +success: + return ptr; +failure: + ptr = nullptr; + goto success; +#undef CHK_ +} +#else // GOOGLE_PROTOBUF_ENABLE_EXPERIMENTAL_PARSER +bool KeyValuePair::MergePartialFromCodedStream( + ::PROTOBUF_NAMESPACE_ID::io::CodedInputStream* input) { +#define DO_(EXPRESSION) if (!PROTOBUF_PREDICT_TRUE(EXPRESSION)) goto failure + ::PROTOBUF_NAMESPACE_ID::uint32 tag; + // @@protoc_insertion_point(parse_start:milvus.grpc.KeyValuePair) + for (;;) { + ::std::pair<::PROTOBUF_NAMESPACE_ID::uint32, bool> p = input->ReadTagWithCutoffNoLastTag(127u); + tag = p.first; + if (!p.second) goto handle_unusual; + switch (::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::GetTagFieldNumber(tag)) { + // string key = 1; + case 1: { + if (static_cast< ::PROTOBUF_NAMESPACE_ID::uint8>(tag) == (10 & 0xFF)) { + DO_(::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::ReadString( + input, this->mutable_key())); + DO_(::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::VerifyUtf8String( + this->key().data(), static_cast(this->key().length()), + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::PARSE, + "milvus.grpc.KeyValuePair.key")); + } else { + goto handle_unusual; + } + break; + } + + // string value = 2; + case 2: { + if (static_cast< ::PROTOBUF_NAMESPACE_ID::uint8>(tag) == (18 & 0xFF)) { + DO_(::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::ReadString( + input, this->mutable_value())); + DO_(::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::VerifyUtf8String( + this->value().data(), static_cast(this->value().length()), + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::PARSE, + "milvus.grpc.KeyValuePair.value")); + } else { + goto handle_unusual; + } + break; + } + + default: { + handle_unusual: + if (tag == 0) { + goto success; + } + DO_(::PROTOBUF_NAMESPACE_ID::internal::WireFormat::SkipField( + input, tag, _internal_metadata_.mutable_unknown_fields())); + break; + } + } + } +success: + // @@protoc_insertion_point(parse_success:milvus.grpc.KeyValuePair) + return true; +failure: + // @@protoc_insertion_point(parse_failure:milvus.grpc.KeyValuePair) + return false; +#undef DO_ +} +#endif // GOOGLE_PROTOBUF_ENABLE_EXPERIMENTAL_PARSER + +void KeyValuePair::SerializeWithCachedSizes( + ::PROTOBUF_NAMESPACE_ID::io::CodedOutputStream* output) const { + // @@protoc_insertion_point(serialize_start:milvus.grpc.KeyValuePair) + ::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0; + (void) cached_has_bits; + + // string key = 1; + if (this->key().size() > 0) { + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::VerifyUtf8String( + this->key().data(), static_cast(this->key().length()), + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::SERIALIZE, + "milvus.grpc.KeyValuePair.key"); + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteStringMaybeAliased( + 1, this->key(), output); + } + + // string value = 2; + if (this->value().size() > 0) { + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::VerifyUtf8String( + this->value().data(), static_cast(this->value().length()), + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::SERIALIZE, + "milvus.grpc.KeyValuePair.value"); + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteStringMaybeAliased( + 2, this->value(), output); + } + + if (_internal_metadata_.have_unknown_fields()) { + ::PROTOBUF_NAMESPACE_ID::internal::WireFormat::SerializeUnknownFields( + _internal_metadata_.unknown_fields(), output); + } + // @@protoc_insertion_point(serialize_end:milvus.grpc.KeyValuePair) +} + +::PROTOBUF_NAMESPACE_ID::uint8* KeyValuePair::InternalSerializeWithCachedSizesToArray( + ::PROTOBUF_NAMESPACE_ID::uint8* target) const { + // @@protoc_insertion_point(serialize_to_array_start:milvus.grpc.KeyValuePair) + ::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0; + (void) cached_has_bits; + + // string key = 1; + if (this->key().size() > 0) { + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::VerifyUtf8String( + this->key().data(), static_cast(this->key().length()), + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::SERIALIZE, + "milvus.grpc.KeyValuePair.key"); + target = + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteStringToArray( + 1, this->key(), target); + } + + // string value = 2; + if (this->value().size() > 0) { + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::VerifyUtf8String( + this->value().data(), static_cast(this->value().length()), + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::SERIALIZE, + "milvus.grpc.KeyValuePair.value"); + target = + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteStringToArray( + 2, this->value(), target); + } + + if (_internal_metadata_.have_unknown_fields()) { + target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormat::SerializeUnknownFieldsToArray( + _internal_metadata_.unknown_fields(), target); + } + // @@protoc_insertion_point(serialize_to_array_end:milvus.grpc.KeyValuePair) + return target; +} + +size_t KeyValuePair::ByteSizeLong() const { +// @@protoc_insertion_point(message_byte_size_start:milvus.grpc.KeyValuePair) + size_t total_size = 0; + + if (_internal_metadata_.have_unknown_fields()) { + total_size += + ::PROTOBUF_NAMESPACE_ID::internal::WireFormat::ComputeUnknownFieldsSize( + _internal_metadata_.unknown_fields()); + } + ::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0; + // Prevent compiler warnings about cached_has_bits being unused + (void) cached_has_bits; + + // string key = 1; + if (this->key().size() > 0) { + total_size += 1 + + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::StringSize( + this->key()); + } + + // string value = 2; + if (this->value().size() > 0) { + total_size += 1 + + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::StringSize( + this->value()); + } + + int cached_size = ::PROTOBUF_NAMESPACE_ID::internal::ToCachedSize(total_size); + SetCachedSize(cached_size); + return total_size; +} + +void KeyValuePair::MergeFrom(const ::PROTOBUF_NAMESPACE_ID::Message& from) { +// @@protoc_insertion_point(generalized_merge_from_start:milvus.grpc.KeyValuePair) + GOOGLE_DCHECK_NE(&from, this); + const KeyValuePair* source = + ::PROTOBUF_NAMESPACE_ID::DynamicCastToGenerated( + &from); + if (source == nullptr) { + // @@protoc_insertion_point(generalized_merge_from_cast_fail:milvus.grpc.KeyValuePair) + ::PROTOBUF_NAMESPACE_ID::internal::ReflectionOps::Merge(from, this); + } else { + // @@protoc_insertion_point(generalized_merge_from_cast_success:milvus.grpc.KeyValuePair) + MergeFrom(*source); + } +} + +void KeyValuePair::MergeFrom(const KeyValuePair& from) { +// @@protoc_insertion_point(class_specific_merge_from_start:milvus.grpc.KeyValuePair) + GOOGLE_DCHECK_NE(&from, this); + _internal_metadata_.MergeFrom(from._internal_metadata_); + ::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0; + (void) cached_has_bits; + + if (from.key().size() > 0) { + + key_.AssignWithDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), from.key_); + } + if (from.value().size() > 0) { + + value_.AssignWithDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), from.value_); + } +} + +void KeyValuePair::CopyFrom(const ::PROTOBUF_NAMESPACE_ID::Message& from) { +// @@protoc_insertion_point(generalized_copy_from_start:milvus.grpc.KeyValuePair) + if (&from == this) return; + Clear(); + MergeFrom(from); +} + +void KeyValuePair::CopyFrom(const KeyValuePair& from) { +// @@protoc_insertion_point(class_specific_copy_from_start:milvus.grpc.KeyValuePair) + if (&from == this) return; + Clear(); + MergeFrom(from); +} + +bool KeyValuePair::IsInitialized() const { + return true; +} + +void KeyValuePair::InternalSwap(KeyValuePair* other) { + using std::swap; + _internal_metadata_.Swap(&other->_internal_metadata_); + key_.Swap(&other->key_, &::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), + GetArenaNoVirtual()); + value_.Swap(&other->value_, &::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), + GetArenaNoVirtual()); +} + +::PROTOBUF_NAMESPACE_ID::Metadata KeyValuePair::GetMetadata() const { + return GetMetadataStatic(); +} + + // =================================================================== void TableName::InitAsDefaultInstance() { @@ -1543,7 +1887,8 @@ TableSchema::TableSchema() } TableSchema::TableSchema(const TableSchema& from) : ::PROTOBUF_NAMESPACE_ID::Message(), - _internal_metadata_(nullptr) { + _internal_metadata_(nullptr), + extra_params_(from.extra_params_) { _internal_metadata_.MergeFrom(from._internal_metadata_); table_name_.UnsafeSetDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); if (!from.table_name().empty()) { @@ -1593,6 +1938,7 @@ void TableSchema::Clear() { // Prevent compiler warnings about cached_has_bits being unused (void) cached_has_bits; + extra_params_.Clear(); table_name_.ClearToEmptyNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); if (GetArenaNoVirtual() == nullptr && status_ != nullptr) { delete status_; @@ -1647,6 +1993,18 @@ const char* TableSchema::_InternalParse(const char* ptr, ::PROTOBUF_NAMESPACE_ID CHK_(ptr); } else goto handle_unusual; continue; + // repeated .milvus.grpc.KeyValuePair extra_params = 6; + case 6: + if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 50)) { + ptr -= 1; + do { + ptr += 1; + ptr = ctx->ParseMessage(add_extra_params(), ptr); + CHK_(ptr); + if (!ctx->DataAvailable(ptr)) break; + } while (::PROTOBUF_NAMESPACE_ID::internal::UnalignedLoad<::PROTOBUF_NAMESPACE_ID::uint8>(ptr) == 50); + } else goto handle_unusual; + continue; default: { handle_unusual: if ((tag & 7) == 4 || tag == 0) { @@ -1742,6 +2100,17 @@ bool TableSchema::MergePartialFromCodedStream( break; } + // repeated .milvus.grpc.KeyValuePair extra_params = 6; + case 6: { + if (static_cast< ::PROTOBUF_NAMESPACE_ID::uint8>(tag) == (50 & 0xFF)) { + DO_(::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::ReadMessage( + input, add_extra_params())); + } else { + goto handle_unusual; + } + break; + } + default: { handle_unusual: if (tag == 0) { @@ -1800,6 +2169,15 @@ void TableSchema::SerializeWithCachedSizes( ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt32(5, this->metric_type(), output); } + // repeated .milvus.grpc.KeyValuePair extra_params = 6; + for (unsigned int i = 0, + n = static_cast(this->extra_params_size()); i < n; i++) { + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteMessageMaybeToArray( + 6, + this->extra_params(static_cast(i)), + output); + } + if (_internal_metadata_.have_unknown_fields()) { ::PROTOBUF_NAMESPACE_ID::internal::WireFormat::SerializeUnknownFields( _internal_metadata_.unknown_fields(), output); @@ -1846,6 +2224,14 @@ void TableSchema::SerializeWithCachedSizes( target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt32ToArray(5, this->metric_type(), target); } + // repeated .milvus.grpc.KeyValuePair extra_params = 6; + for (unsigned int i = 0, + n = static_cast(this->extra_params_size()); i < n; i++) { + target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite:: + InternalWriteMessageToArray( + 6, this->extra_params(static_cast(i)), target); + } + if (_internal_metadata_.have_unknown_fields()) { target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormat::SerializeUnknownFieldsToArray( _internal_metadata_.unknown_fields(), target); @@ -1867,6 +2253,17 @@ size_t TableSchema::ByteSizeLong() const { // Prevent compiler warnings about cached_has_bits being unused (void) cached_has_bits; + // repeated .milvus.grpc.KeyValuePair extra_params = 6; + { + unsigned int count = static_cast(this->extra_params_size()); + total_size += 1UL * count; + for (unsigned int i = 0; i < count; i++) { + total_size += + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::MessageSize( + this->extra_params(static_cast(i))); + } + } + // string table_name = 2; if (this->table_name().size() > 0) { total_size += 1 + @@ -1929,6 +2326,7 @@ void TableSchema::MergeFrom(const TableSchema& from) { ::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0; (void) cached_has_bits; + extra_params_.MergeFrom(from.extra_params_); if (from.table_name().size() > 0) { table_name_.AssignWithDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), from.table_name_); @@ -1968,6 +2366,7 @@ bool TableSchema::IsInitialized() const { void TableSchema::InternalSwap(TableSchema* other) { using std::swap; _internal_metadata_.Swap(&other->_internal_metadata_); + CastToBase(&extra_params_)->InternalSwap(CastToBase(&other->extra_params_)); table_name_.Swap(&other->table_name_, &::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), GetArenaNoVirtual()); swap(status_, other->status_); @@ -2986,7 +3385,8 @@ InsertParam::InsertParam(const InsertParam& from) : ::PROTOBUF_NAMESPACE_ID::Message(), _internal_metadata_(nullptr), row_record_array_(from.row_record_array_), - row_id_array_(from.row_id_array_) { + row_id_array_(from.row_id_array_), + extra_params_(from.extra_params_) { _internal_metadata_.MergeFrom(from._internal_metadata_); table_name_.UnsafeSetDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); if (!from.table_name().empty()) { @@ -3032,6 +3432,7 @@ void InsertParam::Clear() { row_record_array_.Clear(); row_id_array_.Clear(); + extra_params_.Clear(); table_name_.ClearToEmptyNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); partition_tag_.ClearToEmptyNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); _internal_metadata_.Clear(); @@ -3081,6 +3482,18 @@ const char* InsertParam::_InternalParse(const char* ptr, ::PROTOBUF_NAMESPACE_ID CHK_(ptr); } else goto handle_unusual; continue; + // repeated .milvus.grpc.KeyValuePair extra_params = 5; + case 5: + if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 42)) { + ptr -= 1; + do { + ptr += 1; + ptr = ctx->ParseMessage(add_extra_params(), ptr); + CHK_(ptr); + if (!ctx->DataAvailable(ptr)) break; + } while (::PROTOBUF_NAMESPACE_ID::internal::UnalignedLoad<::PROTOBUF_NAMESPACE_ID::uint8>(ptr) == 42); + } else goto handle_unusual; + continue; default: { handle_unusual: if ((tag & 7) == 4 || tag == 0) { @@ -3168,6 +3581,17 @@ bool InsertParam::MergePartialFromCodedStream( break; } + // repeated .milvus.grpc.KeyValuePair extra_params = 5; + case 5: { + if (static_cast< ::PROTOBUF_NAMESPACE_ID::uint8>(tag) == (42 & 0xFF)) { + DO_(::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::ReadMessage( + input, add_extra_params())); + } else { + goto handle_unusual; + } + break; + } + default: { handle_unusual: if (tag == 0) { @@ -3235,6 +3659,15 @@ void InsertParam::SerializeWithCachedSizes( 4, this->partition_tag(), output); } + // repeated .milvus.grpc.KeyValuePair extra_params = 5; + for (unsigned int i = 0, + n = static_cast(this->extra_params_size()); i < n; i++) { + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteMessageMaybeToArray( + 5, + this->extra_params(static_cast(i)), + output); + } + if (_internal_metadata_.have_unknown_fields()) { ::PROTOBUF_NAMESPACE_ID::internal::WireFormat::SerializeUnknownFields( _internal_metadata_.unknown_fields(), output); @@ -3291,6 +3724,14 @@ void InsertParam::SerializeWithCachedSizes( 4, this->partition_tag(), target); } + // repeated .milvus.grpc.KeyValuePair extra_params = 5; + for (unsigned int i = 0, + n = static_cast(this->extra_params_size()); i < n; i++) { + target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite:: + InternalWriteMessageToArray( + 5, this->extra_params(static_cast(i)), target); + } + if (_internal_metadata_.have_unknown_fields()) { target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormat::SerializeUnknownFieldsToArray( _internal_metadata_.unknown_fields(), target); @@ -3338,6 +3779,17 @@ size_t InsertParam::ByteSizeLong() const { total_size += data_size; } + // repeated .milvus.grpc.KeyValuePair extra_params = 5; + { + unsigned int count = static_cast(this->extra_params_size()); + total_size += 1UL * count; + for (unsigned int i = 0; i < count; i++) { + total_size += + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::MessageSize( + this->extra_params(static_cast(i))); + } + } + // string table_name = 1; if (this->table_name().size() > 0) { total_size += 1 + @@ -3381,6 +3833,7 @@ void InsertParam::MergeFrom(const InsertParam& from) { row_record_array_.MergeFrom(from.row_record_array_); row_id_array_.MergeFrom(from.row_id_array_); + extra_params_.MergeFrom(from.extra_params_); if (from.table_name().size() > 0) { table_name_.AssignWithDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), from.table_name_); @@ -3414,6 +3867,7 @@ void InsertParam::InternalSwap(InsertParam* other) { _internal_metadata_.Swap(&other->_internal_metadata_); CastToBase(&row_record_array_)->InternalSwap(CastToBase(&other->row_record_array_)); row_id_array_.InternalSwap(&other->row_id_array_); + CastToBase(&extra_params_)->InternalSwap(CastToBase(&other->extra_params_)); table_name_.Swap(&other->table_name_, &::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), GetArenaNoVirtual()); partition_tag_.Swap(&other->partition_tag_, &::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), @@ -3782,25 +4236,22 @@ SearchParam::SearchParam() SearchParam::SearchParam(const SearchParam& from) : ::PROTOBUF_NAMESPACE_ID::Message(), _internal_metadata_(nullptr), + partition_tag_array_(from.partition_tag_array_), query_record_array_(from.query_record_array_), - partition_tag_array_(from.partition_tag_array_) { + extra_params_(from.extra_params_) { _internal_metadata_.MergeFrom(from._internal_metadata_); table_name_.UnsafeSetDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); if (!from.table_name().empty()) { table_name_.AssignWithDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), from.table_name_); } - ::memcpy(&topk_, &from.topk_, - static_cast(reinterpret_cast(&nprobe_) - - reinterpret_cast(&topk_)) + sizeof(nprobe_)); + topk_ = from.topk_; // @@protoc_insertion_point(copy_constructor:milvus.grpc.SearchParam) } void SearchParam::SharedCtor() { ::PROTOBUF_NAMESPACE_ID::internal::InitSCC(&scc_info_SearchParam_milvus_2eproto.base); table_name_.UnsafeSetDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); - ::memset(&topk_, 0, static_cast( - reinterpret_cast(&nprobe_) - - reinterpret_cast(&topk_)) + sizeof(nprobe_)); + topk_ = PROTOBUF_LONGLONG(0); } SearchParam::~SearchParam() { @@ -3827,12 +4278,11 @@ void SearchParam::Clear() { // Prevent compiler warnings about cached_has_bits being unused (void) cached_has_bits; - query_record_array_.Clear(); partition_tag_array_.Clear(); + query_record_array_.Clear(); + extra_params_.Clear(); table_name_.ClearToEmptyNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); - ::memset(&topk_, 0, static_cast( - reinterpret_cast(&nprobe_) - - reinterpret_cast(&topk_)) + sizeof(nprobe_)); + topk_ = PROTOBUF_LONGLONG(0); _internal_metadata_.Clear(); } @@ -3851,39 +4301,44 @@ const char* SearchParam::_InternalParse(const char* ptr, ::PROTOBUF_NAMESPACE_ID CHK_(ptr); } else goto handle_unusual; continue; - // repeated .milvus.grpc.RowRecord query_record_array = 2; + // repeated string partition_tag_array = 2; case 2: if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 18)) { + ptr -= 1; + do { + ptr += 1; + ptr = ::PROTOBUF_NAMESPACE_ID::internal::InlineGreedyStringParserUTF8(add_partition_tag_array(), ptr, ctx, "milvus.grpc.SearchParam.partition_tag_array"); + CHK_(ptr); + if (!ctx->DataAvailable(ptr)) break; + } while (::PROTOBUF_NAMESPACE_ID::internal::UnalignedLoad<::PROTOBUF_NAMESPACE_ID::uint8>(ptr) == 18); + } else goto handle_unusual; + continue; + // repeated .milvus.grpc.RowRecord query_record_array = 3; + case 3: + if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 26)) { ptr -= 1; do { ptr += 1; ptr = ctx->ParseMessage(add_query_record_array(), ptr); CHK_(ptr); if (!ctx->DataAvailable(ptr)) break; - } while (::PROTOBUF_NAMESPACE_ID::internal::UnalignedLoad<::PROTOBUF_NAMESPACE_ID::uint8>(ptr) == 18); + } while (::PROTOBUF_NAMESPACE_ID::internal::UnalignedLoad<::PROTOBUF_NAMESPACE_ID::uint8>(ptr) == 26); } else goto handle_unusual; continue; - // int64 topk = 3; - case 3: - if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 24)) { + // int64 topk = 4; + case 4: + if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 32)) { topk_ = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint(&ptr); CHK_(ptr); } else goto handle_unusual; continue; - // int64 nprobe = 4; - case 4: - if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 32)) { - nprobe_ = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint(&ptr); - CHK_(ptr); - } else goto handle_unusual; - continue; - // repeated string partition_tag_array = 5; + // repeated .milvus.grpc.KeyValuePair extra_params = 5; case 5: if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 42)) { ptr -= 1; do { ptr += 1; - ptr = ::PROTOBUF_NAMESPACE_ID::internal::InlineGreedyStringParserUTF8(add_partition_tag_array(), ptr, ctx, "milvus.grpc.SearchParam.partition_tag_array"); + ptr = ctx->ParseMessage(add_extra_params(), ptr); CHK_(ptr); if (!ctx->DataAvailable(ptr)) break; } while (::PROTOBUF_NAMESPACE_ID::internal::UnalignedLoad<::PROTOBUF_NAMESPACE_ID::uint8>(ptr) == 42); @@ -3934,9 +4389,25 @@ bool SearchParam::MergePartialFromCodedStream( break; } - // repeated .milvus.grpc.RowRecord query_record_array = 2; + // repeated string partition_tag_array = 2; case 2: { if (static_cast< ::PROTOBUF_NAMESPACE_ID::uint8>(tag) == (18 & 0xFF)) { + DO_(::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::ReadString( + input, this->add_partition_tag_array())); + DO_(::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::VerifyUtf8String( + this->partition_tag_array(this->partition_tag_array_size() - 1).data(), + static_cast(this->partition_tag_array(this->partition_tag_array_size() - 1).length()), + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::PARSE, + "milvus.grpc.SearchParam.partition_tag_array")); + } else { + goto handle_unusual; + } + break; + } + + // repeated .milvus.grpc.RowRecord query_record_array = 3; + case 3: { + if (static_cast< ::PROTOBUF_NAMESPACE_ID::uint8>(tag) == (26 & 0xFF)) { DO_(::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::ReadMessage( input, add_query_record_array())); } else { @@ -3945,9 +4416,9 @@ bool SearchParam::MergePartialFromCodedStream( break; } - // int64 topk = 3; - case 3: { - if (static_cast< ::PROTOBUF_NAMESPACE_ID::uint8>(tag) == (24 & 0xFF)) { + // int64 topk = 4; + case 4: { + if (static_cast< ::PROTOBUF_NAMESPACE_ID::uint8>(tag) == (32 & 0xFF)) { DO_((::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::ReadPrimitive< ::PROTOBUF_NAMESPACE_ID::int64, ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::TYPE_INT64>( @@ -3958,29 +4429,11 @@ bool SearchParam::MergePartialFromCodedStream( break; } - // int64 nprobe = 4; - case 4: { - if (static_cast< ::PROTOBUF_NAMESPACE_ID::uint8>(tag) == (32 & 0xFF)) { - - DO_((::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::ReadPrimitive< - ::PROTOBUF_NAMESPACE_ID::int64, ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::TYPE_INT64>( - input, &nprobe_))); - } else { - goto handle_unusual; - } - break; - } - - // repeated string partition_tag_array = 5; + // repeated .milvus.grpc.KeyValuePair extra_params = 5; case 5: { if (static_cast< ::PROTOBUF_NAMESPACE_ID::uint8>(tag) == (42 & 0xFF)) { - DO_(::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::ReadString( - input, this->add_partition_tag_array())); - DO_(::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::VerifyUtf8String( - this->partition_tag_array(this->partition_tag_array_size() - 1).data(), - static_cast(this->partition_tag_array(this->partition_tag_array_size() - 1).length()), - ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::PARSE, - "milvus.grpc.SearchParam.partition_tag_array")); + DO_(::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::ReadMessage( + input, add_extra_params())); } else { goto handle_unusual; } @@ -4024,33 +4477,37 @@ void SearchParam::SerializeWithCachedSizes( 1, this->table_name(), output); } - // repeated .milvus.grpc.RowRecord query_record_array = 2; - for (unsigned int i = 0, - n = static_cast(this->query_record_array_size()); i < n; i++) { - ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteMessageMaybeToArray( - 2, - this->query_record_array(static_cast(i)), - output); - } - - // int64 topk = 3; - if (this->topk() != 0) { - ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt64(3, this->topk(), output); - } - - // int64 nprobe = 4; - if (this->nprobe() != 0) { - ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt64(4, this->nprobe(), output); - } - - // repeated string partition_tag_array = 5; + // repeated string partition_tag_array = 2; for (int i = 0, n = this->partition_tag_array_size(); i < n; i++) { ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::VerifyUtf8String( this->partition_tag_array(i).data(), static_cast(this->partition_tag_array(i).length()), ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::SERIALIZE, "milvus.grpc.SearchParam.partition_tag_array"); ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteString( - 5, this->partition_tag_array(i), output); + 2, this->partition_tag_array(i), output); + } + + // repeated .milvus.grpc.RowRecord query_record_array = 3; + for (unsigned int i = 0, + n = static_cast(this->query_record_array_size()); i < n; i++) { + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteMessageMaybeToArray( + 3, + this->query_record_array(static_cast(i)), + output); + } + + // int64 topk = 4; + if (this->topk() != 0) { + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt64(4, this->topk(), output); + } + + // repeated .milvus.grpc.KeyValuePair extra_params = 5; + for (unsigned int i = 0, + n = static_cast(this->extra_params_size()); i < n; i++) { + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteMessageMaybeToArray( + 5, + this->extra_params(static_cast(i)), + output); } if (_internal_metadata_.have_unknown_fields()) { @@ -4077,32 +4534,35 @@ void SearchParam::SerializeWithCachedSizes( 1, this->table_name(), target); } - // repeated .milvus.grpc.RowRecord query_record_array = 2; - for (unsigned int i = 0, - n = static_cast(this->query_record_array_size()); i < n; i++) { - target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite:: - InternalWriteMessageToArray( - 2, this->query_record_array(static_cast(i)), target); - } - - // int64 topk = 3; - if (this->topk() != 0) { - target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt64ToArray(3, this->topk(), target); - } - - // int64 nprobe = 4; - if (this->nprobe() != 0) { - target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt64ToArray(4, this->nprobe(), target); - } - - // repeated string partition_tag_array = 5; + // repeated string partition_tag_array = 2; for (int i = 0, n = this->partition_tag_array_size(); i < n; i++) { ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::VerifyUtf8String( this->partition_tag_array(i).data(), static_cast(this->partition_tag_array(i).length()), ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::SERIALIZE, "milvus.grpc.SearchParam.partition_tag_array"); target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite:: - WriteStringToArray(5, this->partition_tag_array(i), target); + WriteStringToArray(2, this->partition_tag_array(i), target); + } + + // repeated .milvus.grpc.RowRecord query_record_array = 3; + for (unsigned int i = 0, + n = static_cast(this->query_record_array_size()); i < n; i++) { + target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite:: + InternalWriteMessageToArray( + 3, this->query_record_array(static_cast(i)), target); + } + + // int64 topk = 4; + if (this->topk() != 0) { + target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt64ToArray(4, this->topk(), target); + } + + // repeated .milvus.grpc.KeyValuePair extra_params = 5; + for (unsigned int i = 0, + n = static_cast(this->extra_params_size()); i < n; i++) { + target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite:: + InternalWriteMessageToArray( + 5, this->extra_params(static_cast(i)), target); } if (_internal_metadata_.have_unknown_fields()) { @@ -4126,7 +4586,15 @@ size_t SearchParam::ByteSizeLong() const { // Prevent compiler warnings about cached_has_bits being unused (void) cached_has_bits; - // repeated .milvus.grpc.RowRecord query_record_array = 2; + // repeated string partition_tag_array = 2; + total_size += 1 * + ::PROTOBUF_NAMESPACE_ID::internal::FromIntSize(this->partition_tag_array_size()); + for (int i = 0, n = this->partition_tag_array_size(); i < n; i++) { + total_size += ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::StringSize( + this->partition_tag_array(i)); + } + + // repeated .milvus.grpc.RowRecord query_record_array = 3; { unsigned int count = static_cast(this->query_record_array_size()); total_size += 1UL * count; @@ -4137,12 +4605,15 @@ size_t SearchParam::ByteSizeLong() const { } } - // repeated string partition_tag_array = 5; - total_size += 1 * - ::PROTOBUF_NAMESPACE_ID::internal::FromIntSize(this->partition_tag_array_size()); - for (int i = 0, n = this->partition_tag_array_size(); i < n; i++) { - total_size += ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::StringSize( - this->partition_tag_array(i)); + // repeated .milvus.grpc.KeyValuePair extra_params = 5; + { + unsigned int count = static_cast(this->extra_params_size()); + total_size += 1UL * count; + for (unsigned int i = 0; i < count; i++) { + total_size += + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::MessageSize( + this->extra_params(static_cast(i))); + } } // string table_name = 1; @@ -4152,20 +4623,13 @@ size_t SearchParam::ByteSizeLong() const { this->table_name()); } - // int64 topk = 3; + // int64 topk = 4; if (this->topk() != 0) { total_size += 1 + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::Int64Size( this->topk()); } - // int64 nprobe = 4; - if (this->nprobe() != 0) { - total_size += 1 + - ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::Int64Size( - this->nprobe()); - } - int cached_size = ::PROTOBUF_NAMESPACE_ID::internal::ToCachedSize(total_size); SetCachedSize(cached_size); return total_size; @@ -4193,8 +4657,9 @@ void SearchParam::MergeFrom(const SearchParam& from) { ::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0; (void) cached_has_bits; - query_record_array_.MergeFrom(from.query_record_array_); partition_tag_array_.MergeFrom(from.partition_tag_array_); + query_record_array_.MergeFrom(from.query_record_array_); + extra_params_.MergeFrom(from.extra_params_); if (from.table_name().size() > 0) { table_name_.AssignWithDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), from.table_name_); @@ -4202,9 +4667,6 @@ void SearchParam::MergeFrom(const SearchParam& from) { if (from.topk() != 0) { set_topk(from.topk()); } - if (from.nprobe() != 0) { - set_nprobe(from.nprobe()); - } } void SearchParam::CopyFrom(const ::PROTOBUF_NAMESPACE_ID::Message& from) { @@ -4228,12 +4690,12 @@ bool SearchParam::IsInitialized() const { void SearchParam::InternalSwap(SearchParam* other) { using std::swap; _internal_metadata_.Swap(&other->_internal_metadata_); - CastToBase(&query_record_array_)->InternalSwap(CastToBase(&other->query_record_array_)); partition_tag_array_.InternalSwap(CastToBase(&other->partition_tag_array_)); + CastToBase(&query_record_array_)->InternalSwap(CastToBase(&other->query_record_array_)); + CastToBase(&extra_params_)->InternalSwap(CastToBase(&other->extra_params_)); table_name_.Swap(&other->table_name_, &::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), GetArenaNoVirtual()); swap(topk_, other->topk_); - swap(nprobe_, other->nprobe_); } ::PROTOBUF_NAMESPACE_ID::Metadata SearchParam::GetMetadata() const { @@ -4583,15 +5045,16 @@ SearchByIDParam::SearchByIDParam() SearchByIDParam::SearchByIDParam(const SearchByIDParam& from) : ::PROTOBUF_NAMESPACE_ID::Message(), _internal_metadata_(nullptr), - partition_tag_array_(from.partition_tag_array_) { + partition_tag_array_(from.partition_tag_array_), + extra_params_(from.extra_params_) { _internal_metadata_.MergeFrom(from._internal_metadata_); table_name_.UnsafeSetDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); if (!from.table_name().empty()) { table_name_.AssignWithDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), from.table_name_); } ::memcpy(&id_, &from.id_, - static_cast(reinterpret_cast(&nprobe_) - - reinterpret_cast(&id_)) + sizeof(nprobe_)); + static_cast(reinterpret_cast(&topk_) - + reinterpret_cast(&id_)) + sizeof(topk_)); // @@protoc_insertion_point(copy_constructor:milvus.grpc.SearchByIDParam) } @@ -4599,8 +5062,8 @@ void SearchByIDParam::SharedCtor() { ::PROTOBUF_NAMESPACE_ID::internal::InitSCC(&scc_info_SearchByIDParam_milvus_2eproto.base); table_name_.UnsafeSetDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); ::memset(&id_, 0, static_cast( - reinterpret_cast(&nprobe_) - - reinterpret_cast(&id_)) + sizeof(nprobe_)); + reinterpret_cast(&topk_) - + reinterpret_cast(&id_)) + sizeof(topk_)); } SearchByIDParam::~SearchByIDParam() { @@ -4628,10 +5091,11 @@ void SearchByIDParam::Clear() { (void) cached_has_bits; partition_tag_array_.Clear(); + extra_params_.Clear(); table_name_.ClearToEmptyNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); ::memset(&id_, 0, static_cast( - reinterpret_cast(&nprobe_) - - reinterpret_cast(&id_)) + sizeof(nprobe_)); + reinterpret_cast(&topk_) - + reinterpret_cast(&id_)) + sizeof(topk_)); _internal_metadata_.Clear(); } @@ -4650,34 +5114,39 @@ const char* SearchByIDParam::_InternalParse(const char* ptr, ::PROTOBUF_NAMESPAC CHK_(ptr); } else goto handle_unusual; continue; - // int64 id = 2; + // repeated string partition_tag_array = 2; case 2: - if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 16)) { + if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 18)) { + ptr -= 1; + do { + ptr += 1; + ptr = ::PROTOBUF_NAMESPACE_ID::internal::InlineGreedyStringParserUTF8(add_partition_tag_array(), ptr, ctx, "milvus.grpc.SearchByIDParam.partition_tag_array"); + CHK_(ptr); + if (!ctx->DataAvailable(ptr)) break; + } while (::PROTOBUF_NAMESPACE_ID::internal::UnalignedLoad<::PROTOBUF_NAMESPACE_ID::uint8>(ptr) == 18); + } else goto handle_unusual; + continue; + // int64 id = 3; + case 3: + if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 24)) { id_ = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint(&ptr); CHK_(ptr); } else goto handle_unusual; continue; - // int64 topk = 3; - case 3: - if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 24)) { + // int64 topk = 4; + case 4: + if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 32)) { topk_ = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint(&ptr); CHK_(ptr); } else goto handle_unusual; continue; - // int64 nprobe = 4; - case 4: - if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 32)) { - nprobe_ = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint(&ptr); - CHK_(ptr); - } else goto handle_unusual; - continue; - // repeated string partition_tag_array = 5; + // repeated .milvus.grpc.KeyValuePair extra_params = 5; case 5: if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 42)) { ptr -= 1; do { ptr += 1; - ptr = ::PROTOBUF_NAMESPACE_ID::internal::InlineGreedyStringParserUTF8(add_partition_tag_array(), ptr, ctx, "milvus.grpc.SearchByIDParam.partition_tag_array"); + ptr = ctx->ParseMessage(add_extra_params(), ptr); CHK_(ptr); if (!ctx->DataAvailable(ptr)) break; } while (::PROTOBUF_NAMESPACE_ID::internal::UnalignedLoad<::PROTOBUF_NAMESPACE_ID::uint8>(ptr) == 42); @@ -4728,9 +5197,25 @@ bool SearchByIDParam::MergePartialFromCodedStream( break; } - // int64 id = 2; + // repeated string partition_tag_array = 2; case 2: { - if (static_cast< ::PROTOBUF_NAMESPACE_ID::uint8>(tag) == (16 & 0xFF)) { + if (static_cast< ::PROTOBUF_NAMESPACE_ID::uint8>(tag) == (18 & 0xFF)) { + DO_(::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::ReadString( + input, this->add_partition_tag_array())); + DO_(::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::VerifyUtf8String( + this->partition_tag_array(this->partition_tag_array_size() - 1).data(), + static_cast(this->partition_tag_array(this->partition_tag_array_size() - 1).length()), + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::PARSE, + "milvus.grpc.SearchByIDParam.partition_tag_array")); + } else { + goto handle_unusual; + } + break; + } + + // int64 id = 3; + case 3: { + if (static_cast< ::PROTOBUF_NAMESPACE_ID::uint8>(tag) == (24 & 0xFF)) { DO_((::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::ReadPrimitive< ::PROTOBUF_NAMESPACE_ID::int64, ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::TYPE_INT64>( @@ -4741,9 +5226,9 @@ bool SearchByIDParam::MergePartialFromCodedStream( break; } - // int64 topk = 3; - case 3: { - if (static_cast< ::PROTOBUF_NAMESPACE_ID::uint8>(tag) == (24 & 0xFF)) { + // int64 topk = 4; + case 4: { + if (static_cast< ::PROTOBUF_NAMESPACE_ID::uint8>(tag) == (32 & 0xFF)) { DO_((::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::ReadPrimitive< ::PROTOBUF_NAMESPACE_ID::int64, ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::TYPE_INT64>( @@ -4754,29 +5239,11 @@ bool SearchByIDParam::MergePartialFromCodedStream( break; } - // int64 nprobe = 4; - case 4: { - if (static_cast< ::PROTOBUF_NAMESPACE_ID::uint8>(tag) == (32 & 0xFF)) { - - DO_((::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::ReadPrimitive< - ::PROTOBUF_NAMESPACE_ID::int64, ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::TYPE_INT64>( - input, &nprobe_))); - } else { - goto handle_unusual; - } - break; - } - - // repeated string partition_tag_array = 5; + // repeated .milvus.grpc.KeyValuePair extra_params = 5; case 5: { if (static_cast< ::PROTOBUF_NAMESPACE_ID::uint8>(tag) == (42 & 0xFF)) { - DO_(::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::ReadString( - input, this->add_partition_tag_array())); - DO_(::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::VerifyUtf8String( - this->partition_tag_array(this->partition_tag_array_size() - 1).data(), - static_cast(this->partition_tag_array(this->partition_tag_array_size() - 1).length()), - ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::PARSE, - "milvus.grpc.SearchByIDParam.partition_tag_array")); + DO_(::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::ReadMessage( + input, add_extra_params())); } else { goto handle_unusual; } @@ -4820,29 +5287,33 @@ void SearchByIDParam::SerializeWithCachedSizes( 1, this->table_name(), output); } - // int64 id = 2; - if (this->id() != 0) { - ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt64(2, this->id(), output); - } - - // int64 topk = 3; - if (this->topk() != 0) { - ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt64(3, this->topk(), output); - } - - // int64 nprobe = 4; - if (this->nprobe() != 0) { - ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt64(4, this->nprobe(), output); - } - - // repeated string partition_tag_array = 5; + // repeated string partition_tag_array = 2; for (int i = 0, n = this->partition_tag_array_size(); i < n; i++) { ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::VerifyUtf8String( this->partition_tag_array(i).data(), static_cast(this->partition_tag_array(i).length()), ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::SERIALIZE, "milvus.grpc.SearchByIDParam.partition_tag_array"); ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteString( - 5, this->partition_tag_array(i), output); + 2, this->partition_tag_array(i), output); + } + + // int64 id = 3; + if (this->id() != 0) { + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt64(3, this->id(), output); + } + + // int64 topk = 4; + if (this->topk() != 0) { + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt64(4, this->topk(), output); + } + + // repeated .milvus.grpc.KeyValuePair extra_params = 5; + for (unsigned int i = 0, + n = static_cast(this->extra_params_size()); i < n; i++) { + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteMessageMaybeToArray( + 5, + this->extra_params(static_cast(i)), + output); } if (_internal_metadata_.have_unknown_fields()) { @@ -4869,29 +5340,32 @@ void SearchByIDParam::SerializeWithCachedSizes( 1, this->table_name(), target); } - // int64 id = 2; - if (this->id() != 0) { - target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt64ToArray(2, this->id(), target); - } - - // int64 topk = 3; - if (this->topk() != 0) { - target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt64ToArray(3, this->topk(), target); - } - - // int64 nprobe = 4; - if (this->nprobe() != 0) { - target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt64ToArray(4, this->nprobe(), target); - } - - // repeated string partition_tag_array = 5; + // repeated string partition_tag_array = 2; for (int i = 0, n = this->partition_tag_array_size(); i < n; i++) { ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::VerifyUtf8String( this->partition_tag_array(i).data(), static_cast(this->partition_tag_array(i).length()), ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::SERIALIZE, "milvus.grpc.SearchByIDParam.partition_tag_array"); target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite:: - WriteStringToArray(5, this->partition_tag_array(i), target); + WriteStringToArray(2, this->partition_tag_array(i), target); + } + + // int64 id = 3; + if (this->id() != 0) { + target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt64ToArray(3, this->id(), target); + } + + // int64 topk = 4; + if (this->topk() != 0) { + target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt64ToArray(4, this->topk(), target); + } + + // repeated .milvus.grpc.KeyValuePair extra_params = 5; + for (unsigned int i = 0, + n = static_cast(this->extra_params_size()); i < n; i++) { + target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite:: + InternalWriteMessageToArray( + 5, this->extra_params(static_cast(i)), target); } if (_internal_metadata_.have_unknown_fields()) { @@ -4915,7 +5389,7 @@ size_t SearchByIDParam::ByteSizeLong() const { // Prevent compiler warnings about cached_has_bits being unused (void) cached_has_bits; - // repeated string partition_tag_array = 5; + // repeated string partition_tag_array = 2; total_size += 1 * ::PROTOBUF_NAMESPACE_ID::internal::FromIntSize(this->partition_tag_array_size()); for (int i = 0, n = this->partition_tag_array_size(); i < n; i++) { @@ -4923,6 +5397,17 @@ size_t SearchByIDParam::ByteSizeLong() const { this->partition_tag_array(i)); } + // repeated .milvus.grpc.KeyValuePair extra_params = 5; + { + unsigned int count = static_cast(this->extra_params_size()); + total_size += 1UL * count; + for (unsigned int i = 0; i < count; i++) { + total_size += + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::MessageSize( + this->extra_params(static_cast(i))); + } + } + // string table_name = 1; if (this->table_name().size() > 0) { total_size += 1 + @@ -4930,27 +5415,20 @@ size_t SearchByIDParam::ByteSizeLong() const { this->table_name()); } - // int64 id = 2; + // int64 id = 3; if (this->id() != 0) { total_size += 1 + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::Int64Size( this->id()); } - // int64 topk = 3; + // int64 topk = 4; if (this->topk() != 0) { total_size += 1 + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::Int64Size( this->topk()); } - // int64 nprobe = 4; - if (this->nprobe() != 0) { - total_size += 1 + - ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::Int64Size( - this->nprobe()); - } - int cached_size = ::PROTOBUF_NAMESPACE_ID::internal::ToCachedSize(total_size); SetCachedSize(cached_size); return total_size; @@ -4979,6 +5457,7 @@ void SearchByIDParam::MergeFrom(const SearchByIDParam& from) { (void) cached_has_bits; partition_tag_array_.MergeFrom(from.partition_tag_array_); + extra_params_.MergeFrom(from.extra_params_); if (from.table_name().size() > 0) { table_name_.AssignWithDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), from.table_name_); @@ -4989,9 +5468,6 @@ void SearchByIDParam::MergeFrom(const SearchByIDParam& from) { if (from.topk() != 0) { set_topk(from.topk()); } - if (from.nprobe() != 0) { - set_nprobe(from.nprobe()); - } } void SearchByIDParam::CopyFrom(const ::PROTOBUF_NAMESPACE_ID::Message& from) { @@ -5016,11 +5492,11 @@ void SearchByIDParam::InternalSwap(SearchByIDParam* other) { using std::swap; _internal_metadata_.Swap(&other->_internal_metadata_); partition_tag_array_.InternalSwap(CastToBase(&other->partition_tag_array_)); + CastToBase(&extra_params_)->InternalSwap(CastToBase(&other->extra_params_)); table_name_.Swap(&other->table_name_, &::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), GetArenaNoVirtual()); swap(id_, other->id_); swap(topk_, other->topk_); - swap(nprobe_, other->nprobe_); } ::PROTOBUF_NAMESPACE_ID::Metadata SearchByIDParam::GetMetadata() const { @@ -6717,324 +7193,21 @@ void Command::InternalSwap(Command* other) { } -// =================================================================== - -void Index::InitAsDefaultInstance() { -} -class Index::_Internal { - public: -}; - -Index::Index() - : ::PROTOBUF_NAMESPACE_ID::Message(), _internal_metadata_(nullptr) { - SharedCtor(); - // @@protoc_insertion_point(constructor:milvus.grpc.Index) -} -Index::Index(const Index& from) - : ::PROTOBUF_NAMESPACE_ID::Message(), - _internal_metadata_(nullptr) { - _internal_metadata_.MergeFrom(from._internal_metadata_); - ::memcpy(&index_type_, &from.index_type_, - static_cast(reinterpret_cast(&nlist_) - - reinterpret_cast(&index_type_)) + sizeof(nlist_)); - // @@protoc_insertion_point(copy_constructor:milvus.grpc.Index) -} - -void Index::SharedCtor() { - ::memset(&index_type_, 0, static_cast( - reinterpret_cast(&nlist_) - - reinterpret_cast(&index_type_)) + sizeof(nlist_)); -} - -Index::~Index() { - // @@protoc_insertion_point(destructor:milvus.grpc.Index) - SharedDtor(); -} - -void Index::SharedDtor() { -} - -void Index::SetCachedSize(int size) const { - _cached_size_.Set(size); -} -const Index& Index::default_instance() { - ::PROTOBUF_NAMESPACE_ID::internal::InitSCC(&::scc_info_Index_milvus_2eproto.base); - return *internal_default_instance(); -} - - -void Index::Clear() { -// @@protoc_insertion_point(message_clear_start:milvus.grpc.Index) - ::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0; - // Prevent compiler warnings about cached_has_bits being unused - (void) cached_has_bits; - - ::memset(&index_type_, 0, static_cast( - reinterpret_cast(&nlist_) - - reinterpret_cast(&index_type_)) + sizeof(nlist_)); - _internal_metadata_.Clear(); -} - -#if GOOGLE_PROTOBUF_ENABLE_EXPERIMENTAL_PARSER -const char* Index::_InternalParse(const char* ptr, ::PROTOBUF_NAMESPACE_ID::internal::ParseContext* ctx) { -#define CHK_(x) if (PROTOBUF_PREDICT_FALSE(!(x))) goto failure - while (!ctx->Done(&ptr)) { - ::PROTOBUF_NAMESPACE_ID::uint32 tag; - ptr = ::PROTOBUF_NAMESPACE_ID::internal::ReadTag(ptr, &tag); - CHK_(ptr); - switch (tag >> 3) { - // int32 index_type = 1; - case 1: - if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 8)) { - index_type_ = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint(&ptr); - CHK_(ptr); - } else goto handle_unusual; - continue; - // int32 nlist = 2; - case 2: - if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 16)) { - nlist_ = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint(&ptr); - CHK_(ptr); - } else goto handle_unusual; - continue; - default: { - handle_unusual: - if ((tag & 7) == 4 || tag == 0) { - ctx->SetLastTag(tag); - goto success; - } - ptr = UnknownFieldParse(tag, &_internal_metadata_, ptr, ctx); - CHK_(ptr != nullptr); - continue; - } - } // switch - } // while -success: - return ptr; -failure: - ptr = nullptr; - goto success; -#undef CHK_ -} -#else // GOOGLE_PROTOBUF_ENABLE_EXPERIMENTAL_PARSER -bool Index::MergePartialFromCodedStream( - ::PROTOBUF_NAMESPACE_ID::io::CodedInputStream* input) { -#define DO_(EXPRESSION) if (!PROTOBUF_PREDICT_TRUE(EXPRESSION)) goto failure - ::PROTOBUF_NAMESPACE_ID::uint32 tag; - // @@protoc_insertion_point(parse_start:milvus.grpc.Index) - for (;;) { - ::std::pair<::PROTOBUF_NAMESPACE_ID::uint32, bool> p = input->ReadTagWithCutoffNoLastTag(127u); - tag = p.first; - if (!p.second) goto handle_unusual; - switch (::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::GetTagFieldNumber(tag)) { - // int32 index_type = 1; - case 1: { - if (static_cast< ::PROTOBUF_NAMESPACE_ID::uint8>(tag) == (8 & 0xFF)) { - - DO_((::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::ReadPrimitive< - ::PROTOBUF_NAMESPACE_ID::int32, ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::TYPE_INT32>( - input, &index_type_))); - } else { - goto handle_unusual; - } - break; - } - - // int32 nlist = 2; - case 2: { - if (static_cast< ::PROTOBUF_NAMESPACE_ID::uint8>(tag) == (16 & 0xFF)) { - - DO_((::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::ReadPrimitive< - ::PROTOBUF_NAMESPACE_ID::int32, ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::TYPE_INT32>( - input, &nlist_))); - } else { - goto handle_unusual; - } - break; - } - - default: { - handle_unusual: - if (tag == 0) { - goto success; - } - DO_(::PROTOBUF_NAMESPACE_ID::internal::WireFormat::SkipField( - input, tag, _internal_metadata_.mutable_unknown_fields())); - break; - } - } - } -success: - // @@protoc_insertion_point(parse_success:milvus.grpc.Index) - return true; -failure: - // @@protoc_insertion_point(parse_failure:milvus.grpc.Index) - return false; -#undef DO_ -} -#endif // GOOGLE_PROTOBUF_ENABLE_EXPERIMENTAL_PARSER - -void Index::SerializeWithCachedSizes( - ::PROTOBUF_NAMESPACE_ID::io::CodedOutputStream* output) const { - // @@protoc_insertion_point(serialize_start:milvus.grpc.Index) - ::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0; - (void) cached_has_bits; - - // int32 index_type = 1; - if (this->index_type() != 0) { - ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt32(1, this->index_type(), output); - } - - // int32 nlist = 2; - if (this->nlist() != 0) { - ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt32(2, this->nlist(), output); - } - - if (_internal_metadata_.have_unknown_fields()) { - ::PROTOBUF_NAMESPACE_ID::internal::WireFormat::SerializeUnknownFields( - _internal_metadata_.unknown_fields(), output); - } - // @@protoc_insertion_point(serialize_end:milvus.grpc.Index) -} - -::PROTOBUF_NAMESPACE_ID::uint8* Index::InternalSerializeWithCachedSizesToArray( - ::PROTOBUF_NAMESPACE_ID::uint8* target) const { - // @@protoc_insertion_point(serialize_to_array_start:milvus.grpc.Index) - ::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0; - (void) cached_has_bits; - - // int32 index_type = 1; - if (this->index_type() != 0) { - target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt32ToArray(1, this->index_type(), target); - } - - // int32 nlist = 2; - if (this->nlist() != 0) { - target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt32ToArray(2, this->nlist(), target); - } - - if (_internal_metadata_.have_unknown_fields()) { - target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormat::SerializeUnknownFieldsToArray( - _internal_metadata_.unknown_fields(), target); - } - // @@protoc_insertion_point(serialize_to_array_end:milvus.grpc.Index) - return target; -} - -size_t Index::ByteSizeLong() const { -// @@protoc_insertion_point(message_byte_size_start:milvus.grpc.Index) - size_t total_size = 0; - - if (_internal_metadata_.have_unknown_fields()) { - total_size += - ::PROTOBUF_NAMESPACE_ID::internal::WireFormat::ComputeUnknownFieldsSize( - _internal_metadata_.unknown_fields()); - } - ::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0; - // Prevent compiler warnings about cached_has_bits being unused - (void) cached_has_bits; - - // int32 index_type = 1; - if (this->index_type() != 0) { - total_size += 1 + - ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::Int32Size( - this->index_type()); - } - - // int32 nlist = 2; - if (this->nlist() != 0) { - total_size += 1 + - ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::Int32Size( - this->nlist()); - } - - int cached_size = ::PROTOBUF_NAMESPACE_ID::internal::ToCachedSize(total_size); - SetCachedSize(cached_size); - return total_size; -} - -void Index::MergeFrom(const ::PROTOBUF_NAMESPACE_ID::Message& from) { -// @@protoc_insertion_point(generalized_merge_from_start:milvus.grpc.Index) - GOOGLE_DCHECK_NE(&from, this); - const Index* source = - ::PROTOBUF_NAMESPACE_ID::DynamicCastToGenerated( - &from); - if (source == nullptr) { - // @@protoc_insertion_point(generalized_merge_from_cast_fail:milvus.grpc.Index) - ::PROTOBUF_NAMESPACE_ID::internal::ReflectionOps::Merge(from, this); - } else { - // @@protoc_insertion_point(generalized_merge_from_cast_success:milvus.grpc.Index) - MergeFrom(*source); - } -} - -void Index::MergeFrom(const Index& from) { -// @@protoc_insertion_point(class_specific_merge_from_start:milvus.grpc.Index) - GOOGLE_DCHECK_NE(&from, this); - _internal_metadata_.MergeFrom(from._internal_metadata_); - ::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0; - (void) cached_has_bits; - - if (from.index_type() != 0) { - set_index_type(from.index_type()); - } - if (from.nlist() != 0) { - set_nlist(from.nlist()); - } -} - -void Index::CopyFrom(const ::PROTOBUF_NAMESPACE_ID::Message& from) { -// @@protoc_insertion_point(generalized_copy_from_start:milvus.grpc.Index) - if (&from == this) return; - Clear(); - MergeFrom(from); -} - -void Index::CopyFrom(const Index& from) { -// @@protoc_insertion_point(class_specific_copy_from_start:milvus.grpc.Index) - if (&from == this) return; - Clear(); - MergeFrom(from); -} - -bool Index::IsInitialized() const { - return true; -} - -void Index::InternalSwap(Index* other) { - using std::swap; - _internal_metadata_.Swap(&other->_internal_metadata_); - swap(index_type_, other->index_type_); - swap(nlist_, other->nlist_); -} - -::PROTOBUF_NAMESPACE_ID::Metadata Index::GetMetadata() const { - return GetMetadataStatic(); -} - - // =================================================================== void IndexParam::InitAsDefaultInstance() { ::milvus::grpc::_IndexParam_default_instance_._instance.get_mutable()->status_ = const_cast< ::milvus::grpc::Status*>( ::milvus::grpc::Status::internal_default_instance()); - ::milvus::grpc::_IndexParam_default_instance_._instance.get_mutable()->index_ = const_cast< ::milvus::grpc::Index*>( - ::milvus::grpc::Index::internal_default_instance()); } class IndexParam::_Internal { public: static const ::milvus::grpc::Status& status(const IndexParam* msg); - static const ::milvus::grpc::Index& index(const IndexParam* msg); }; const ::milvus::grpc::Status& IndexParam::_Internal::status(const IndexParam* msg) { return *msg->status_; } -const ::milvus::grpc::Index& -IndexParam::_Internal::index(const IndexParam* msg) { - return *msg->index_; -} void IndexParam::clear_status() { if (GetArenaNoVirtual() == nullptr && status_ != nullptr) { delete status_; @@ -7048,7 +7221,8 @@ IndexParam::IndexParam() } IndexParam::IndexParam(const IndexParam& from) : ::PROTOBUF_NAMESPACE_ID::Message(), - _internal_metadata_(nullptr) { + _internal_metadata_(nullptr), + extra_params_(from.extra_params_) { _internal_metadata_.MergeFrom(from._internal_metadata_); table_name_.UnsafeSetDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); if (!from.table_name().empty()) { @@ -7059,11 +7233,7 @@ IndexParam::IndexParam(const IndexParam& from) } else { status_ = nullptr; } - if (from.has_index()) { - index_ = new ::milvus::grpc::Index(*from.index_); - } else { - index_ = nullptr; - } + index_type_ = from.index_type_; // @@protoc_insertion_point(copy_constructor:milvus.grpc.IndexParam) } @@ -7071,8 +7241,8 @@ void IndexParam::SharedCtor() { ::PROTOBUF_NAMESPACE_ID::internal::InitSCC(&scc_info_IndexParam_milvus_2eproto.base); table_name_.UnsafeSetDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); ::memset(&status_, 0, static_cast( - reinterpret_cast(&index_) - - reinterpret_cast(&status_)) + sizeof(index_)); + reinterpret_cast(&index_type_) - + reinterpret_cast(&status_)) + sizeof(index_type_)); } IndexParam::~IndexParam() { @@ -7083,7 +7253,6 @@ IndexParam::~IndexParam() { void IndexParam::SharedDtor() { table_name_.DestroyNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); if (this != internal_default_instance()) delete status_; - if (this != internal_default_instance()) delete index_; } void IndexParam::SetCachedSize(int size) const { @@ -7101,15 +7270,13 @@ void IndexParam::Clear() { // Prevent compiler warnings about cached_has_bits being unused (void) cached_has_bits; + extra_params_.Clear(); table_name_.ClearToEmptyNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); if (GetArenaNoVirtual() == nullptr && status_ != nullptr) { delete status_; } status_ = nullptr; - if (GetArenaNoVirtual() == nullptr && index_ != nullptr) { - delete index_; - } - index_ = nullptr; + index_type_ = 0; _internal_metadata_.Clear(); } @@ -7135,13 +7302,25 @@ const char* IndexParam::_InternalParse(const char* ptr, ::PROTOBUF_NAMESPACE_ID: CHK_(ptr); } else goto handle_unusual; continue; - // .milvus.grpc.Index index = 3; + // int32 index_type = 3; case 3: - if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 26)) { - ptr = ctx->ParseMessage(mutable_index(), ptr); + if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 24)) { + index_type_ = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint(&ptr); CHK_(ptr); } else goto handle_unusual; continue; + // repeated .milvus.grpc.KeyValuePair extra_params = 4; + case 4: + if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 34)) { + ptr -= 1; + do { + ptr += 1; + ptr = ctx->ParseMessage(add_extra_params(), ptr); + CHK_(ptr); + if (!ctx->DataAvailable(ptr)) break; + } while (::PROTOBUF_NAMESPACE_ID::internal::UnalignedLoad<::PROTOBUF_NAMESPACE_ID::uint8>(ptr) == 34); + } else goto handle_unusual; + continue; default: { handle_unusual: if ((tag & 7) == 4 || tag == 0) { @@ -7198,11 +7377,24 @@ bool IndexParam::MergePartialFromCodedStream( break; } - // .milvus.grpc.Index index = 3; + // int32 index_type = 3; case 3: { - if (static_cast< ::PROTOBUF_NAMESPACE_ID::uint8>(tag) == (26 & 0xFF)) { + if (static_cast< ::PROTOBUF_NAMESPACE_ID::uint8>(tag) == (24 & 0xFF)) { + + DO_((::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::ReadPrimitive< + ::PROTOBUF_NAMESPACE_ID::int32, ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::TYPE_INT32>( + input, &index_type_))); + } else { + goto handle_unusual; + } + break; + } + + // repeated .milvus.grpc.KeyValuePair extra_params = 4; + case 4: { + if (static_cast< ::PROTOBUF_NAMESPACE_ID::uint8>(tag) == (34 & 0xFF)) { DO_(::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::ReadMessage( - input, mutable_index())); + input, add_extra_params())); } else { goto handle_unusual; } @@ -7252,10 +7444,18 @@ void IndexParam::SerializeWithCachedSizes( 2, this->table_name(), output); } - // .milvus.grpc.Index index = 3; - if (this->has_index()) { + // int32 index_type = 3; + if (this->index_type() != 0) { + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt32(3, this->index_type(), output); + } + + // repeated .milvus.grpc.KeyValuePair extra_params = 4; + for (unsigned int i = 0, + n = static_cast(this->extra_params_size()); i < n; i++) { ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteMessageMaybeToArray( - 3, _Internal::index(this), output); + 4, + this->extra_params(static_cast(i)), + output); } if (_internal_metadata_.have_unknown_fields()) { @@ -7289,11 +7489,17 @@ void IndexParam::SerializeWithCachedSizes( 2, this->table_name(), target); } - // .milvus.grpc.Index index = 3; - if (this->has_index()) { + // int32 index_type = 3; + if (this->index_type() != 0) { + target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt32ToArray(3, this->index_type(), target); + } + + // repeated .milvus.grpc.KeyValuePair extra_params = 4; + for (unsigned int i = 0, + n = static_cast(this->extra_params_size()); i < n; i++) { target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite:: InternalWriteMessageToArray( - 3, _Internal::index(this), target); + 4, this->extra_params(static_cast(i)), target); } if (_internal_metadata_.have_unknown_fields()) { @@ -7317,6 +7523,17 @@ size_t IndexParam::ByteSizeLong() const { // Prevent compiler warnings about cached_has_bits being unused (void) cached_has_bits; + // repeated .milvus.grpc.KeyValuePair extra_params = 4; + { + unsigned int count = static_cast(this->extra_params_size()); + total_size += 1UL * count; + for (unsigned int i = 0; i < count; i++) { + total_size += + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::MessageSize( + this->extra_params(static_cast(i))); + } + } + // string table_name = 2; if (this->table_name().size() > 0) { total_size += 1 + @@ -7331,11 +7548,11 @@ size_t IndexParam::ByteSizeLong() const { *status_); } - // .milvus.grpc.Index index = 3; - if (this->has_index()) { + // int32 index_type = 3; + if (this->index_type() != 0) { total_size += 1 + - ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::MessageSize( - *index_); + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::Int32Size( + this->index_type()); } int cached_size = ::PROTOBUF_NAMESPACE_ID::internal::ToCachedSize(total_size); @@ -7365,6 +7582,7 @@ void IndexParam::MergeFrom(const IndexParam& from) { ::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0; (void) cached_has_bits; + extra_params_.MergeFrom(from.extra_params_); if (from.table_name().size() > 0) { table_name_.AssignWithDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), from.table_name_); @@ -7372,8 +7590,8 @@ void IndexParam::MergeFrom(const IndexParam& from) { if (from.has_status()) { mutable_status()->::milvus::grpc::Status::MergeFrom(from.status()); } - if (from.has_index()) { - mutable_index()->::milvus::grpc::Index::MergeFrom(from.index()); + if (from.index_type() != 0) { + set_index_type(from.index_type()); } } @@ -7398,10 +7616,11 @@ bool IndexParam::IsInitialized() const { void IndexParam::InternalSwap(IndexParam* other) { using std::swap; _internal_metadata_.Swap(&other->_internal_metadata_); + CastToBase(&extra_params_)->InternalSwap(CastToBase(&other->extra_params_)); table_name_.Swap(&other->table_name_, &::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), GetArenaNoVirtual()); swap(status_, other->status_); - swap(index_, other->index_); + swap(index_type_, other->index_type_); } ::PROTOBUF_NAMESPACE_ID::Metadata IndexParam::GetMetadata() const { @@ -10158,6 +10377,9 @@ void GetVectorIDsParam::InternalSwap(GetVectorIDsParam* other) { } // namespace grpc } // namespace milvus PROTOBUF_NAMESPACE_OPEN +template<> PROTOBUF_NOINLINE ::milvus::grpc::KeyValuePair* Arena::CreateMaybeMessage< ::milvus::grpc::KeyValuePair >(Arena* arena) { + return Arena::CreateInternal< ::milvus::grpc::KeyValuePair >(arena); +} template<> PROTOBUF_NOINLINE ::milvus::grpc::TableName* Arena::CreateMaybeMessage< ::milvus::grpc::TableName >(Arena* arena) { return Arena::CreateInternal< ::milvus::grpc::TableName >(arena); } @@ -10206,9 +10428,6 @@ template<> PROTOBUF_NOINLINE ::milvus::grpc::TableRowCount* Arena::CreateMaybeMe template<> PROTOBUF_NOINLINE ::milvus::grpc::Command* Arena::CreateMaybeMessage< ::milvus::grpc::Command >(Arena* arena) { return Arena::CreateInternal< ::milvus::grpc::Command >(arena); } -template<> PROTOBUF_NOINLINE ::milvus::grpc::Index* Arena::CreateMaybeMessage< ::milvus::grpc::Index >(Arena* arena) { - return Arena::CreateInternal< ::milvus::grpc::Index >(arena); -} template<> PROTOBUF_NOINLINE ::milvus::grpc::IndexParam* Arena::CreateMaybeMessage< ::milvus::grpc::IndexParam >(Arena* arena) { return Arena::CreateInternal< ::milvus::grpc::IndexParam >(arena); } diff --git a/core/src/grpc/gen-milvus/milvus.pb.h b/core/src/grpc/gen-milvus/milvus.pb.h index cc03c9bd26..4528f278e1 100644 --- a/core/src/grpc/gen-milvus/milvus.pb.h +++ b/core/src/grpc/gen-milvus/milvus.pb.h @@ -72,15 +72,15 @@ extern FlushParamDefaultTypeInternal _FlushParam_default_instance_; class GetVectorIDsParam; class GetVectorIDsParamDefaultTypeInternal; extern GetVectorIDsParamDefaultTypeInternal _GetVectorIDsParam_default_instance_; -class Index; -class IndexDefaultTypeInternal; -extern IndexDefaultTypeInternal _Index_default_instance_; class IndexParam; class IndexParamDefaultTypeInternal; extern IndexParamDefaultTypeInternal _IndexParam_default_instance_; class InsertParam; class InsertParamDefaultTypeInternal; extern InsertParamDefaultTypeInternal _InsertParam_default_instance_; +class KeyValuePair; +class KeyValuePairDefaultTypeInternal; +extern KeyValuePairDefaultTypeInternal _KeyValuePair_default_instance_; class PartitionList; class PartitionListDefaultTypeInternal; extern PartitionListDefaultTypeInternal _PartitionList_default_instance_; @@ -143,9 +143,9 @@ template<> ::milvus::grpc::Command* Arena::CreateMaybeMessage<::milvus::grpc::Co template<> ::milvus::grpc::DeleteByIDParam* Arena::CreateMaybeMessage<::milvus::grpc::DeleteByIDParam>(Arena*); template<> ::milvus::grpc::FlushParam* Arena::CreateMaybeMessage<::milvus::grpc::FlushParam>(Arena*); template<> ::milvus::grpc::GetVectorIDsParam* Arena::CreateMaybeMessage<::milvus::grpc::GetVectorIDsParam>(Arena*); -template<> ::milvus::grpc::Index* Arena::CreateMaybeMessage<::milvus::grpc::Index>(Arena*); template<> ::milvus::grpc::IndexParam* Arena::CreateMaybeMessage<::milvus::grpc::IndexParam>(Arena*); template<> ::milvus::grpc::InsertParam* Arena::CreateMaybeMessage<::milvus::grpc::InsertParam>(Arena*); +template<> ::milvus::grpc::KeyValuePair* Arena::CreateMaybeMessage<::milvus::grpc::KeyValuePair>(Arena*); template<> ::milvus::grpc::PartitionList* Arena::CreateMaybeMessage<::milvus::grpc::PartitionList>(Arena*); template<> ::milvus::grpc::PartitionParam* Arena::CreateMaybeMessage<::milvus::grpc::PartitionParam>(Arena*); template<> ::milvus::grpc::PartitionStat* Arena::CreateMaybeMessage<::milvus::grpc::PartitionStat>(Arena*); @@ -170,6 +170,156 @@ namespace grpc { // =================================================================== +class KeyValuePair : + public ::PROTOBUF_NAMESPACE_ID::Message /* @@protoc_insertion_point(class_definition:milvus.grpc.KeyValuePair) */ { + public: + KeyValuePair(); + virtual ~KeyValuePair(); + + KeyValuePair(const KeyValuePair& from); + KeyValuePair(KeyValuePair&& from) noexcept + : KeyValuePair() { + *this = ::std::move(from); + } + + inline KeyValuePair& operator=(const KeyValuePair& from) { + CopyFrom(from); + return *this; + } + inline KeyValuePair& operator=(KeyValuePair&& from) noexcept { + if (GetArenaNoVirtual() == from.GetArenaNoVirtual()) { + if (this != &from) InternalSwap(&from); + } else { + CopyFrom(from); + } + return *this; + } + + static const ::PROTOBUF_NAMESPACE_ID::Descriptor* descriptor() { + return GetDescriptor(); + } + static const ::PROTOBUF_NAMESPACE_ID::Descriptor* GetDescriptor() { + return GetMetadataStatic().descriptor; + } + static const ::PROTOBUF_NAMESPACE_ID::Reflection* GetReflection() { + return GetMetadataStatic().reflection; + } + static const KeyValuePair& default_instance(); + + static void InitAsDefaultInstance(); // FOR INTERNAL USE ONLY + static inline const KeyValuePair* internal_default_instance() { + return reinterpret_cast( + &_KeyValuePair_default_instance_); + } + static constexpr int kIndexInFileMessages = + 0; + + friend void swap(KeyValuePair& a, KeyValuePair& b) { + a.Swap(&b); + } + inline void Swap(KeyValuePair* other) { + if (other == this) return; + InternalSwap(other); + } + + // implements Message ---------------------------------------------- + + inline KeyValuePair* New() const final { + return CreateMaybeMessage(nullptr); + } + + KeyValuePair* New(::PROTOBUF_NAMESPACE_ID::Arena* arena) const final { + return CreateMaybeMessage(arena); + } + void CopyFrom(const ::PROTOBUF_NAMESPACE_ID::Message& from) final; + void MergeFrom(const ::PROTOBUF_NAMESPACE_ID::Message& from) final; + void CopyFrom(const KeyValuePair& from); + void MergeFrom(const KeyValuePair& from); + PROTOBUF_ATTRIBUTE_REINITIALIZES void Clear() final; + bool IsInitialized() const final; + + size_t ByteSizeLong() const final; + #if GOOGLE_PROTOBUF_ENABLE_EXPERIMENTAL_PARSER + const char* _InternalParse(const char* ptr, ::PROTOBUF_NAMESPACE_ID::internal::ParseContext* ctx) final; + #else + bool MergePartialFromCodedStream( + ::PROTOBUF_NAMESPACE_ID::io::CodedInputStream* input) final; + #endif // GOOGLE_PROTOBUF_ENABLE_EXPERIMENTAL_PARSER + void SerializeWithCachedSizes( + ::PROTOBUF_NAMESPACE_ID::io::CodedOutputStream* output) const final; + ::PROTOBUF_NAMESPACE_ID::uint8* InternalSerializeWithCachedSizesToArray( + ::PROTOBUF_NAMESPACE_ID::uint8* target) const final; + int GetCachedSize() const final { return _cached_size_.Get(); } + + private: + inline void SharedCtor(); + inline void SharedDtor(); + void SetCachedSize(int size) const final; + void InternalSwap(KeyValuePair* other); + friend class ::PROTOBUF_NAMESPACE_ID::internal::AnyMetadata; + static ::PROTOBUF_NAMESPACE_ID::StringPiece FullMessageName() { + return "milvus.grpc.KeyValuePair"; + } + private: + inline ::PROTOBUF_NAMESPACE_ID::Arena* GetArenaNoVirtual() const { + return nullptr; + } + inline void* MaybeArenaPtr() const { + return nullptr; + } + public: + + ::PROTOBUF_NAMESPACE_ID::Metadata GetMetadata() const final; + private: + static ::PROTOBUF_NAMESPACE_ID::Metadata GetMetadataStatic() { + ::PROTOBUF_NAMESPACE_ID::internal::AssignDescriptors(&::descriptor_table_milvus_2eproto); + return ::descriptor_table_milvus_2eproto.file_level_metadata[kIndexInFileMessages]; + } + + public: + + // nested types ---------------------------------------------------- + + // accessors ------------------------------------------------------- + + enum : int { + kKeyFieldNumber = 1, + kValueFieldNumber = 2, + }; + // string key = 1; + void clear_key(); + const std::string& key() const; + void set_key(const std::string& value); + void set_key(std::string&& value); + void set_key(const char* value); + void set_key(const char* value, size_t size); + std::string* mutable_key(); + std::string* release_key(); + void set_allocated_key(std::string* key); + + // string value = 2; + void clear_value(); + const std::string& value() const; + void set_value(const std::string& value); + void set_value(std::string&& value); + void set_value(const char* value); + void set_value(const char* value, size_t size); + std::string* mutable_value(); + std::string* release_value(); + void set_allocated_value(std::string* value); + + // @@protoc_insertion_point(class_scope:milvus.grpc.KeyValuePair) + private: + class _Internal; + + ::PROTOBUF_NAMESPACE_ID::internal::InternalMetadataWithArena _internal_metadata_; + ::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr key_; + ::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr value_; + mutable ::PROTOBUF_NAMESPACE_ID::internal::CachedSize _cached_size_; + friend struct ::TableStruct_milvus_2eproto; +}; +// ------------------------------------------------------------------- + class TableName : public ::PROTOBUF_NAMESPACE_ID::Message /* @@protoc_insertion_point(class_definition:milvus.grpc.TableName) */ { public: @@ -212,7 +362,7 @@ class TableName : &_TableName_default_instance_); } static constexpr int kIndexInFileMessages = - 0; + 1; friend void swap(TableName& a, TableName& b) { a.Swap(&b); @@ -349,7 +499,7 @@ class TableNameList : &_TableNameList_default_instance_); } static constexpr int kIndexInFileMessages = - 1; + 2; friend void swap(TableNameList& a, TableNameList& b) { a.Swap(&b); @@ -502,7 +652,7 @@ class TableSchema : &_TableSchema_default_instance_); } static constexpr int kIndexInFileMessages = - 2; + 3; friend void swap(TableSchema& a, TableSchema& b) { a.Swap(&b); @@ -573,12 +723,24 @@ class TableSchema : // accessors ------------------------------------------------------- enum : int { + kExtraParamsFieldNumber = 6, kTableNameFieldNumber = 2, kStatusFieldNumber = 1, kDimensionFieldNumber = 3, kIndexFileSizeFieldNumber = 4, kMetricTypeFieldNumber = 5, }; + // repeated .milvus.grpc.KeyValuePair extra_params = 6; + int extra_params_size() const; + void clear_extra_params(); + ::milvus::grpc::KeyValuePair* mutable_extra_params(int index); + ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::milvus::grpc::KeyValuePair >* + mutable_extra_params(); + const ::milvus::grpc::KeyValuePair& extra_params(int index) const; + ::milvus::grpc::KeyValuePair* add_extra_params(); + const ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::milvus::grpc::KeyValuePair >& + extra_params() const; + // string table_name = 2; void clear_table_name(); const std::string& table_name() const; @@ -618,6 +780,7 @@ class TableSchema : class _Internal; ::PROTOBUF_NAMESPACE_ID::internal::InternalMetadataWithArena _internal_metadata_; + ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::milvus::grpc::KeyValuePair > extra_params_; ::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr table_name_; ::milvus::grpc::Status* status_; ::PROTOBUF_NAMESPACE_ID::int64 dimension_; @@ -670,7 +833,7 @@ class PartitionParam : &_PartitionParam_default_instance_); } static constexpr int kIndexInFileMessages = - 3; + 4; friend void swap(PartitionParam& a, PartitionParam& b) { a.Swap(&b); @@ -820,7 +983,7 @@ class PartitionList : &_PartitionList_default_instance_); } static constexpr int kIndexInFileMessages = - 4; + 5; friend void swap(PartitionList& a, PartitionList& b) { a.Swap(&b); @@ -973,7 +1136,7 @@ class RowRecord : &_RowRecord_default_instance_); } static constexpr int kIndexInFileMessages = - 5; + 6; friend void swap(RowRecord& a, RowRecord& b) { a.Swap(&b); @@ -1124,7 +1287,7 @@ class InsertParam : &_InsertParam_default_instance_); } static constexpr int kIndexInFileMessages = - 6; + 7; friend void swap(InsertParam& a, InsertParam& b) { a.Swap(&b); @@ -1197,6 +1360,7 @@ class InsertParam : enum : int { kRowRecordArrayFieldNumber = 2, kRowIdArrayFieldNumber = 3, + kExtraParamsFieldNumber = 5, kTableNameFieldNumber = 1, kPartitionTagFieldNumber = 4, }; @@ -1222,6 +1386,17 @@ class InsertParam : ::PROTOBUF_NAMESPACE_ID::RepeatedField< ::PROTOBUF_NAMESPACE_ID::int64 >* mutable_row_id_array(); + // repeated .milvus.grpc.KeyValuePair extra_params = 5; + int extra_params_size() const; + void clear_extra_params(); + ::milvus::grpc::KeyValuePair* mutable_extra_params(int index); + ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::milvus::grpc::KeyValuePair >* + mutable_extra_params(); + const ::milvus::grpc::KeyValuePair& extra_params(int index) const; + ::milvus::grpc::KeyValuePair* add_extra_params(); + const ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::milvus::grpc::KeyValuePair >& + extra_params() const; + // string table_name = 1; void clear_table_name(); const std::string& table_name() const; @@ -1252,6 +1427,7 @@ class InsertParam : ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::milvus::grpc::RowRecord > row_record_array_; ::PROTOBUF_NAMESPACE_ID::RepeatedField< ::PROTOBUF_NAMESPACE_ID::int64 > row_id_array_; mutable std::atomic _row_id_array_cached_byte_size_; + ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::milvus::grpc::KeyValuePair > extra_params_; ::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr table_name_; ::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr partition_tag_; mutable ::PROTOBUF_NAMESPACE_ID::internal::CachedSize _cached_size_; @@ -1301,7 +1477,7 @@ class VectorIds : &_VectorIds_default_instance_); } static constexpr int kIndexInFileMessages = - 7; + 8; friend void swap(VectorIds& a, VectorIds& b) { a.Swap(&b); @@ -1449,7 +1625,7 @@ class SearchParam : &_SearchParam_default_instance_); } static constexpr int kIndexInFileMessages = - 8; + 9; friend void swap(SearchParam& a, SearchParam& b) { a.Swap(&b); @@ -1520,24 +1696,13 @@ class SearchParam : // accessors ------------------------------------------------------- enum : int { - kQueryRecordArrayFieldNumber = 2, - kPartitionTagArrayFieldNumber = 5, + kPartitionTagArrayFieldNumber = 2, + kQueryRecordArrayFieldNumber = 3, + kExtraParamsFieldNumber = 5, kTableNameFieldNumber = 1, - kTopkFieldNumber = 3, - kNprobeFieldNumber = 4, + kTopkFieldNumber = 4, }; - // repeated .milvus.grpc.RowRecord query_record_array = 2; - int query_record_array_size() const; - void clear_query_record_array(); - ::milvus::grpc::RowRecord* mutable_query_record_array(int index); - ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::milvus::grpc::RowRecord >* - mutable_query_record_array(); - const ::milvus::grpc::RowRecord& query_record_array(int index) const; - ::milvus::grpc::RowRecord* add_query_record_array(); - const ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::milvus::grpc::RowRecord >& - query_record_array() const; - - // repeated string partition_tag_array = 5; + // repeated string partition_tag_array = 2; int partition_tag_array_size() const; void clear_partition_tag_array(); const std::string& partition_tag_array(int index) const; @@ -1554,6 +1719,28 @@ class SearchParam : const ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField& partition_tag_array() const; ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField* mutable_partition_tag_array(); + // repeated .milvus.grpc.RowRecord query_record_array = 3; + int query_record_array_size() const; + void clear_query_record_array(); + ::milvus::grpc::RowRecord* mutable_query_record_array(int index); + ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::milvus::grpc::RowRecord >* + mutable_query_record_array(); + const ::milvus::grpc::RowRecord& query_record_array(int index) const; + ::milvus::grpc::RowRecord* add_query_record_array(); + const ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::milvus::grpc::RowRecord >& + query_record_array() const; + + // repeated .milvus.grpc.KeyValuePair extra_params = 5; + int extra_params_size() const; + void clear_extra_params(); + ::milvus::grpc::KeyValuePair* mutable_extra_params(int index); + ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::milvus::grpc::KeyValuePair >* + mutable_extra_params(); + const ::milvus::grpc::KeyValuePair& extra_params(int index) const; + ::milvus::grpc::KeyValuePair* add_extra_params(); + const ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::milvus::grpc::KeyValuePair >& + extra_params() const; + // string table_name = 1; void clear_table_name(); const std::string& table_name() const; @@ -1565,26 +1752,21 @@ class SearchParam : std::string* release_table_name(); void set_allocated_table_name(std::string* table_name); - // int64 topk = 3; + // int64 topk = 4; void clear_topk(); ::PROTOBUF_NAMESPACE_ID::int64 topk() const; void set_topk(::PROTOBUF_NAMESPACE_ID::int64 value); - // int64 nprobe = 4; - void clear_nprobe(); - ::PROTOBUF_NAMESPACE_ID::int64 nprobe() const; - void set_nprobe(::PROTOBUF_NAMESPACE_ID::int64 value); - // @@protoc_insertion_point(class_scope:milvus.grpc.SearchParam) private: class _Internal; ::PROTOBUF_NAMESPACE_ID::internal::InternalMetadataWithArena _internal_metadata_; - ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::milvus::grpc::RowRecord > query_record_array_; ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField partition_tag_array_; + ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::milvus::grpc::RowRecord > query_record_array_; + ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::milvus::grpc::KeyValuePair > extra_params_; ::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr table_name_; ::PROTOBUF_NAMESPACE_ID::int64 topk_; - ::PROTOBUF_NAMESPACE_ID::int64 nprobe_; mutable ::PROTOBUF_NAMESPACE_ID::internal::CachedSize _cached_size_; friend struct ::TableStruct_milvus_2eproto; }; @@ -1632,7 +1814,7 @@ class SearchInFilesParam : &_SearchInFilesParam_default_instance_); } static constexpr int kIndexInFileMessages = - 9; + 10; friend void swap(SearchInFilesParam& a, SearchInFilesParam& b) { a.Swap(&b); @@ -1785,7 +1967,7 @@ class SearchByIDParam : &_SearchByIDParam_default_instance_); } static constexpr int kIndexInFileMessages = - 10; + 11; friend void swap(SearchByIDParam& a, SearchByIDParam& b) { a.Swap(&b); @@ -1856,13 +2038,13 @@ class SearchByIDParam : // accessors ------------------------------------------------------- enum : int { - kPartitionTagArrayFieldNumber = 5, + kPartitionTagArrayFieldNumber = 2, + kExtraParamsFieldNumber = 5, kTableNameFieldNumber = 1, - kIdFieldNumber = 2, - kTopkFieldNumber = 3, - kNprobeFieldNumber = 4, + kIdFieldNumber = 3, + kTopkFieldNumber = 4, }; - // repeated string partition_tag_array = 5; + // repeated string partition_tag_array = 2; int partition_tag_array_size() const; void clear_partition_tag_array(); const std::string& partition_tag_array(int index) const; @@ -1879,6 +2061,17 @@ class SearchByIDParam : const ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField& partition_tag_array() const; ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField* mutable_partition_tag_array(); + // repeated .milvus.grpc.KeyValuePair extra_params = 5; + int extra_params_size() const; + void clear_extra_params(); + ::milvus::grpc::KeyValuePair* mutable_extra_params(int index); + ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::milvus::grpc::KeyValuePair >* + mutable_extra_params(); + const ::milvus::grpc::KeyValuePair& extra_params(int index) const; + ::milvus::grpc::KeyValuePair* add_extra_params(); + const ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::milvus::grpc::KeyValuePair >& + extra_params() const; + // string table_name = 1; void clear_table_name(); const std::string& table_name() const; @@ -1890,31 +2083,26 @@ class SearchByIDParam : std::string* release_table_name(); void set_allocated_table_name(std::string* table_name); - // int64 id = 2; + // int64 id = 3; void clear_id(); ::PROTOBUF_NAMESPACE_ID::int64 id() const; void set_id(::PROTOBUF_NAMESPACE_ID::int64 value); - // int64 topk = 3; + // int64 topk = 4; void clear_topk(); ::PROTOBUF_NAMESPACE_ID::int64 topk() const; void set_topk(::PROTOBUF_NAMESPACE_ID::int64 value); - // int64 nprobe = 4; - void clear_nprobe(); - ::PROTOBUF_NAMESPACE_ID::int64 nprobe() const; - void set_nprobe(::PROTOBUF_NAMESPACE_ID::int64 value); - // @@protoc_insertion_point(class_scope:milvus.grpc.SearchByIDParam) private: class _Internal; ::PROTOBUF_NAMESPACE_ID::internal::InternalMetadataWithArena _internal_metadata_; ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField partition_tag_array_; + ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::milvus::grpc::KeyValuePair > extra_params_; ::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr table_name_; ::PROTOBUF_NAMESPACE_ID::int64 id_; ::PROTOBUF_NAMESPACE_ID::int64 topk_; - ::PROTOBUF_NAMESPACE_ID::int64 nprobe_; mutable ::PROTOBUF_NAMESPACE_ID::internal::CachedSize _cached_size_; friend struct ::TableStruct_milvus_2eproto; }; @@ -1962,7 +2150,7 @@ class TopKQueryResult : &_TopKQueryResult_default_instance_); } static constexpr int kIndexInFileMessages = - 11; + 12; friend void swap(TopKQueryResult& a, TopKQueryResult& b) { a.Swap(&b); @@ -2131,7 +2319,7 @@ class StringReply : &_StringReply_default_instance_); } static constexpr int kIndexInFileMessages = - 12; + 13; friend void swap(StringReply& a, StringReply& b) { a.Swap(&b); @@ -2278,7 +2466,7 @@ class BoolReply : &_BoolReply_default_instance_); } static constexpr int kIndexInFileMessages = - 13; + 14; friend void swap(BoolReply& a, BoolReply& b) { a.Swap(&b); @@ -2419,7 +2607,7 @@ class TableRowCount : &_TableRowCount_default_instance_); } static constexpr int kIndexInFileMessages = - 14; + 15; friend void swap(TableRowCount& a, TableRowCount& b) { a.Swap(&b); @@ -2560,7 +2748,7 @@ class Command : &_Command_default_instance_); } static constexpr int kIndexInFileMessages = - 15; + 16; friend void swap(Command& a, Command& b) { a.Swap(&b); @@ -2655,144 +2843,6 @@ class Command : }; // ------------------------------------------------------------------- -class Index : - public ::PROTOBUF_NAMESPACE_ID::Message /* @@protoc_insertion_point(class_definition:milvus.grpc.Index) */ { - public: - Index(); - virtual ~Index(); - - Index(const Index& from); - Index(Index&& from) noexcept - : Index() { - *this = ::std::move(from); - } - - inline Index& operator=(const Index& from) { - CopyFrom(from); - return *this; - } - inline Index& operator=(Index&& from) noexcept { - if (GetArenaNoVirtual() == from.GetArenaNoVirtual()) { - if (this != &from) InternalSwap(&from); - } else { - CopyFrom(from); - } - return *this; - } - - static const ::PROTOBUF_NAMESPACE_ID::Descriptor* descriptor() { - return GetDescriptor(); - } - static const ::PROTOBUF_NAMESPACE_ID::Descriptor* GetDescriptor() { - return GetMetadataStatic().descriptor; - } - static const ::PROTOBUF_NAMESPACE_ID::Reflection* GetReflection() { - return GetMetadataStatic().reflection; - } - static const Index& default_instance(); - - static void InitAsDefaultInstance(); // FOR INTERNAL USE ONLY - static inline const Index* internal_default_instance() { - return reinterpret_cast( - &_Index_default_instance_); - } - static constexpr int kIndexInFileMessages = - 16; - - friend void swap(Index& a, Index& b) { - a.Swap(&b); - } - inline void Swap(Index* other) { - if (other == this) return; - InternalSwap(other); - } - - // implements Message ---------------------------------------------- - - inline Index* New() const final { - return CreateMaybeMessage(nullptr); - } - - Index* New(::PROTOBUF_NAMESPACE_ID::Arena* arena) const final { - return CreateMaybeMessage(arena); - } - void CopyFrom(const ::PROTOBUF_NAMESPACE_ID::Message& from) final; - void MergeFrom(const ::PROTOBUF_NAMESPACE_ID::Message& from) final; - void CopyFrom(const Index& from); - void MergeFrom(const Index& from); - PROTOBUF_ATTRIBUTE_REINITIALIZES void Clear() final; - bool IsInitialized() const final; - - size_t ByteSizeLong() const final; - #if GOOGLE_PROTOBUF_ENABLE_EXPERIMENTAL_PARSER - const char* _InternalParse(const char* ptr, ::PROTOBUF_NAMESPACE_ID::internal::ParseContext* ctx) final; - #else - bool MergePartialFromCodedStream( - ::PROTOBUF_NAMESPACE_ID::io::CodedInputStream* input) final; - #endif // GOOGLE_PROTOBUF_ENABLE_EXPERIMENTAL_PARSER - void SerializeWithCachedSizes( - ::PROTOBUF_NAMESPACE_ID::io::CodedOutputStream* output) const final; - ::PROTOBUF_NAMESPACE_ID::uint8* InternalSerializeWithCachedSizesToArray( - ::PROTOBUF_NAMESPACE_ID::uint8* target) const final; - int GetCachedSize() const final { return _cached_size_.Get(); } - - private: - inline void SharedCtor(); - inline void SharedDtor(); - void SetCachedSize(int size) const final; - void InternalSwap(Index* other); - friend class ::PROTOBUF_NAMESPACE_ID::internal::AnyMetadata; - static ::PROTOBUF_NAMESPACE_ID::StringPiece FullMessageName() { - return "milvus.grpc.Index"; - } - private: - inline ::PROTOBUF_NAMESPACE_ID::Arena* GetArenaNoVirtual() const { - return nullptr; - } - inline void* MaybeArenaPtr() const { - return nullptr; - } - public: - - ::PROTOBUF_NAMESPACE_ID::Metadata GetMetadata() const final; - private: - static ::PROTOBUF_NAMESPACE_ID::Metadata GetMetadataStatic() { - ::PROTOBUF_NAMESPACE_ID::internal::AssignDescriptors(&::descriptor_table_milvus_2eproto); - return ::descriptor_table_milvus_2eproto.file_level_metadata[kIndexInFileMessages]; - } - - public: - - // nested types ---------------------------------------------------- - - // accessors ------------------------------------------------------- - - enum : int { - kIndexTypeFieldNumber = 1, - kNlistFieldNumber = 2, - }; - // int32 index_type = 1; - void clear_index_type(); - ::PROTOBUF_NAMESPACE_ID::int32 index_type() const; - void set_index_type(::PROTOBUF_NAMESPACE_ID::int32 value); - - // int32 nlist = 2; - void clear_nlist(); - ::PROTOBUF_NAMESPACE_ID::int32 nlist() const; - void set_nlist(::PROTOBUF_NAMESPACE_ID::int32 value); - - // @@protoc_insertion_point(class_scope:milvus.grpc.Index) - private: - class _Internal; - - ::PROTOBUF_NAMESPACE_ID::internal::InternalMetadataWithArena _internal_metadata_; - ::PROTOBUF_NAMESPACE_ID::int32 index_type_; - ::PROTOBUF_NAMESPACE_ID::int32 nlist_; - mutable ::PROTOBUF_NAMESPACE_ID::internal::CachedSize _cached_size_; - friend struct ::TableStruct_milvus_2eproto; -}; -// ------------------------------------------------------------------- - class IndexParam : public ::PROTOBUF_NAMESPACE_ID::Message /* @@protoc_insertion_point(class_definition:milvus.grpc.IndexParam) */ { public: @@ -2906,10 +2956,22 @@ class IndexParam : // accessors ------------------------------------------------------- enum : int { + kExtraParamsFieldNumber = 4, kTableNameFieldNumber = 2, kStatusFieldNumber = 1, - kIndexFieldNumber = 3, + kIndexTypeFieldNumber = 3, }; + // repeated .milvus.grpc.KeyValuePair extra_params = 4; + int extra_params_size() const; + void clear_extra_params(); + ::milvus::grpc::KeyValuePair* mutable_extra_params(int index); + ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::milvus::grpc::KeyValuePair >* + mutable_extra_params(); + const ::milvus::grpc::KeyValuePair& extra_params(int index) const; + ::milvus::grpc::KeyValuePair* add_extra_params(); + const ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::milvus::grpc::KeyValuePair >& + extra_params() const; + // string table_name = 2; void clear_table_name(); const std::string& table_name() const; @@ -2929,22 +2991,20 @@ class IndexParam : ::milvus::grpc::Status* mutable_status(); void set_allocated_status(::milvus::grpc::Status* status); - // .milvus.grpc.Index index = 3; - bool has_index() const; - void clear_index(); - const ::milvus::grpc::Index& index() const; - ::milvus::grpc::Index* release_index(); - ::milvus::grpc::Index* mutable_index(); - void set_allocated_index(::milvus::grpc::Index* index); + // int32 index_type = 3; + void clear_index_type(); + ::PROTOBUF_NAMESPACE_ID::int32 index_type() const; + void set_index_type(::PROTOBUF_NAMESPACE_ID::int32 value); // @@protoc_insertion_point(class_scope:milvus.grpc.IndexParam) private: class _Internal; ::PROTOBUF_NAMESPACE_ID::internal::InternalMetadataWithArena _internal_metadata_; + ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::milvus::grpc::KeyValuePair > extra_params_; ::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr table_name_; ::milvus::grpc::Status* status_; - ::milvus::grpc::Index* index_; + ::PROTOBUF_NAMESPACE_ID::int32 index_type_; mutable ::PROTOBUF_NAMESPACE_ID::internal::CachedSize _cached_size_; friend struct ::TableStruct_milvus_2eproto; }; @@ -4164,6 +4224,112 @@ class GetVectorIDsParam : #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wstrict-aliasing" #endif // __GNUC__ +// KeyValuePair + +// string key = 1; +inline void KeyValuePair::clear_key() { + key_.ClearToEmptyNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); +} +inline const std::string& KeyValuePair::key() const { + // @@protoc_insertion_point(field_get:milvus.grpc.KeyValuePair.key) + return key_.GetNoArena(); +} +inline void KeyValuePair::set_key(const std::string& value) { + + key_.SetNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), value); + // @@protoc_insertion_point(field_set:milvus.grpc.KeyValuePair.key) +} +inline void KeyValuePair::set_key(std::string&& value) { + + key_.SetNoArena( + &::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), ::std::move(value)); + // @@protoc_insertion_point(field_set_rvalue:milvus.grpc.KeyValuePair.key) +} +inline void KeyValuePair::set_key(const char* value) { + GOOGLE_DCHECK(value != nullptr); + + key_.SetNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), ::std::string(value)); + // @@protoc_insertion_point(field_set_char:milvus.grpc.KeyValuePair.key) +} +inline void KeyValuePair::set_key(const char* value, size_t size) { + + key_.SetNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), + ::std::string(reinterpret_cast(value), size)); + // @@protoc_insertion_point(field_set_pointer:milvus.grpc.KeyValuePair.key) +} +inline std::string* KeyValuePair::mutable_key() { + + // @@protoc_insertion_point(field_mutable:milvus.grpc.KeyValuePair.key) + return key_.MutableNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); +} +inline std::string* KeyValuePair::release_key() { + // @@protoc_insertion_point(field_release:milvus.grpc.KeyValuePair.key) + + return key_.ReleaseNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); +} +inline void KeyValuePair::set_allocated_key(std::string* key) { + if (key != nullptr) { + + } else { + + } + key_.SetAllocatedNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), key); + // @@protoc_insertion_point(field_set_allocated:milvus.grpc.KeyValuePair.key) +} + +// string value = 2; +inline void KeyValuePair::clear_value() { + value_.ClearToEmptyNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); +} +inline const std::string& KeyValuePair::value() const { + // @@protoc_insertion_point(field_get:milvus.grpc.KeyValuePair.value) + return value_.GetNoArena(); +} +inline void KeyValuePair::set_value(const std::string& value) { + + value_.SetNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), value); + // @@protoc_insertion_point(field_set:milvus.grpc.KeyValuePair.value) +} +inline void KeyValuePair::set_value(std::string&& value) { + + value_.SetNoArena( + &::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), ::std::move(value)); + // @@protoc_insertion_point(field_set_rvalue:milvus.grpc.KeyValuePair.value) +} +inline void KeyValuePair::set_value(const char* value) { + GOOGLE_DCHECK(value != nullptr); + + value_.SetNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), ::std::string(value)); + // @@protoc_insertion_point(field_set_char:milvus.grpc.KeyValuePair.value) +} +inline void KeyValuePair::set_value(const char* value, size_t size) { + + value_.SetNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), + ::std::string(reinterpret_cast(value), size)); + // @@protoc_insertion_point(field_set_pointer:milvus.grpc.KeyValuePair.value) +} +inline std::string* KeyValuePair::mutable_value() { + + // @@protoc_insertion_point(field_mutable:milvus.grpc.KeyValuePair.value) + return value_.MutableNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); +} +inline std::string* KeyValuePair::release_value() { + // @@protoc_insertion_point(field_release:milvus.grpc.KeyValuePair.value) + + return value_.ReleaseNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); +} +inline void KeyValuePair::set_allocated_value(std::string* value) { + if (value != nullptr) { + + } else { + + } + value_.SetAllocatedNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), value); + // @@protoc_insertion_point(field_set_allocated:milvus.grpc.KeyValuePair.value) +} + +// ------------------------------------------------------------------- + // TableName // string table_name = 1; @@ -4473,6 +4639,36 @@ inline void TableSchema::set_metric_type(::PROTOBUF_NAMESPACE_ID::int32 value) { // @@protoc_insertion_point(field_set:milvus.grpc.TableSchema.metric_type) } +// repeated .milvus.grpc.KeyValuePair extra_params = 6; +inline int TableSchema::extra_params_size() const { + return extra_params_.size(); +} +inline void TableSchema::clear_extra_params() { + extra_params_.Clear(); +} +inline ::milvus::grpc::KeyValuePair* TableSchema::mutable_extra_params(int index) { + // @@protoc_insertion_point(field_mutable:milvus.grpc.TableSchema.extra_params) + return extra_params_.Mutable(index); +} +inline ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::milvus::grpc::KeyValuePair >* +TableSchema::mutable_extra_params() { + // @@protoc_insertion_point(field_mutable_list:milvus.grpc.TableSchema.extra_params) + return &extra_params_; +} +inline const ::milvus::grpc::KeyValuePair& TableSchema::extra_params(int index) const { + // @@protoc_insertion_point(field_get:milvus.grpc.TableSchema.extra_params) + return extra_params_.Get(index); +} +inline ::milvus::grpc::KeyValuePair* TableSchema::add_extra_params() { + // @@protoc_insertion_point(field_add:milvus.grpc.TableSchema.extra_params) + return extra_params_.Add(); +} +inline const ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::milvus::grpc::KeyValuePair >& +TableSchema::extra_params() const { + // @@protoc_insertion_point(field_list:milvus.grpc.TableSchema.extra_params) + return extra_params_; +} + // ------------------------------------------------------------------- // PartitionParam @@ -4944,6 +5140,36 @@ inline void InsertParam::set_allocated_partition_tag(std::string* partition_tag) // @@protoc_insertion_point(field_set_allocated:milvus.grpc.InsertParam.partition_tag) } +// repeated .milvus.grpc.KeyValuePair extra_params = 5; +inline int InsertParam::extra_params_size() const { + return extra_params_.size(); +} +inline void InsertParam::clear_extra_params() { + extra_params_.Clear(); +} +inline ::milvus::grpc::KeyValuePair* InsertParam::mutable_extra_params(int index) { + // @@protoc_insertion_point(field_mutable:milvus.grpc.InsertParam.extra_params) + return extra_params_.Mutable(index); +} +inline ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::milvus::grpc::KeyValuePair >* +InsertParam::mutable_extra_params() { + // @@protoc_insertion_point(field_mutable_list:milvus.grpc.InsertParam.extra_params) + return &extra_params_; +} +inline const ::milvus::grpc::KeyValuePair& InsertParam::extra_params(int index) const { + // @@protoc_insertion_point(field_get:milvus.grpc.InsertParam.extra_params) + return extra_params_.Get(index); +} +inline ::milvus::grpc::KeyValuePair* InsertParam::add_extra_params() { + // @@protoc_insertion_point(field_add:milvus.grpc.InsertParam.extra_params) + return extra_params_.Add(); +} +inline const ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::milvus::grpc::KeyValuePair >& +InsertParam::extra_params() const { + // @@protoc_insertion_point(field_list:milvus.grpc.InsertParam.extra_params) + return extra_params_; +} + // ------------------------------------------------------------------- // VectorIds @@ -5078,65 +5304,7 @@ inline void SearchParam::set_allocated_table_name(std::string* table_name) { // @@protoc_insertion_point(field_set_allocated:milvus.grpc.SearchParam.table_name) } -// repeated .milvus.grpc.RowRecord query_record_array = 2; -inline int SearchParam::query_record_array_size() const { - return query_record_array_.size(); -} -inline void SearchParam::clear_query_record_array() { - query_record_array_.Clear(); -} -inline ::milvus::grpc::RowRecord* SearchParam::mutable_query_record_array(int index) { - // @@protoc_insertion_point(field_mutable:milvus.grpc.SearchParam.query_record_array) - return query_record_array_.Mutable(index); -} -inline ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::milvus::grpc::RowRecord >* -SearchParam::mutable_query_record_array() { - // @@protoc_insertion_point(field_mutable_list:milvus.grpc.SearchParam.query_record_array) - return &query_record_array_; -} -inline const ::milvus::grpc::RowRecord& SearchParam::query_record_array(int index) const { - // @@protoc_insertion_point(field_get:milvus.grpc.SearchParam.query_record_array) - return query_record_array_.Get(index); -} -inline ::milvus::grpc::RowRecord* SearchParam::add_query_record_array() { - // @@protoc_insertion_point(field_add:milvus.grpc.SearchParam.query_record_array) - return query_record_array_.Add(); -} -inline const ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::milvus::grpc::RowRecord >& -SearchParam::query_record_array() const { - // @@protoc_insertion_point(field_list:milvus.grpc.SearchParam.query_record_array) - return query_record_array_; -} - -// int64 topk = 3; -inline void SearchParam::clear_topk() { - topk_ = PROTOBUF_LONGLONG(0); -} -inline ::PROTOBUF_NAMESPACE_ID::int64 SearchParam::topk() const { - // @@protoc_insertion_point(field_get:milvus.grpc.SearchParam.topk) - return topk_; -} -inline void SearchParam::set_topk(::PROTOBUF_NAMESPACE_ID::int64 value) { - - topk_ = value; - // @@protoc_insertion_point(field_set:milvus.grpc.SearchParam.topk) -} - -// int64 nprobe = 4; -inline void SearchParam::clear_nprobe() { - nprobe_ = PROTOBUF_LONGLONG(0); -} -inline ::PROTOBUF_NAMESPACE_ID::int64 SearchParam::nprobe() const { - // @@protoc_insertion_point(field_get:milvus.grpc.SearchParam.nprobe) - return nprobe_; -} -inline void SearchParam::set_nprobe(::PROTOBUF_NAMESPACE_ID::int64 value) { - - nprobe_ = value; - // @@protoc_insertion_point(field_set:milvus.grpc.SearchParam.nprobe) -} - -// repeated string partition_tag_array = 5; +// repeated string partition_tag_array = 2; inline int SearchParam::partition_tag_array_size() const { return partition_tag_array_.size(); } @@ -5201,6 +5369,80 @@ SearchParam::mutable_partition_tag_array() { return &partition_tag_array_; } +// repeated .milvus.grpc.RowRecord query_record_array = 3; +inline int SearchParam::query_record_array_size() const { + return query_record_array_.size(); +} +inline void SearchParam::clear_query_record_array() { + query_record_array_.Clear(); +} +inline ::milvus::grpc::RowRecord* SearchParam::mutable_query_record_array(int index) { + // @@protoc_insertion_point(field_mutable:milvus.grpc.SearchParam.query_record_array) + return query_record_array_.Mutable(index); +} +inline ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::milvus::grpc::RowRecord >* +SearchParam::mutable_query_record_array() { + // @@protoc_insertion_point(field_mutable_list:milvus.grpc.SearchParam.query_record_array) + return &query_record_array_; +} +inline const ::milvus::grpc::RowRecord& SearchParam::query_record_array(int index) const { + // @@protoc_insertion_point(field_get:milvus.grpc.SearchParam.query_record_array) + return query_record_array_.Get(index); +} +inline ::milvus::grpc::RowRecord* SearchParam::add_query_record_array() { + // @@protoc_insertion_point(field_add:milvus.grpc.SearchParam.query_record_array) + return query_record_array_.Add(); +} +inline const ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::milvus::grpc::RowRecord >& +SearchParam::query_record_array() const { + // @@protoc_insertion_point(field_list:milvus.grpc.SearchParam.query_record_array) + return query_record_array_; +} + +// int64 topk = 4; +inline void SearchParam::clear_topk() { + topk_ = PROTOBUF_LONGLONG(0); +} +inline ::PROTOBUF_NAMESPACE_ID::int64 SearchParam::topk() const { + // @@protoc_insertion_point(field_get:milvus.grpc.SearchParam.topk) + return topk_; +} +inline void SearchParam::set_topk(::PROTOBUF_NAMESPACE_ID::int64 value) { + + topk_ = value; + // @@protoc_insertion_point(field_set:milvus.grpc.SearchParam.topk) +} + +// repeated .milvus.grpc.KeyValuePair extra_params = 5; +inline int SearchParam::extra_params_size() const { + return extra_params_.size(); +} +inline void SearchParam::clear_extra_params() { + extra_params_.Clear(); +} +inline ::milvus::grpc::KeyValuePair* SearchParam::mutable_extra_params(int index) { + // @@protoc_insertion_point(field_mutable:milvus.grpc.SearchParam.extra_params) + return extra_params_.Mutable(index); +} +inline ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::milvus::grpc::KeyValuePair >* +SearchParam::mutable_extra_params() { + // @@protoc_insertion_point(field_mutable_list:milvus.grpc.SearchParam.extra_params) + return &extra_params_; +} +inline const ::milvus::grpc::KeyValuePair& SearchParam::extra_params(int index) const { + // @@protoc_insertion_point(field_get:milvus.grpc.SearchParam.extra_params) + return extra_params_.Get(index); +} +inline ::milvus::grpc::KeyValuePair* SearchParam::add_extra_params() { + // @@protoc_insertion_point(field_add:milvus.grpc.SearchParam.extra_params) + return extra_params_.Add(); +} +inline const ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::milvus::grpc::KeyValuePair >& +SearchParam::extra_params() const { + // @@protoc_insertion_point(field_list:milvus.grpc.SearchParam.extra_params) + return extra_params_; +} + // ------------------------------------------------------------------- // SearchInFilesParam @@ -5376,49 +5618,7 @@ inline void SearchByIDParam::set_allocated_table_name(std::string* table_name) { // @@protoc_insertion_point(field_set_allocated:milvus.grpc.SearchByIDParam.table_name) } -// int64 id = 2; -inline void SearchByIDParam::clear_id() { - id_ = PROTOBUF_LONGLONG(0); -} -inline ::PROTOBUF_NAMESPACE_ID::int64 SearchByIDParam::id() const { - // @@protoc_insertion_point(field_get:milvus.grpc.SearchByIDParam.id) - return id_; -} -inline void SearchByIDParam::set_id(::PROTOBUF_NAMESPACE_ID::int64 value) { - - id_ = value; - // @@protoc_insertion_point(field_set:milvus.grpc.SearchByIDParam.id) -} - -// int64 topk = 3; -inline void SearchByIDParam::clear_topk() { - topk_ = PROTOBUF_LONGLONG(0); -} -inline ::PROTOBUF_NAMESPACE_ID::int64 SearchByIDParam::topk() const { - // @@protoc_insertion_point(field_get:milvus.grpc.SearchByIDParam.topk) - return topk_; -} -inline void SearchByIDParam::set_topk(::PROTOBUF_NAMESPACE_ID::int64 value) { - - topk_ = value; - // @@protoc_insertion_point(field_set:milvus.grpc.SearchByIDParam.topk) -} - -// int64 nprobe = 4; -inline void SearchByIDParam::clear_nprobe() { - nprobe_ = PROTOBUF_LONGLONG(0); -} -inline ::PROTOBUF_NAMESPACE_ID::int64 SearchByIDParam::nprobe() const { - // @@protoc_insertion_point(field_get:milvus.grpc.SearchByIDParam.nprobe) - return nprobe_; -} -inline void SearchByIDParam::set_nprobe(::PROTOBUF_NAMESPACE_ID::int64 value) { - - nprobe_ = value; - // @@protoc_insertion_point(field_set:milvus.grpc.SearchByIDParam.nprobe) -} - -// repeated string partition_tag_array = 5; +// repeated string partition_tag_array = 2; inline int SearchByIDParam::partition_tag_array_size() const { return partition_tag_array_.size(); } @@ -5483,6 +5683,64 @@ SearchByIDParam::mutable_partition_tag_array() { return &partition_tag_array_; } +// int64 id = 3; +inline void SearchByIDParam::clear_id() { + id_ = PROTOBUF_LONGLONG(0); +} +inline ::PROTOBUF_NAMESPACE_ID::int64 SearchByIDParam::id() const { + // @@protoc_insertion_point(field_get:milvus.grpc.SearchByIDParam.id) + return id_; +} +inline void SearchByIDParam::set_id(::PROTOBUF_NAMESPACE_ID::int64 value) { + + id_ = value; + // @@protoc_insertion_point(field_set:milvus.grpc.SearchByIDParam.id) +} + +// int64 topk = 4; +inline void SearchByIDParam::clear_topk() { + topk_ = PROTOBUF_LONGLONG(0); +} +inline ::PROTOBUF_NAMESPACE_ID::int64 SearchByIDParam::topk() const { + // @@protoc_insertion_point(field_get:milvus.grpc.SearchByIDParam.topk) + return topk_; +} +inline void SearchByIDParam::set_topk(::PROTOBUF_NAMESPACE_ID::int64 value) { + + topk_ = value; + // @@protoc_insertion_point(field_set:milvus.grpc.SearchByIDParam.topk) +} + +// repeated .milvus.grpc.KeyValuePair extra_params = 5; +inline int SearchByIDParam::extra_params_size() const { + return extra_params_.size(); +} +inline void SearchByIDParam::clear_extra_params() { + extra_params_.Clear(); +} +inline ::milvus::grpc::KeyValuePair* SearchByIDParam::mutable_extra_params(int index) { + // @@protoc_insertion_point(field_mutable:milvus.grpc.SearchByIDParam.extra_params) + return extra_params_.Mutable(index); +} +inline ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::milvus::grpc::KeyValuePair >* +SearchByIDParam::mutable_extra_params() { + // @@protoc_insertion_point(field_mutable_list:milvus.grpc.SearchByIDParam.extra_params) + return &extra_params_; +} +inline const ::milvus::grpc::KeyValuePair& SearchByIDParam::extra_params(int index) const { + // @@protoc_insertion_point(field_get:milvus.grpc.SearchByIDParam.extra_params) + return extra_params_.Get(index); +} +inline ::milvus::grpc::KeyValuePair* SearchByIDParam::add_extra_params() { + // @@protoc_insertion_point(field_add:milvus.grpc.SearchByIDParam.extra_params) + return extra_params_.Add(); +} +inline const ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::milvus::grpc::KeyValuePair >& +SearchByIDParam::extra_params() const { + // @@protoc_insertion_point(field_list:milvus.grpc.SearchByIDParam.extra_params) + return extra_params_; +} + // ------------------------------------------------------------------- // TopKQueryResult @@ -5889,38 +6147,6 @@ inline void Command::set_allocated_cmd(std::string* cmd) { // ------------------------------------------------------------------- -// Index - -// int32 index_type = 1; -inline void Index::clear_index_type() { - index_type_ = 0; -} -inline ::PROTOBUF_NAMESPACE_ID::int32 Index::index_type() const { - // @@protoc_insertion_point(field_get:milvus.grpc.Index.index_type) - return index_type_; -} -inline void Index::set_index_type(::PROTOBUF_NAMESPACE_ID::int32 value) { - - index_type_ = value; - // @@protoc_insertion_point(field_set:milvus.grpc.Index.index_type) -} - -// int32 nlist = 2; -inline void Index::clear_nlist() { - nlist_ = 0; -} -inline ::PROTOBUF_NAMESPACE_ID::int32 Index::nlist() const { - // @@protoc_insertion_point(field_get:milvus.grpc.Index.nlist) - return nlist_; -} -inline void Index::set_nlist(::PROTOBUF_NAMESPACE_ID::int32 value) { - - nlist_ = value; - // @@protoc_insertion_point(field_set:milvus.grpc.Index.nlist) -} - -// ------------------------------------------------------------------- - // IndexParam // .milvus.grpc.Status status = 1; @@ -6019,55 +6245,48 @@ inline void IndexParam::set_allocated_table_name(std::string* table_name) { // @@protoc_insertion_point(field_set_allocated:milvus.grpc.IndexParam.table_name) } -// .milvus.grpc.Index index = 3; -inline bool IndexParam::has_index() const { - return this != internal_default_instance() && index_ != nullptr; +// int32 index_type = 3; +inline void IndexParam::clear_index_type() { + index_type_ = 0; } -inline void IndexParam::clear_index() { - if (GetArenaNoVirtual() == nullptr && index_ != nullptr) { - delete index_; - } - index_ = nullptr; +inline ::PROTOBUF_NAMESPACE_ID::int32 IndexParam::index_type() const { + // @@protoc_insertion_point(field_get:milvus.grpc.IndexParam.index_type) + return index_type_; } -inline const ::milvus::grpc::Index& IndexParam::index() const { - const ::milvus::grpc::Index* p = index_; - // @@protoc_insertion_point(field_get:milvus.grpc.IndexParam.index) - return p != nullptr ? *p : *reinterpret_cast( - &::milvus::grpc::_Index_default_instance_); -} -inline ::milvus::grpc::Index* IndexParam::release_index() { - // @@protoc_insertion_point(field_release:milvus.grpc.IndexParam.index) +inline void IndexParam::set_index_type(::PROTOBUF_NAMESPACE_ID::int32 value) { - ::milvus::grpc::Index* temp = index_; - index_ = nullptr; - return temp; + index_type_ = value; + // @@protoc_insertion_point(field_set:milvus.grpc.IndexParam.index_type) } -inline ::milvus::grpc::Index* IndexParam::mutable_index() { - - if (index_ == nullptr) { - auto* p = CreateMaybeMessage<::milvus::grpc::Index>(GetArenaNoVirtual()); - index_ = p; - } - // @@protoc_insertion_point(field_mutable:milvus.grpc.IndexParam.index) - return index_; + +// repeated .milvus.grpc.KeyValuePair extra_params = 4; +inline int IndexParam::extra_params_size() const { + return extra_params_.size(); } -inline void IndexParam::set_allocated_index(::milvus::grpc::Index* index) { - ::PROTOBUF_NAMESPACE_ID::Arena* message_arena = GetArenaNoVirtual(); - if (message_arena == nullptr) { - delete index_; - } - if (index) { - ::PROTOBUF_NAMESPACE_ID::Arena* submessage_arena = nullptr; - if (message_arena != submessage_arena) { - index = ::PROTOBUF_NAMESPACE_ID::internal::GetOwnedMessage( - message_arena, index, submessage_arena); - } - - } else { - - } - index_ = index; - // @@protoc_insertion_point(field_set_allocated:milvus.grpc.IndexParam.index) +inline void IndexParam::clear_extra_params() { + extra_params_.Clear(); +} +inline ::milvus::grpc::KeyValuePair* IndexParam::mutable_extra_params(int index) { + // @@protoc_insertion_point(field_mutable:milvus.grpc.IndexParam.extra_params) + return extra_params_.Mutable(index); +} +inline ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::milvus::grpc::KeyValuePair >* +IndexParam::mutable_extra_params() { + // @@protoc_insertion_point(field_mutable_list:milvus.grpc.IndexParam.extra_params) + return &extra_params_; +} +inline const ::milvus::grpc::KeyValuePair& IndexParam::extra_params(int index) const { + // @@protoc_insertion_point(field_get:milvus.grpc.IndexParam.extra_params) + return extra_params_.Get(index); +} +inline ::milvus::grpc::KeyValuePair* IndexParam::add_extra_params() { + // @@protoc_insertion_point(field_add:milvus.grpc.IndexParam.extra_params) + return extra_params_.Add(); +} +inline const ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::milvus::grpc::KeyValuePair >& +IndexParam::extra_params() const { + // @@protoc_insertion_point(field_list:milvus.grpc.IndexParam.extra_params) + return extra_params_; } // ------------------------------------------------------------------- diff --git a/core/src/grpc/milvus.proto b/core/src/grpc/milvus.proto index ffc290d93c..7a7393cef9 100644 --- a/core/src/grpc/milvus.proto +++ b/core/src/grpc/milvus.proto @@ -4,6 +4,14 @@ import "status.proto"; package milvus.grpc; +/** + * @brief general usage + */ +message KeyValuePair { + string key = 1; + string value = 2; +} + /** * @brief Table name */ @@ -21,6 +29,7 @@ message TableNameList { /** * @brief Table schema + * metric_type: 1-L2, 2-IP */ message TableSchema { Status status = 1; @@ -28,6 +37,7 @@ message TableSchema { int64 dimension = 3; int64 index_file_size = 4; int32 metric_type = 5; + repeated KeyValuePair extra_params = 6; } /** @@ -62,6 +72,7 @@ message InsertParam { repeated RowRecord row_record_array = 2; repeated int64 row_id_array = 3; //optional string partition_tag = 4; + repeated KeyValuePair extra_params = 5; } /** @@ -77,10 +88,10 @@ message VectorIds { */ message SearchParam { string table_name = 1; - repeated RowRecord query_record_array = 2; - int64 topk = 3; - int64 nprobe = 4; - repeated string partition_tag_array = 5; + repeated string partition_tag_array = 2; + repeated RowRecord query_record_array = 3; + int64 topk = 4; + repeated KeyValuePair extra_params = 5; } /** @@ -96,10 +107,10 @@ message SearchInFilesParam { */ message SearchByIDParam { string table_name = 1; - int64 id = 2; - int64 topk = 3; - int64 nprobe = 4; - repeated string partition_tag_array = 5; + repeated string partition_tag_array = 2; + int64 id = 3; + int64 topk = 4; + repeated KeyValuePair extra_params = 5; } /** @@ -143,23 +154,15 @@ message Command { string cmd = 1; } -/** - * @brief Index - * @index_type: 0-invalid, 1-idmap, 2-ivflat, 3-ivfsq8, 4-nsgmix - * @metric_type: 1-L2, 2-IP - */ -message Index { - int32 index_type = 1; - int32 nlist = 2; -} - /** * @brief Index params + * @index_type: 0-invalid, 1-idmap, 2-ivflat, 3-ivfsq8, 4-nsgmix */ message IndexParam { Status status = 1; string table_name = 2; - Index index = 3; + int32 index_type = 3; + repeated KeyValuePair extra_params = 4; } /** diff --git a/core/src/index/knowhere/CMakeLists.txt b/core/src/index/knowhere/CMakeLists.txt index 646c0fddf0..1aae53d0d4 100644 --- a/core/src/index/knowhere/CMakeLists.txt +++ b/core/src/index/knowhere/CMakeLists.txt @@ -22,7 +22,6 @@ endif () set(external_srcs knowhere/adapter/SptagAdapter.cpp - knowhere/adapter/VectorAdapter.cpp knowhere/common/Exception.cpp knowhere/common/Timer.cpp ) @@ -117,4 +116,4 @@ set(INDEX_INCLUDE_DIRS ${LAPACK_INCLUDE_DIR} ) -set(INDEX_INCLUDE_DIRS ${INDEX_INCLUDE_DIRS} PARENT_SCOPE) \ No newline at end of file +set(INDEX_INCLUDE_DIRS ${INDEX_INCLUDE_DIRS} PARENT_SCOPE) diff --git a/core/src/index/knowhere/knowhere/adapter/SptagAdapter.cpp b/core/src/index/knowhere/knowhere/adapter/SptagAdapter.cpp index f6a78dcd72..c31e262dd8 100644 --- a/core/src/index/knowhere/knowhere/adapter/SptagAdapter.cpp +++ b/core/src/index/knowhere/knowhere/adapter/SptagAdapter.cpp @@ -43,7 +43,8 @@ std::vector ConvertToQueryResult(const DatasetPtr& dataset, const Config& config) { GETTENSOR(dataset); - std::vector query_results(rows, SPTAG::QueryResult(nullptr, config->k, true)); + std::vector query_results(rows, + SPTAG::QueryResult(nullptr, config[meta::TOPK].get(), true)); for (auto i = 0; i < rows; ++i) { query_results[i].SetTarget(&p_data[i * dim]); } diff --git a/core/src/index/knowhere/knowhere/adapter/VectorAdapter.cpp b/core/src/index/knowhere/knowhere/adapter/VectorAdapter.cpp deleted file mode 100644 index 75c2809e90..0000000000 --- a/core/src/index/knowhere/knowhere/adapter/VectorAdapter.cpp +++ /dev/null @@ -1,24 +0,0 @@ -// Copyright (C) 2019-2020 Zilliz. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software distributed under the License -// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -// or implied. See the License for the specific language governing permissions and limitations under the License. - -#include "knowhere/adapter/VectorAdapter.h" - -namespace knowhere { - -namespace meta { -const char* DIM = "dim"; -const char* TENSOR = "tensor"; -const char* ROWS = "rows"; -const char* IDS = "ids"; -const char* DISTANCE = "distance"; -}; // namespace meta - -} // namespace knowhere diff --git a/core/src/index/knowhere/knowhere/adapter/VectorAdapter.h b/core/src/index/knowhere/knowhere/adapter/VectorAdapter.h index e675857453..ae89038855 100644 --- a/core/src/index/knowhere/knowhere/adapter/VectorAdapter.h +++ b/core/src/index/knowhere/knowhere/adapter/VectorAdapter.h @@ -13,17 +13,10 @@ #include #include "knowhere/common/Dataset.h" +#include "knowhere/index/vector_index/helpers/IndexParameter.h" namespace knowhere { -namespace meta { -extern const char* DIM; -extern const char* TENSOR; -extern const char* ROWS; -extern const char* IDS; -extern const char* DISTANCE; -}; // namespace meta - #define GETTENSOR(dataset) \ auto dim = dataset->Get(meta::DIM); \ auto rows = dataset->Get(meta::ROWS); \ diff --git a/core/src/index/knowhere/knowhere/common/Config.h b/core/src/index/knowhere/knowhere/common/Config.h index b918bf0946..4ab9ac0e67 100644 --- a/core/src/index/knowhere/knowhere/common/Config.h +++ b/core/src/index/knowhere/knowhere/common/Config.h @@ -11,64 +11,10 @@ #pragma once -#include -#include -#include "Log.h" -#include "knowhere/common/Exception.h" +#include "src/utils/Json.h" namespace knowhere { -enum class METRICTYPE { - INVALID = 0, - L2 = 1, - IP = 2, - HAMMING = 20, - JACCARD = 21, - TANIMOTO = 22, -}; - -// General Config -constexpr int64_t INVALID_VALUE = -1; -constexpr int64_t DEFAULT_K = INVALID_VALUE; -constexpr int64_t DEFAULT_DIM = INVALID_VALUE; -constexpr int64_t DEFAULT_GPUID = INVALID_VALUE; -constexpr METRICTYPE DEFAULT_TYPE = METRICTYPE::INVALID; - -struct Cfg { - METRICTYPE metric_type = DEFAULT_TYPE; - int64_t k = DEFAULT_K; - int64_t gpu_id = DEFAULT_GPUID; - int64_t d = DEFAULT_DIM; - - Cfg(const int64_t& dim, const int64_t& k, const int64_t& gpu_id, METRICTYPE type) - : metric_type(type), k(k), gpu_id(gpu_id), d(dim) { - } - - Cfg() = default; - - virtual bool - CheckValid() { - if (metric_type == METRICTYPE::IP || metric_type == METRICTYPE::L2) { - return true; - } - std::stringstream ss; - ss << "MetricType: " << int(metric_type) << " not support!"; - KNOWHERE_THROW_MSG(ss.str()); - return false; - } - - void - Dump() { - KNOWHERE_LOG_DEBUG << DumpImpl().str(); - } - - virtual std::stringstream - DumpImpl() { - std::stringstream ss; - ss << "dim: " << d << ", metric: " << int(metric_type) << ", gpuid: " << gpu_id << ", k: " << k; - return ss; - } -}; -using Config = std::shared_ptr; +using Config = milvus::json; } // namespace knowhere diff --git a/core/src/index/knowhere/knowhere/index/Index.h b/core/src/index/knowhere/knowhere/index/Index.h index 01f3233bc7..7fbcdeb81a 100644 --- a/core/src/index/knowhere/knowhere/index/Index.h +++ b/core/src/index/knowhere/knowhere/index/Index.h @@ -16,6 +16,7 @@ #include "IndexModel.h" #include "IndexType.h" #include "knowhere/common/BinarySet.h" +#include "knowhere/common/Config.h" #include "knowhere/common/Dataset.h" #include "knowhere/index/preprocessor/Preprocessor.h" diff --git a/core/src/index/knowhere/knowhere/index/vector_index/FaissBaseIndex.cpp b/core/src/index/knowhere/knowhere/index/vector_index/FaissBaseIndex.cpp index 2ccbb7ba8d..fbfcdf8ac9 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/FaissBaseIndex.cpp +++ b/core/src/index/knowhere/knowhere/index/vector_index/FaissBaseIndex.cpp @@ -14,6 +14,7 @@ #include #include "knowhere/common/Exception.h" +#include "knowhere/common/Log.h" #include "knowhere/index/vector_index/FaissBaseIndex.h" #include "knowhere/index/vector_index/IndexIVF.h" #include "knowhere/index/vector_index/helpers/FaissIO.h" diff --git a/core/src/index/knowhere/knowhere/index/vector_index/IndexBinaryIDMAP.cpp b/core/src/index/knowhere/knowhere/index/vector_index/IndexBinaryIDMAP.cpp index cf0cc8d4bc..bf60f2deee 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/IndexBinaryIDMAP.cpp +++ b/core/src/index/knowhere/knowhere/index/vector_index/IndexBinaryIDMAP.cpp @@ -15,6 +15,8 @@ #include #include +#include + #include "knowhere/adapter/VectorAdapter.h" #include "knowhere/common/Exception.h" @@ -43,13 +45,13 @@ BinaryIDMAP::Search(const DatasetPtr& dataset, const Config& config) { } GETBINARYTENSOR(dataset) - auto elems = rows * config->k; + auto elems = rows * config[meta::TOPK].get(); size_t p_id_size = sizeof(int64_t) * elems; size_t p_dist_size = sizeof(float) * elems; auto p_id = (int64_t*)malloc(p_id_size); auto p_dist = (float*)malloc(p_dist_size); - search_impl(rows, (uint8_t*)p_data, config->k, p_dist, p_id, Config()); + search_impl(rows, (uint8_t*)p_data, config[meta::TOPK].get(), p_dist, p_id, Config()); auto ret_ds = std::make_shared(); if (index_->metric_type == faiss::METRIC_Hamming) { @@ -90,14 +92,9 @@ BinaryIDMAP::Add(const DatasetPtr& dataset, const Config& config) { void BinaryIDMAP::Train(const Config& config) { - auto build_cfg = std::dynamic_pointer_cast(config); - if (build_cfg == nullptr) { - KNOWHERE_THROW_MSG("not support this kind of config"); - } - config->CheckValid(); - const char* type = "BFlat"; - auto index = faiss::index_binary_factory(config->d, type, GetMetricType(config->metric_type)); + auto index = faiss::index_binary_factory(config[meta::DIM].get(), type, + GetMetricType(config[Metric::TYPE].get())); index_.reset(index); } @@ -181,26 +178,18 @@ BinaryIDMAP::SearchById(const DatasetPtr& dataset, const Config& config) { KNOWHERE_THROW_MSG("index not initialize"); } - // auto search_cfg = std::dynamic_pointer_cast(config); - // if (search_cfg == nullptr) { - // KNOWHERE_THROW_MSG("not support this kind of config"); - // } - - // GETBINARYTENSOR(dataset) auto dim = dataset->Get(meta::DIM); auto rows = dataset->Get(meta::ROWS); auto p_data = dataset->Get(meta::IDS); - auto elems = rows * config->k; + auto elems = rows * config[meta::TOPK].get(); size_t p_id_size = sizeof(int64_t) * elems; size_t p_dist_size = sizeof(float) * elems; auto p_id = (int64_t*)malloc(p_id_size); auto p_dist = (float*)malloc(p_dist_size); auto* pdistances = (int32_t*)p_dist; - // index_->searchById(rows, (uint8_t*)p_data, config->k, pdistances, p_id, bitset_); - // auto blacklist = dataset->Get("bitset"); - index_->search_by_id(rows, p_data, config->k, pdistances, p_id, bitset_); + index_->search_by_id(rows, p_data, config[meta::TOPK].get(), pdistances, p_id, bitset_); auto ret_ds = std::make_shared(); if (index_->metric_type == faiss::METRIC_Hamming) { diff --git a/core/src/index/knowhere/knowhere/index/vector_index/IndexBinaryIVF.cpp b/core/src/index/knowhere/knowhere/index/vector_index/IndexBinaryIVF.cpp index 0ad1445b74..f52b919e6b 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/IndexBinaryIVF.cpp +++ b/core/src/index/knowhere/knowhere/index/vector_index/IndexBinaryIVF.cpp @@ -15,9 +15,11 @@ #include #include +#include #include "knowhere/adapter/VectorAdapter.h" #include "knowhere/common/Exception.h" +#include "knowhere/common/Log.h" namespace knowhere { @@ -45,22 +47,17 @@ BinaryIVF::Search(const DatasetPtr& dataset, const Config& config) { KNOWHERE_THROW_MSG("index not initialize or trained"); } - // auto search_cfg = std::dynamic_pointer_cast(config); - // if (search_cfg == nullptr) { - // KNOWHERE_THROW_MSG("not support this kind of config"); - // } - GETBINARYTENSOR(dataset) try { - auto elems = rows * config->k; + auto elems = rows * config[meta::TOPK].get(); size_t p_id_size = sizeof(int64_t) * elems; size_t p_dist_size = sizeof(float) * elems; auto p_id = (int64_t*)malloc(p_id_size); auto p_dist = (float*)malloc(p_dist_size); - search_impl(rows, (uint8_t*)p_data, config->k, p_dist, p_id, config); + search_impl(rows, (uint8_t*)p_data, config[meta::TOPK].get(), p_dist, p_id, config); auto ret_ds = std::make_shared(); if (index_->metric_type == faiss::METRIC_Hamming) { @@ -108,29 +105,20 @@ BinaryIVF::search_impl(int64_t n, const uint8_t* data, int64_t k, float* distanc std::shared_ptr BinaryIVF::GenParams(const Config& config) { auto params = std::make_shared(); - - auto search_cfg = std::dynamic_pointer_cast(config); - params->nprobe = search_cfg->nprobe; - // params->max_codes = config.get_with_default("max_codes", size_t(0)); - + params->nprobe = config[IndexParams::nprobe]; + // params->max_codes = config["max_code"]; return params; } IndexModelPtr BinaryIVF::Train(const DatasetPtr& dataset, const Config& config) { - std::lock_guard lk(mutex_); - - auto build_cfg = std::dynamic_pointer_cast(config); - if (build_cfg != nullptr) { - build_cfg->CheckValid(); // throw exception - } - GETBINARYTENSOR(dataset) auto p_ids = dataset->Get(meta::IDS); - faiss::IndexBinary* coarse_quantizer = new faiss::IndexBinaryFlat(dim, GetMetricType(build_cfg->metric_type)); - auto index = std::make_shared(coarse_quantizer, dim, build_cfg->nlist, - GetMetricType(build_cfg->metric_type)); + faiss::IndexBinary* coarse_quantizer = + new faiss::IndexBinaryFlat(dim, GetMetricType(config[Metric::TYPE].get())); + auto index = std::make_shared(coarse_quantizer, dim, config[IndexParams::nlist], + GetMetricType(config[Metric::TYPE].get())); index->train(rows, (uint8_t*)p_data); index->add_with_ids(rows, (uint8_t*)p_data, p_ids); index_ = index; @@ -190,17 +178,11 @@ BinaryIVF::SearchById(const DatasetPtr& dataset, const Config& config) { KNOWHERE_THROW_MSG("index not initialize or trained"); } - // auto search_cfg = std::dynamic_pointer_cast(config); - // if (search_cfg == nullptr) { - // KNOWHERE_THROW_MSG("not support this kind of config"); - // } - - // GETBINARYTENSOR(dataset) auto rows = dataset->Get(meta::ROWS); auto p_data = dataset->Get(meta::IDS); try { - auto elems = rows * config->k; + auto elems = rows * config[meta::TOPK].get(); size_t p_id_size = sizeof(int64_t) * elems; size_t p_dist_size = sizeof(float) * elems; @@ -208,9 +190,7 @@ BinaryIVF::SearchById(const DatasetPtr& dataset, const Config& config) { auto p_dist = (float*)malloc(p_dist_size); int32_t* pdistances = (int32_t*)p_dist; - // auto blacklist = dataset->Get("bitset"); - // index_->searchById(rows, (uint8_t*)p_data, config->k, pdistances, p_id, blacklist); - index_->search_by_id(rows, p_data, config->k, pdistances, p_id, bitset_); + index_->search_by_id(rows, p_data, config[meta::TOPK].get(), pdistances, p_id, bitset_); auto ret_ds = std::make_shared(); if (index_->metric_type == faiss::METRIC_Hamming) { diff --git a/core/src/index/knowhere/knowhere/index/vector_index/IndexGPUIDMAP.cpp b/core/src/index/knowhere/knowhere/index/vector_index/IndexGPUIDMAP.cpp index c1d376229a..d659f8fef4 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/IndexGPUIDMAP.cpp +++ b/core/src/index/knowhere/knowhere/index/vector_index/IndexGPUIDMAP.cpp @@ -16,6 +16,7 @@ #include #include #include +#include #ifdef MILVUS_GPU_VERSION @@ -127,7 +128,7 @@ GPUIDMAP::GenGraph(const float* data, const int64_t& k, Graph& graph, const Conf int64_t K = k + 1; auto ntotal = Count(); - size_t dim = config->d; + size_t dim = config[meta::DIM]; auto batch_size = 1000; auto tail_batch_size = ntotal % batch_size; auto batch_search_count = ntotal / batch_size; diff --git a/core/src/index/knowhere/knowhere/index/vector_index/IndexGPUIVF.cpp b/core/src/index/knowhere/knowhere/index/vector_index/IndexGPUIVF.cpp index 99c84d492a..1feb08e4f1 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/IndexGPUIVF.cpp +++ b/core/src/index/knowhere/knowhere/index/vector_index/IndexGPUIVF.cpp @@ -10,6 +10,7 @@ // or implied. See the License for the specific language governing permissions and limitations under the License. #include +#include #include #include @@ -28,21 +29,16 @@ namespace knowhere { IndexModelPtr GPUIVF::Train(const DatasetPtr& dataset, const Config& config) { - auto build_cfg = std::dynamic_pointer_cast(config); - if (build_cfg != nullptr) { - build_cfg->CheckValid(); // throw exception - } - gpu_id_ = build_cfg->gpu_id; - GETTENSOR(dataset) + gpu_id_ = config[knowhere::meta::DEVICEID]; auto temp_resource = FaissGpuResourceMgr::GetInstance().GetRes(gpu_id_); if (temp_resource != nullptr) { ResScope rs(temp_resource, gpu_id_, true); faiss::gpu::GpuIndexIVFFlatConfig idx_config; idx_config.device = gpu_id_; - faiss::gpu::GpuIndexIVFFlat device_index(temp_resource->faiss_res.get(), dim, build_cfg->nlist, - GetMetricType(build_cfg->metric_type), idx_config); + faiss::gpu::GpuIndexIVFFlat device_index(temp_resource->faiss_res.get(), dim, config[IndexParams::nlist], + GetMetricType(config[Metric::TYPE].get()), idx_config); device_index.train(rows, (float*)p_data); std::shared_ptr host_index = nullptr; @@ -121,15 +117,13 @@ GPUIVF::LoadImpl(const BinarySet& index_binary) { } void -GPUIVF::search_impl(int64_t n, const float* data, int64_t k, float* distances, int64_t* labels, const Config& cfg) { +GPUIVF::search_impl(int64_t n, const float* data, int64_t k, float* distances, int64_t* labels, const Config& config) { std::lock_guard lk(mutex_); auto device_index = std::dynamic_pointer_cast(index_); fiu_do_on("GPUIVF.search_impl.invald_index", device_index = nullptr); if (device_index) { - auto search_cfg = std::dynamic_pointer_cast(cfg); - device_index->nprobe = search_cfg->nprobe; - // assert(device_index->getNumProbes() == search_cfg->nprobe); + device_index->nprobe = config[IndexParams::nprobe]; ResScope rs(res_, gpu_id_); device_index->search(n, (float*)data, k, distances, labels); } else { diff --git a/core/src/index/knowhere/knowhere/index/vector_index/IndexGPUIVFPQ.cpp b/core/src/index/knowhere/knowhere/index/vector_index/IndexGPUIVFPQ.cpp index 051a562b71..dfe3602931 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/IndexGPUIVFPQ.cpp +++ b/core/src/index/knowhere/knowhere/index/vector_index/IndexGPUIVFPQ.cpp @@ -15,6 +15,7 @@ #include #include +#include #include "knowhere/adapter/VectorAdapter.h" #include "knowhere/common/Exception.h" @@ -25,20 +26,16 @@ namespace knowhere { IndexModelPtr GPUIVFPQ::Train(const DatasetPtr& dataset, const Config& config) { - auto build_cfg = std::dynamic_pointer_cast(config); - if (build_cfg != nullptr) { - build_cfg->CheckValid(); // throw exception - } - gpu_id_ = build_cfg->gpu_id; - GETTENSOR(dataset) + gpu_id_ = config[knowhere::meta::DEVICEID]; auto temp_resource = FaissGpuResourceMgr::GetInstance().GetRes(gpu_id_); if (temp_resource != nullptr) { ResScope rs(temp_resource, gpu_id_, true); - auto device_index = new faiss::gpu::GpuIndexIVFPQ(temp_resource->faiss_res.get(), dim, build_cfg->nlist, - build_cfg->m, build_cfg->nbits, - GetMetricType(build_cfg->metric_type)); // IP not support + auto device_index = new faiss::gpu::GpuIndexIVFPQ( + temp_resource->faiss_res.get(), dim, config[IndexParams::nlist].get(), config[IndexParams::m], + config[IndexParams::nbits], + GetMetricType(config[Metric::TYPE].get())); // IP not support device_index->train(rows, (float*)p_data); std::shared_ptr host_index = nullptr; host_index.reset(faiss::gpu::index_gpu_to_cpu(device_index)); @@ -51,11 +48,10 @@ GPUIVFPQ::Train(const DatasetPtr& dataset, const Config& config) { std::shared_ptr GPUIVFPQ::GenParams(const Config& config) { auto params = std::make_shared(); - auto search_cfg = std::dynamic_pointer_cast(config); - params->nprobe = search_cfg->nprobe; - // params->scan_table_threshold = conf->scan_table_threhold; - // params->polysemous_ht = conf->polysemous_ht; - // params->max_codes = conf->max_codes; + params->nprobe = config[IndexParams::nprobe]; + // params->scan_table_threshold = config["scan_table_threhold"] + // params->polysemous_ht = config["polysemous_ht"] + // params->max_codes = config["max_codes"] return params; } diff --git a/core/src/index/knowhere/knowhere/index/vector_index/IndexGPUIVFSQ.cpp b/core/src/index/knowhere/knowhere/index/vector_index/IndexGPUIVFSQ.cpp index f9449dc34a..f1390c0fcb 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/IndexGPUIVFSQ.cpp +++ b/core/src/index/knowhere/knowhere/index/vector_index/IndexGPUIVFSQ.cpp @@ -13,6 +13,7 @@ #include #include +#include #include "knowhere/adapter/VectorAdapter.h" #include "knowhere/common/Exception.h" @@ -23,18 +24,14 @@ namespace knowhere { IndexModelPtr GPUIVFSQ::Train(const DatasetPtr& dataset, const Config& config) { - auto build_cfg = std::dynamic_pointer_cast(config); - if (build_cfg != nullptr) { - build_cfg->CheckValid(); // throw exception - } - gpu_id_ = build_cfg->gpu_id; - GETTENSOR(dataset) + gpu_id_ = config[knowhere::meta::DEVICEID]; std::stringstream index_type; - index_type << "IVF" << build_cfg->nlist << "," - << "SQ" << build_cfg->nbits; - auto build_index = faiss::index_factory(dim, index_type.str().c_str(), GetMetricType(build_cfg->metric_type)); + index_type << "IVF" << config[IndexParams::nlist] << "," + << "SQ" << config[IndexParams::nbits]; + auto build_index = + faiss::index_factory(dim, index_type.str().c_str(), GetMetricType(config[Metric::TYPE].get())); auto temp_resource = FaissGpuResourceMgr::GetInstance().GetRes(gpu_id_); if (temp_resource != nullptr) { diff --git a/core/src/index/knowhere/knowhere/index/vector_index/IndexHNSW.cpp b/core/src/index/knowhere/knowhere/index/vector_index/IndexHNSW.cpp index 3c05858be2..596de9e8c5 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/IndexHNSW.cpp +++ b/core/src/index/knowhere/knowhere/index/vector_index/IndexHNSW.cpp @@ -79,20 +79,15 @@ IndexHNSW::Search(const DatasetPtr& dataset, const Config& config) { if (!index_) { KNOWHERE_THROW_MSG("index not initialize or trained"); } - - auto search_cfg = std::dynamic_pointer_cast(config); - if (search_cfg == nullptr) { - KNOWHERE_THROW_MSG("search conf is null"); - } - index_->setEf(search_cfg->ef); - GETTENSOR(dataset) - size_t id_size = sizeof(int64_t) * config->k; - size_t dist_size = sizeof(float) * config->k; + size_t id_size = sizeof(int64_t) * config[meta::TOPK].get(); + size_t dist_size = sizeof(float) * config[meta::TOPK].get(); auto p_id = (int64_t*)malloc(id_size * rows); auto p_dist = (float*)malloc(dist_size * rows); + index_->setEf(config[IndexParams::ef]); + using P = std::pair; auto compare = [](const P& v1, const P& v2) { return v1.first < v2.first; }; #pragma omp parallel for @@ -103,13 +98,13 @@ IndexHNSW::Search(const DatasetPtr& dataset, const Config& config) { // if (normalize) { // std::vector norm_vector(Dimension()); // normalize_vector((float*)(single_query), norm_vector.data(), Dimension()); - // ret = index_->searchKnn((float*)(norm_vector.data()), config->k, compare); + // ret = index_->searchKnn((float*)(norm_vector.data()), config[meta::TOPK].get(), compare); // } else { - // ret = index_->searchKnn((float*)single_query, config->k, compare); + // ret = index_->searchKnn((float*)single_query, config[meta::TOPK].get(), compare); // } - ret = index_->searchKnn((float*)single_query, config->k, compare); + ret = index_->searchKnn((float*)single_query, config[meta::TOPK].get(), compare); - while (ret.size() < config->k) { + while (ret.size() < config[meta::TOPK]) { ret.push_back(std::make_pair(-1, -1)); } std::vector dist; @@ -125,8 +120,8 @@ IndexHNSW::Search(const DatasetPtr& dataset, const Config& config) { std::transform(ret.begin(), ret.end(), std::back_inserter(ids), [](const std::pair& e) { return e.second; }); - memcpy(p_dist + i * config->k, dist.data(), dist_size); - memcpy(p_id + i * config->k, ids.data(), id_size); + memcpy(p_dist + i * config[meta::TOPK].get(), dist.data(), dist_size); + memcpy(p_id + i * config[meta::TOPK].get(), ids.data(), id_size); } auto ret_ds = std::make_shared(); @@ -137,21 +132,17 @@ IndexHNSW::Search(const DatasetPtr& dataset, const Config& config) { IndexModelPtr IndexHNSW::Train(const DatasetPtr& dataset, const Config& config) { - auto build_cfg = std::dynamic_pointer_cast(config); - if (build_cfg == nullptr) { - KNOWHERE_THROW_MSG("build conf is null"); - } - GETTENSOR(dataset) hnswlib::SpaceInterface* space; - if (config->metric_type == METRICTYPE::L2) { + if (config[Metric::TYPE] == Metric::L2) { space = new hnswlib::L2Space(dim); - } else if (config->metric_type == METRICTYPE::IP) { + } else if (config[Metric::TYPE] == Metric::IP) { space = new hnswlib::InnerProductSpace(dim); normalize = true; } - index_ = std::make_shared>(space, rows, build_cfg->M, build_cfg->ef); + index_ = std::make_shared>(space, rows, config[IndexParams::M].get(), + config[IndexParams::efConstruction].get()); return nullptr; } diff --git a/core/src/index/knowhere/knowhere/index/vector_index/IndexIDMAP.cpp b/core/src/index/knowhere/knowhere/index/vector_index/IndexIDMAP.cpp index 3333d97f5f..6cac1539d2 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/IndexIDMAP.cpp +++ b/core/src/index/knowhere/knowhere/index/vector_index/IndexIDMAP.cpp @@ -22,6 +22,7 @@ #endif +#include #include #include "knowhere/adapter/VectorAdapter.h" @@ -61,13 +62,13 @@ IDMAP::Search(const DatasetPtr& dataset, const Config& config) { } GETTENSOR(dataset) - auto elems = rows * config->k; + auto elems = rows * config[meta::TOPK].get(); size_t p_id_size = sizeof(int64_t) * elems; size_t p_dist_size = sizeof(float) * elems; auto p_id = (int64_t*)malloc(p_id_size); auto p_dist = (float*)malloc(p_dist_size); - search_impl(rows, (float*)p_data, config->k, p_dist, p_id, Config()); + search_impl(rows, (float*)p_data, config[meta::TOPK].get(), p_dist, p_id, Config()); auto ret_ds = std::make_shared(); ret_ds->Set(meta::IDS, p_id); @@ -144,10 +145,9 @@ IDMAP::GetRawIds() { void IDMAP::Train(const Config& config) { - config->CheckValid(); - const char* type = "IDMap,Flat"; - auto index = faiss::index_factory(config->d, type, GetMetricType(config->metric_type)); + auto index = faiss::index_factory(config[meta::DIM].get(), type, + GetMetricType(config[Metric::TYPE].get())); index_.reset(index); } @@ -214,7 +214,7 @@ IDMAP::SearchById(const DatasetPtr& dataset, const Config& config) { auto rows = dataset->Get(meta::ROWS); auto p_data = dataset->Get(meta::IDS); - auto elems = rows * config->k; + auto elems = rows * config[meta::TOPK].get(); size_t p_id_size = sizeof(int64_t) * elems; size_t p_dist_size = sizeof(float) * elems; auto p_id = (int64_t*)malloc(p_id_size); @@ -222,8 +222,8 @@ IDMAP::SearchById(const DatasetPtr& dataset, const Config& config) { // todo: enable search by id (zhiru) // auto blacklist = dataset->Get("bitset"); - // index_->searchById(rows, (float*)p_data, config->k, p_dist, p_id, blacklist); - index_->search_by_id(rows, p_data, config->k, p_dist, p_id, bitset_); + // index_->searchById(rows, (float*)p_data, config[meta::TOPK].get(), p_dist, p_id, blacklist); + index_->search_by_id(rows, p_data, config[meta::TOPK].get(), p_dist, p_id, bitset_); auto ret_ds = std::make_shared(); ret_ds->Set(meta::IDS, p_id); diff --git a/core/src/index/knowhere/knowhere/index/vector_index/IndexIVF.cpp b/core/src/index/knowhere/knowhere/index/vector_index/IndexIVF.cpp index a6a11eb423..523cdbaaf3 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/IndexIVF.cpp +++ b/core/src/index/knowhere/knowhere/index/vector_index/IndexIVF.cpp @@ -26,6 +26,7 @@ #include #include #include +#include #include #include @@ -43,16 +44,11 @@ using stdclock = std::chrono::high_resolution_clock; IndexModelPtr IVF::Train(const DatasetPtr& dataset, const Config& config) { - auto build_cfg = std::dynamic_pointer_cast(config); - if (build_cfg != nullptr) { - build_cfg->CheckValid(); // throw exception - } - GETTENSOR(dataset) faiss::Index* coarse_quantizer = new faiss::IndexFlatL2(dim); - auto index = std::make_shared(coarse_quantizer, dim, build_cfg->nlist, - GetMetricType(build_cfg->metric_type)); + auto index = std::make_shared(coarse_quantizer, dim, config[IndexParams::nlist].get(), + GetMetricType(config[Metric::TYPE].get())); index->train(rows, (float*)p_data); // TODO(linxj): override here. train return model or not. @@ -106,24 +102,19 @@ IVF::Search(const DatasetPtr& dataset, const Config& config) { KNOWHERE_THROW_MSG("index not initialize or trained"); } - auto search_cfg = std::dynamic_pointer_cast(config); - if (search_cfg == nullptr) { - KNOWHERE_THROW_MSG("not support this kind of config"); - } - GETTENSOR(dataset) try { fiu_do_on("IVF.Search.throw_std_exception", throw std::exception()); fiu_do_on("IVF.Search.throw_faiss_exception", throw faiss::FaissException("")); - auto elems = rows * search_cfg->k; + auto elems = rows * config[meta::TOPK].get(); size_t p_id_size = sizeof(int64_t) * elems; size_t p_dist_size = sizeof(float) * elems; auto p_id = (int64_t*)malloc(p_id_size); auto p_dist = (float*)malloc(p_dist_size); - search_impl(rows, (float*)p_data, search_cfg->k, p_dist, p_id, config); + search_impl(rows, (float*)p_data, config[meta::TOPK].get(), p_dist, p_id, config); // std::stringstream ss_res_id, ss_res_dist; // for (int i = 0; i < 10; ++i) { @@ -163,9 +154,8 @@ std::shared_ptr IVF::GenParams(const Config& config) { auto params = std::make_shared(); - auto search_cfg = std::dynamic_pointer_cast(config); - params->nprobe = search_cfg->nprobe; - // params->max_codes = config.get_with_default("max_codes", size_t(0)); + params->nprobe = config[IndexParams::nprobe]; + // params->max_codes = config["max_codes"]; return params; } @@ -185,7 +175,7 @@ IVF::GenGraph(const float* data, const int64_t& k, Graph& graph, const Config& c int64_t K = k + 1; auto ntotal = Count(); - size_t dim = config->d; + size_t dim = config[meta::DIM]; auto batch_size = 1000; auto tail_batch_size = ntotal % batch_size; auto batch_search_count = ntotal / batch_size; @@ -279,12 +269,6 @@ IVF::GetVectorById(const DatasetPtr& dataset, const Config& config) { KNOWHERE_THROW_MSG("index not initialize or trained"); } - auto search_cfg = std::dynamic_pointer_cast(config); - if (search_cfg == nullptr) { - KNOWHERE_THROW_MSG("not support this kind of config"); - } - - // auto rows = dataset->Get(meta::ROWS); auto p_data = dataset->Get(meta::IDS); auto elems = dataset->Get(meta::DIM); @@ -311,16 +295,11 @@ IVF::SearchById(const DatasetPtr& dataset, const Config& config) { KNOWHERE_THROW_MSG("index not initialize or trained"); } - auto search_cfg = std::dynamic_pointer_cast(config); - if (search_cfg == nullptr) { - KNOWHERE_THROW_MSG("not support this kind of config"); - } - auto rows = dataset->Get(meta::ROWS); auto p_data = dataset->Get(meta::IDS); try { - auto elems = rows * search_cfg->k; + auto elems = rows * config[meta::TOPK].get(); size_t p_id_size = sizeof(int64_t) * elems; size_t p_dist_size = sizeof(float) * elems; @@ -330,7 +309,7 @@ IVF::SearchById(const DatasetPtr& dataset, const Config& config) { // todo: enable search by id (zhiru) // auto blacklist = dataset->Get("bitset"); auto index_ivf = std::static_pointer_cast(index_); - index_ivf->search_by_id(rows, p_data, search_cfg->k, p_dist, p_id, bitset_); + index_ivf->search_by_id(rows, p_data, config[meta::TOPK].get(), p_dist, p_id, bitset_); // std::stringstream ss_res_id, ss_res_dist; // for (int i = 0; i < 10; ++i) { diff --git a/core/src/index/knowhere/knowhere/index/vector_index/IndexIVFPQ.cpp b/core/src/index/knowhere/knowhere/index/vector_index/IndexIVFPQ.cpp index a2b60f3796..b7278107b5 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/IndexIVFPQ.cpp +++ b/core/src/index/knowhere/knowhere/index/vector_index/IndexIVFPQ.cpp @@ -16,6 +16,7 @@ #endif #include +#include #include #include "knowhere/adapter/VectorAdapter.h" @@ -30,16 +31,12 @@ namespace knowhere { IndexModelPtr IVFPQ::Train(const DatasetPtr& dataset, const Config& config) { - auto build_cfg = std::dynamic_pointer_cast(config); - if (build_cfg != nullptr) { - build_cfg->CheckValid(); // throw exception - } - GETTENSOR(dataset) - faiss::Index* coarse_quantizer = new faiss::IndexFlat(dim, GetMetricType(build_cfg->metric_type)); - auto index = - std::make_shared(coarse_quantizer, dim, build_cfg->nlist, build_cfg->m, build_cfg->nbits); + faiss::Index* coarse_quantizer = new faiss::IndexFlat(dim, GetMetricType(config[Metric::TYPE].get())); + auto index = std::make_shared(coarse_quantizer, dim, config[IndexParams::nlist].get(), + config[IndexParams::m].get(), + config[IndexParams::nbits].get()); index->train(rows, (float*)p_data); return std::make_shared(index); @@ -48,11 +45,10 @@ IVFPQ::Train(const DatasetPtr& dataset, const Config& config) { std::shared_ptr IVFPQ::GenParams(const Config& config) { auto params = std::make_shared(); - auto search_cfg = std::dynamic_pointer_cast(config); - params->nprobe = search_cfg->nprobe; - // params->scan_table_threshold = conf->scan_table_threhold; - // params->polysemous_ht = conf->polysemous_ht; - // params->max_codes = conf->max_codes; + params->nprobe = config[IndexParams::nprobe]; + // params->scan_table_threshold = config["scan_table_threhold"] + // params->polysemous_ht = config["polysemous_ht"] + // params->max_codes = config["max_codes"] return params; } diff --git a/core/src/index/knowhere/knowhere/index/vector_index/IndexIVFSQ.cpp b/core/src/index/knowhere/knowhere/index/vector_index/IndexIVFSQ.cpp index 27a343daaf..e37abaf321 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/IndexIVFSQ.cpp +++ b/core/src/index/knowhere/knowhere/index/vector_index/IndexIVFSQ.cpp @@ -16,6 +16,7 @@ #include #include +#include #include "knowhere/adapter/VectorAdapter.h" #include "knowhere/common/Exception.h" @@ -30,17 +31,13 @@ namespace knowhere { IndexModelPtr IVFSQ::Train(const DatasetPtr& dataset, const Config& config) { - auto build_cfg = std::dynamic_pointer_cast(config); - if (build_cfg != nullptr) { - build_cfg->CheckValid(); // throw exception - } - GETTENSOR(dataset) std::stringstream index_type; - index_type << "IVF" << build_cfg->nlist << "," - << "SQ" << build_cfg->nbits; - auto build_index = faiss::index_factory(dim, index_type.str().c_str(), GetMetricType(build_cfg->metric_type)); + index_type << "IVF" << config[IndexParams::nlist] << "," + << "SQ" << config[IndexParams::nbits]; + auto build_index = + faiss::index_factory(dim, index_type.str().c_str(), GetMetricType(config[Metric::TYPE].get())); build_index->train(rows, (float*)p_data); std::shared_ptr ret_index; diff --git a/core/src/index/knowhere/knowhere/index/vector_index/IndexIVFSQHybrid.cpp b/core/src/index/knowhere/knowhere/index/vector_index/IndexIVFSQHybrid.cpp index a7094b2d24..1b8002bc26 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/IndexIVFSQHybrid.cpp +++ b/core/src/index/knowhere/knowhere/index/vector_index/IndexIVFSQHybrid.cpp @@ -19,6 +19,7 @@ #include #include #include +#include #include namespace knowhere { @@ -30,19 +31,14 @@ namespace knowhere { IndexModelPtr IVFSQHybrid::Train(const DatasetPtr& dataset, const Config& config) { // std::lock_guard lk(g_mutex); - - auto build_cfg = std::dynamic_pointer_cast(config); - if (build_cfg != nullptr) { - build_cfg->CheckValid(); // throw exception - } - gpu_id_ = build_cfg->gpu_id; - GETTENSOR(dataset) + gpu_id_ = config[knowhere::meta::DEVICEID]; std::stringstream index_type; - index_type << "IVF" << build_cfg->nlist << "," + index_type << "IVF" << config[IndexParams::nlist] << "," << "SQ8Hybrid"; - auto build_index = faiss::index_factory(dim, index_type.str().c_str(), GetMetricType(build_cfg->metric_type)); + auto build_index = + faiss::index_factory(dim, index_type.str().c_str(), GetMetricType(config[Metric::TYPE].get())); auto temp_resource = FaissGpuResourceMgr::GetInstance().GetRes(gpu_id_); if (temp_resource != nullptr) { @@ -133,17 +129,10 @@ IVFSQHybrid::search_impl(int64_t n, const float* data, int64_t k, float* distanc } QuantizerPtr -IVFSQHybrid::LoadQuantizer(const Config& conf) { +IVFSQHybrid::LoadQuantizer(const Config& config) { // std::lock_guard lk(g_mutex); - auto quantizer_conf = std::dynamic_pointer_cast(conf); - if (quantizer_conf != nullptr) { - if (quantizer_conf->mode != 1) { - KNOWHERE_THROW_MSG("mode only support 1 in this func"); - } - } - auto gpu_id = quantizer_conf->gpu_id; - + auto gpu_id = config[knowhere::meta::DEVICEID].get(); if (auto res = FaissGpuResourceMgr::GetInstance().GetRes(gpu_id)) { ResScope rs(res, gpu_id, false); faiss::gpu::GpuClonerOptions option; @@ -152,7 +141,7 @@ IVFSQHybrid::LoadQuantizer(const Config& conf) { auto index_composition = new faiss::IndexComposition; index_composition->index = index_.get(); index_composition->quantizer = nullptr; - index_composition->mode = quantizer_conf->mode; // only 1 + index_composition->mode = 1; // only 1 auto gpu_index = faiss::gpu::index_cpu_to_gpu(res->faiss_res.get(), gpu_id, index_composition, &option); delete gpu_index; @@ -205,19 +194,10 @@ IVFSQHybrid::UnsetQuantizer() { } VectorIndexPtr -IVFSQHybrid::LoadData(const knowhere::QuantizerPtr& q, const Config& conf) { +IVFSQHybrid::LoadData(const knowhere::QuantizerPtr& q, const Config& config) { // std::lock_guard lk(g_mutex); - auto quantizer_conf = std::dynamic_pointer_cast(conf); - if (quantizer_conf != nullptr) { - if (quantizer_conf->mode != 2) { - KNOWHERE_THROW_MSG("mode only support 2 in this func"); - } - } else { - KNOWHERE_THROW_MSG("conf error"); - } - - auto gpu_id = quantizer_conf->gpu_id; + int64_t gpu_id = config[knowhere::meta::DEVICEID]; if (auto res = FaissGpuResourceMgr::GetInstance().GetRes(gpu_id)) { ResScope rs(res, gpu_id, false); @@ -231,7 +211,7 @@ IVFSQHybrid::LoadData(const knowhere::QuantizerPtr& q, const Config& conf) { auto index_composition = new faiss::IndexComposition; index_composition->index = index_.get(); index_composition->quantizer = ivf_quantizer->quantizer; - index_composition->mode = quantizer_conf->mode; // only 2 + index_composition->mode = 2; // only 2 auto gpu_index = faiss::gpu::index_cpu_to_gpu(res->faiss_res.get(), gpu_id, index_composition, &option); std::shared_ptr new_idx; diff --git a/core/src/index/knowhere/knowhere/index/vector_index/IndexNSG.cpp b/core/src/index/knowhere/knowhere/index/vector_index/IndexNSG.cpp index 20caf476aa..3f3d725bdd 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/IndexNSG.cpp +++ b/core/src/index/knowhere/knowhere/index/vector_index/IndexNSG.cpp @@ -10,6 +10,7 @@ // or implied. See the License for the specific language governing permissions and limitations under the License. #include "knowhere/index/vector_index/IndexNSG.h" + #include "knowhere/adapter/VectorAdapter.h" #include "knowhere/common/Exception.h" #include "knowhere/common/Timer.h" @@ -23,6 +24,7 @@ #endif #include + #include "knowhere/index/vector_index/IndexIDMAP.h" #include "knowhere/index/vector_index/IndexIVF.h" #include "knowhere/index/vector_index/nsg/NSG.h" @@ -72,23 +74,21 @@ NSG::Load(const BinarySet& index_binary) { DatasetPtr NSG::Search(const DatasetPtr& dataset, const Config& config) { - auto build_cfg = std::dynamic_pointer_cast(config); - if (!index_ || !index_->is_trained) { KNOWHERE_THROW_MSG("index not initialize or trained"); } GETTENSOR(dataset) - auto elems = rows * build_cfg->k; + auto elems = rows * config[meta::TOPK].get(); size_t p_id_size = sizeof(int64_t) * elems; size_t p_dist_size = sizeof(float) * elems; auto p_id = (int64_t*)malloc(p_id_size); auto p_dist = (float*)malloc(p_dist_size); algo::SearchParams s_params; - s_params.search_length = build_cfg->search_length; - index_->Search((float*)p_data, rows, dim, build_cfg->k, p_dist, p_id, s_params); + s_params.search_length = config[IndexParams::search_length]; + index_->Search((float*)p_data, rows, dim, config[meta::TOPK].get(), p_dist, p_id, s_params); auto ret_ds = std::make_shared(); ret_ds->Set(meta::IDS, p_id); @@ -98,41 +98,35 @@ NSG::Search(const DatasetPtr& dataset, const Config& config) { IndexModelPtr NSG::Train(const DatasetPtr& dataset, const Config& config) { - config->Dump(); - auto build_cfg = std::dynamic_pointer_cast(config); - if (build_cfg != nullptr) { - build_cfg->CheckValid(); // throw exception - } - auto idmap = std::make_shared(); idmap->Train(config); idmap->AddWithoutId(dataset, config); Graph knng; const float* raw_data = idmap->GetRawVectors(); #ifdef MILVUS_GPU_VERSION - if (build_cfg->gpu_id == knowhere::INVALID_VALUE) { + if (config[knowhere::meta::DEVICEID].get() == -1) { auto preprocess_index = std::make_shared(); auto model = preprocess_index->Train(dataset, config); preprocess_index->set_index_model(model); - preprocess_index->Add(dataset, config); - preprocess_index->GenGraph(raw_data, build_cfg->knng, knng, config); + preprocess_index->AddWithoutIds(dataset, config); + preprocess_index->GenGraph(raw_data, config[IndexParams::knng].get(), knng, config); } else { - auto gpu_idx = cloner::CopyCpuToGpu(idmap, build_cfg->gpu_id, config); + auto gpu_idx = cloner::CopyCpuToGpu(idmap, config[knowhere::meta::DEVICEID].get(), config); auto gpu_idmap = std::dynamic_pointer_cast(gpu_idx); - gpu_idmap->GenGraph(raw_data, build_cfg->knng, knng, config); + gpu_idmap->GenGraph(raw_data, config[IndexParams::knng].get(), knng, config); } #else auto preprocess_index = std::make_shared(); auto model = preprocess_index->Train(dataset, config); preprocess_index->set_index_model(model); preprocess_index->AddWithoutIds(dataset, config); - preprocess_index->GenGraph(raw_data, build_cfg->knng, knng, config); + preprocess_index->GenGraph(raw_data, config[IndexParams::knng].get(), knng, config); #endif algo::BuildParams b_params; - b_params.candidate_pool_size = build_cfg->candidate_pool_size; - b_params.out_degree = build_cfg->out_degree; - b_params.search_length = build_cfg->search_length; + b_params.candidate_pool_size = config[IndexParams::candidate]; + b_params.out_degree = config[IndexParams::out_degree]; + b_params.search_length = config[IndexParams::search_length]; auto p_ids = dataset->Get(meta::IDS); diff --git a/core/src/index/knowhere/knowhere/index/vector_index/IndexSPTAG.cpp b/core/src/index/knowhere/knowhere/index/vector_index/IndexSPTAG.cpp index fdea05bf4d..2b9f7a79c2 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/IndexSPTAG.cpp +++ b/core/src/index/knowhere/knowhere/index/vector_index/IndexSPTAG.cpp @@ -123,9 +123,6 @@ CPUSPTAGRNG::Load(const BinarySet& binary_set) { IndexModelPtr CPUSPTAGRNG::Train(const DatasetPtr& origin, const Config& train_config) { SetParameters(train_config); - if (train_config != nullptr) { - train_config->CheckValid(); // throw exception - } DatasetPtr dataset = origin; // TODO(linxj): copy or reference? @@ -159,62 +156,56 @@ CPUSPTAGRNG::Add(const DatasetPtr& origin, const Config& add_config) { void CPUSPTAGRNG::SetParameters(const Config& config) { -#define Assign(param_name, str_name) \ - conf->param_name == INVALID_VALUE ? index_ptr_->SetParameter(str_name, std::to_string(build_cfg->param_name)) \ - : index_ptr_->SetParameter(str_name, std::to_string(conf->param_name)) +#define Assign(param_name, str_name) \ + index_ptr_->SetParameter(str_name, std::to_string(build_cfg[param_name].get())) if (index_type_ == SPTAG::IndexAlgoType::KDT) { - auto conf = std::dynamic_pointer_cast(config); auto build_cfg = SPTAGParameterMgr::GetInstance().GetKDTParameters(); - Assign(kdtnumber, "KDTNumber"); - Assign(numtopdimensionkdtsplit, "NumTopDimensionKDTSplit"); - Assign(samples, "Samples"); - Assign(tptnumber, "TPTNumber"); - Assign(tptleafsize, "TPTLeafSize"); - Assign(numtopdimensiontptsplit, "NumTopDimensionTPTSplit"); - Assign(neighborhoodsize, "NeighborhoodSize"); - Assign(graphneighborhoodscale, "GraphNeighborhoodScale"); - Assign(graphcefscale, "GraphCEFScale"); - Assign(refineiterations, "RefineIterations"); - Assign(cef, "CEF"); - Assign(maxcheckforrefinegraph, "MaxCheckForRefineGraph"); - Assign(numofthreads, "NumberOfThreads"); - Assign(maxcheck, "MaxCheck"); - Assign(thresholdofnumberofcontinuousnobetterpropagation, "ThresholdOfNumberOfContinuousNoBetterPropagation"); - Assign(numberofinitialdynamicpivots, "NumberOfInitialDynamicPivots"); - Assign(numberofotherdynamicpivots, "NumberOfOtherDynamicPivots"); + Assign("kdtnumber", "KDTNumber"); + Assign("numtopdimensionkdtsplit", "NumTopDimensionKDTSplit"); + Assign("samples", "Samples"); + Assign("tptnumber", "TPTNumber"); + Assign("tptleafsize", "TPTLeafSize"); + Assign("numtopdimensiontptsplit", "NumTopDimensionTPTSplit"); + Assign("neighborhoodsize", "NeighborhoodSize"); + Assign("graphneighborhoodscale", "GraphNeighborhoodScale"); + Assign("graphcefscale", "GraphCEFScale"); + Assign("refineiterations", "RefineIterations"); + Assign("cef", "CEF"); + Assign("maxcheckforrefinegraph", "MaxCheckForRefineGraph"); + Assign("numofthreads", "NumberOfThreads"); + Assign("maxcheck", "MaxCheck"); + Assign("thresholdofnumberofcontinuousnobetterpropagation", "ThresholdOfNumberOfContinuousNoBetterPropagation"); + Assign("numberofinitialdynamicpivots", "NumberOfInitialDynamicPivots"); + Assign("numberofotherdynamicpivots", "NumberOfOtherDynamicPivots"); } else { - auto conf = std::dynamic_pointer_cast(config); auto build_cfg = SPTAGParameterMgr::GetInstance().GetBKTParameters(); - Assign(bktnumber, "BKTNumber"); - Assign(bktkmeansk, "BKTKMeansK"); - Assign(bktleafsize, "BKTLeafSize"); - Assign(samples, "Samples"); - Assign(tptnumber, "TPTNumber"); - Assign(tptleafsize, "TPTLeafSize"); - Assign(numtopdimensiontptsplit, "NumTopDimensionTPTSplit"); - Assign(neighborhoodsize, "NeighborhoodSize"); - Assign(graphneighborhoodscale, "GraphNeighborhoodScale"); - Assign(graphcefscale, "GraphCEFScale"); - Assign(refineiterations, "RefineIterations"); - Assign(cef, "CEF"); - Assign(maxcheckforrefinegraph, "MaxCheckForRefineGraph"); - Assign(numofthreads, "NumberOfThreads"); - Assign(maxcheck, "MaxCheck"); - Assign(thresholdofnumberofcontinuousnobetterpropagation, "ThresholdOfNumberOfContinuousNoBetterPropagation"); - Assign(numberofinitialdynamicpivots, "NumberOfInitialDynamicPivots"); - Assign(numberofotherdynamicpivots, "NumberOfOtherDynamicPivots"); + Assign("bktnumber", "BKTNumber"); + Assign("bktkmeansk", "BKTKMeansK"); + Assign("bktleafsize", "BKTLeafSize"); + Assign("samples", "Samples"); + Assign("tptnumber", "TPTNumber"); + Assign("tptleafsize", "TPTLeafSize"); + Assign("numtopdimensiontptsplit", "NumTopDimensionTPTSplit"); + Assign("neighborhoodsize", "NeighborhoodSize"); + Assign("graphneighborhoodscale", "GraphNeighborhoodScale"); + Assign("graphcefscale", "GraphCEFScale"); + Assign("refineiterations", "RefineIterations"); + Assign("cef", "CEF"); + Assign("maxcheckforrefinegraph", "MaxCheckForRefineGraph"); + Assign("numofthreads", "NumberOfThreads"); + Assign("maxcheck", "MaxCheck"); + Assign("thresholdofnumberofcontinuousnobetterpropagation", "ThresholdOfNumberOfContinuousNoBetterPropagation"); + Assign("numberofinitialdynamicpivots", "NumberOfInitialDynamicPivots"); + Assign("numberofotherdynamicpivots", "NumberOfOtherDynamicPivots"); } } DatasetPtr CPUSPTAGRNG::Search(const DatasetPtr& dataset, const Config& config) { SetParameters(config); - // if (config != nullptr) { - // config->CheckValid(); // throw exception - // } auto p_data = dataset->Get(meta::TENSOR); for (auto i = 0; i < 10; ++i) { diff --git a/core/src/index/knowhere/knowhere/index/vector_index/Quantizer.h b/core/src/index/knowhere/knowhere/index/vector_index/Quantizer.h index efb6259a67..fd5752a1f3 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/Quantizer.h +++ b/core/src/index/knowhere/knowhere/index/vector_index/Quantizer.h @@ -23,9 +23,9 @@ struct Quantizer { }; using QuantizerPtr = std::shared_ptr; -struct QuantizerCfg : Cfg { - int64_t mode = -1; // 0: all data, 1: copy quantizer, 2: copy data -}; -using QuantizerConfig = std::shared_ptr; +// struct QuantizerCfg : Cfg { +// int64_t mode = -1; // 0: all data, 1: copy quantizer, 2: copy data +// }; +// using QuantizerConfig = std::shared_ptr; } // namespace knowhere diff --git a/core/src/index/knowhere/knowhere/index/vector_index/helpers/Cloner.cpp b/core/src/index/knowhere/knowhere/index/vector_index/helpers/Cloner.cpp index 80409ede09..330ee79978 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/helpers/Cloner.cpp +++ b/core/src/index/knowhere/knowhere/index/vector_index/helpers/Cloner.cpp @@ -24,7 +24,10 @@ namespace cloner { VectorIndexPtr CopyGpuToCpu(const VectorIndexPtr& index, const Config& config) { if (auto device_index = std::dynamic_pointer_cast(index)) { - return device_index->CopyGpuToCpu(config); + VectorIndexPtr result = device_index->CopyGpuToCpu(config); + auto uids = index->GetUids(); + result->SetUids(uids); + return result; } else { KNOWHERE_THROW_MSG("index type is not gpuindex"); } diff --git a/core/src/index/knowhere/knowhere/index/vector_index/helpers/IndexParameter.cpp b/core/src/index/knowhere/knowhere/index/vector_index/helpers/IndexParameter.cpp index 3414ff7a5d..0912d78b65 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/helpers/IndexParameter.cpp +++ b/core/src/index/knowhere/knowhere/index/vector_index/helpers/IndexParameter.cpp @@ -17,47 +17,23 @@ namespace knowhere { faiss::MetricType -GetMetricType(METRICTYPE& type) { - if (type == METRICTYPE::L2) { +GetMetricType(const std::string& type) { + if (type == Metric::L2) { return faiss::METRIC_L2; } - if (type == METRICTYPE::IP) { + if (type == Metric::IP) { return faiss::METRIC_INNER_PRODUCT; } - // binary only - if (type == METRICTYPE::JACCARD) { + if (type == Metric::JACCARD) { return faiss::METRIC_Jaccard; } - if (type == METRICTYPE::TANIMOTO) { + if (type == Metric::TANIMOTO) { return faiss::METRIC_Tanimoto; } - if (type == METRICTYPE::HAMMING) { + if (type == Metric::HAMMING) { return faiss::METRIC_Hamming; } - KNOWHERE_THROW_MSG("Metric type is invalid"); } -std::stringstream -IVFCfg::DumpImpl() { - auto ss = Cfg::DumpImpl(); - ss << ", nlist: " << nlist << ", nprobe: " << nprobe; - return ss; -} - -std::stringstream -IVFSQCfg::DumpImpl() { - auto ss = IVFCfg::DumpImpl(); - ss << ", nbits: " << nbits; - return ss; -} - -std::stringstream -NSGCfg::DumpImpl() { - auto ss = IVFCfg::DumpImpl(); - ss << ", knng: " << knng << ", search_length: " << search_length << ", out_degree: " << out_degree - << ", candidate: " << candidate_pool_size; - return ss; -} - } // namespace knowhere diff --git a/core/src/index/knowhere/knowhere/index/vector_index/helpers/IndexParameter.h b/core/src/index/knowhere/knowhere/index/vector_index/helpers/IndexParameter.h index cd8a685767..f67e01d2a7 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/helpers/IndexParameter.h +++ b/core/src/index/knowhere/knowhere/index/vector_index/helpers/IndexParameter.h @@ -12,240 +12,49 @@ #pragma once #include -#include - -#include "knowhere/common/Config.h" +#include namespace knowhere { +namespace meta { +constexpr const char* DIM = "dim"; +constexpr const char* TENSOR = "tensor"; +constexpr const char* ROWS = "rows"; +constexpr const char* IDS = "ids"; +constexpr const char* DISTANCE = "distance"; +constexpr const char* TOPK = "k"; +constexpr const char* DEVICEID = "gpu_id"; +}; // namespace meta + +namespace IndexParams { +// IVF Params +constexpr const char* nprobe = "nprobe"; +constexpr const char* nlist = "nlist"; +constexpr const char* m = "m"; // PQ +constexpr const char* nbits = "nbits"; // PQ/SQ + +// NSG Params +constexpr const char* knng = "knng"; +constexpr const char* search_length = "search_length"; +constexpr const char* out_degree = "out_degree"; +constexpr const char* candidate = "candidate_pool_size"; + +// HNSW Params +constexpr const char* efConstruction = "efConstruction"; +constexpr const char* M = "M"; +constexpr const char* ef = "ef"; +} // namespace IndexParams + +namespace Metric { +constexpr const char* TYPE = "metric_type"; +constexpr const char* IP = "IP"; +constexpr const char* L2 = "L2"; +constexpr const char* HAMMING = "HAMMING"; +constexpr const char* JACCARD = "JACCARD"; +constexpr const char* TANIMOTO = "TANIMOTO"; +} // namespace Metric + extern faiss::MetricType -GetMetricType(METRICTYPE& type); - -// IVF Config -constexpr int64_t DEFAULT_NLIST = INVALID_VALUE; -constexpr int64_t DEFAULT_NPROBE = INVALID_VALUE; -constexpr int64_t DEFAULT_NSUBVECTORS = INVALID_VALUE; -constexpr int64_t DEFAULT_NBITS = INVALID_VALUE; -constexpr int64_t DEFAULT_SCAN_TABLE_THREHOLD = INVALID_VALUE; -constexpr int64_t DEFAULT_POLYSEMOUS_HT = INVALID_VALUE; -constexpr int64_t DEFAULT_MAX_CODES = INVALID_VALUE; - -// NSG Config -constexpr int64_t DEFAULT_SEARCH_LENGTH = INVALID_VALUE; -constexpr int64_t DEFAULT_OUT_DEGREE = INVALID_VALUE; -constexpr int64_t DEFAULT_CANDIDATE_SISE = INVALID_VALUE; -constexpr int64_t DEFAULT_NNG_K = INVALID_VALUE; - -// SPTAG Config -constexpr int64_t DEFAULT_SAMPLES = INVALID_VALUE; -constexpr int64_t DEFAULT_TPTNUMBER = INVALID_VALUE; -constexpr int64_t DEFAULT_TPTLEAFSIZE = INVALID_VALUE; -constexpr int64_t DEFAULT_NUMTOPDIMENSIONTPTSPLIT = INVALID_VALUE; -constexpr int64_t DEFAULT_NEIGHBORHOODSIZE = INVALID_VALUE; -constexpr int64_t DEFAULT_GRAPHNEIGHBORHOODSCALE = INVALID_VALUE; -constexpr int64_t DEFAULT_GRAPHCEFSCALE = INVALID_VALUE; -constexpr int64_t DEFAULT_REFINEITERATIONS = INVALID_VALUE; -constexpr int64_t DEFAULT_CEF = INVALID_VALUE; -constexpr int64_t DEFAULT_MAXCHECKFORREFINEGRAPH = INVALID_VALUE; -constexpr int64_t DEFAULT_NUMOFTHREADS = INVALID_VALUE; -constexpr int64_t DEFAULT_MAXCHECK = INVALID_VALUE; -constexpr int64_t DEFAULT_THRESHOLDOFNUMBEROFCONTINUOUSNOBETTERPROPAGATION = INVALID_VALUE; -constexpr int64_t DEFAULT_NUMBEROFINITIALDYNAMICPIVOTS = INVALID_VALUE; -constexpr int64_t DEFAULT_NUMBEROFOTHERDYNAMICPIVOTS = INVALID_VALUE; - -// KDT Config -constexpr int64_t DEFAULT_KDTNUMBER = INVALID_VALUE; -constexpr int64_t DEFAULT_NUMTOPDIMENSIONKDTSPLIT = INVALID_VALUE; - -// BKT Config -constexpr int64_t DEFAULT_BKTNUMBER = INVALID_VALUE; -constexpr int64_t DEFAULT_BKTKMEANSK = INVALID_VALUE; -constexpr int64_t DEFAULT_BKTLEAFSIZE = INVALID_VALUE; - -// HNSW Config -constexpr int64_t DEFAULT_M = INVALID_VALUE; -constexpr int64_t DEFAULT_EF = INVALID_VALUE; - -struct IVFCfg : public Cfg { - int64_t nlist = DEFAULT_NLIST; - int64_t nprobe = DEFAULT_NPROBE; - - IVFCfg(const int64_t& dim, const int64_t& k, const int64_t& gpu_id, const int64_t& nlist, const int64_t& nprobe, - METRICTYPE type) - : Cfg(dim, k, gpu_id, type), nlist(nlist), nprobe(nprobe) { - } - - IVFCfg() = default; - - std::stringstream - DumpImpl() override; - - // bool - // CheckValid() override { - // return true; - // }; -}; -using IVFConfig = std::shared_ptr; - -struct IVFBinCfg : public IVFCfg { - bool - CheckValid() override { - if (metric_type == METRICTYPE::HAMMING || metric_type == METRICTYPE::TANIMOTO || - metric_type == METRICTYPE::JACCARD) { - return true; - } - std::stringstream ss; - ss << "MetricType: " << int(metric_type) << " not support!"; - KNOWHERE_THROW_MSG(ss.str()); - return false; - } -}; - -struct IVFSQCfg : public IVFCfg { - // TODO(linxj): cpu only support SQ4 SQ6 SQ8 SQ16, gpu only support SQ4, SQ8, SQ16 - int64_t nbits = DEFAULT_NBITS; - - IVFSQCfg(const int64_t& dim, const int64_t& k, const int64_t& gpu_id, const int64_t& nlist, const int64_t& nprobe, - const int64_t& nbits, METRICTYPE type) - : IVFCfg(dim, k, gpu_id, nlist, nprobe, type), nbits(nbits) { - } - - std::stringstream - DumpImpl() override; - - IVFSQCfg() = default; - - // bool - // CheckValid() override { - // return true; - // }; -}; -using IVFSQConfig = std::shared_ptr; - -struct IVFPQCfg : public IVFCfg { - int64_t m = DEFAULT_NSUBVECTORS; // number of subquantizers(subvector) - int64_t nbits = DEFAULT_NBITS; // number of bit per subvector index - - // TODO(linxj): not use yet - int64_t scan_table_threhold = DEFAULT_SCAN_TABLE_THREHOLD; - int64_t polysemous_ht = DEFAULT_POLYSEMOUS_HT; - int64_t max_codes = DEFAULT_MAX_CODES; - - IVFPQCfg(const int64_t& dim, const int64_t& k, const int64_t& gpu_id, const int64_t& nlist, const int64_t& nprobe, - const int64_t& nbits, const int64_t& m, METRICTYPE type) - : IVFCfg(dim, k, gpu_id, nlist, nprobe, type), m(m), nbits(nbits) { - } - - IVFPQCfg() = default; - - // bool - // CheckValid() override { - // return true; - // }; -}; -using IVFPQConfig = std::shared_ptr; - -struct NSGCfg : public IVFCfg { - int64_t knng = DEFAULT_NNG_K; - int64_t search_length = DEFAULT_SEARCH_LENGTH; - int64_t out_degree = DEFAULT_OUT_DEGREE; - int64_t candidate_pool_size = DEFAULT_CANDIDATE_SISE; - - NSGCfg(const int64_t& dim, const int64_t& k, const int64_t& gpu_id, const int64_t& nlist, const int64_t& nprobe, - const int64_t& knng, const int64_t& search_length, const int64_t& out_degree, const int64_t& candidate_size, - METRICTYPE type) - : IVFCfg(dim, k, gpu_id, nlist, nprobe, type), - knng(knng), - search_length(search_length), - out_degree(out_degree), - candidate_pool_size(candidate_size) { - } - - NSGCfg() = default; - - std::stringstream - DumpImpl() override; - - // bool - // CheckValid() override { - // return true; - // }; -}; -using NSGConfig = std::shared_ptr; - -struct SPTAGCfg : public Cfg { - int64_t samples = DEFAULT_SAMPLES; - int64_t tptnumber = DEFAULT_TPTNUMBER; - int64_t tptleafsize = DEFAULT_TPTLEAFSIZE; - int64_t numtopdimensiontptsplit = DEFAULT_NUMTOPDIMENSIONTPTSPLIT; - int64_t neighborhoodsize = DEFAULT_NEIGHBORHOODSIZE; - int64_t graphneighborhoodscale = DEFAULT_GRAPHNEIGHBORHOODSCALE; - int64_t graphcefscale = DEFAULT_GRAPHCEFSCALE; - int64_t refineiterations = DEFAULT_REFINEITERATIONS; - int64_t cef = DEFAULT_CEF; - int64_t maxcheckforrefinegraph = DEFAULT_MAXCHECKFORREFINEGRAPH; - int64_t numofthreads = DEFAULT_NUMOFTHREADS; - int64_t maxcheck = DEFAULT_MAXCHECK; - int64_t thresholdofnumberofcontinuousnobetterpropagation = DEFAULT_THRESHOLDOFNUMBEROFCONTINUOUSNOBETTERPROPAGATION; - int64_t numberofinitialdynamicpivots = DEFAULT_NUMBEROFINITIALDYNAMICPIVOTS; - int64_t numberofotherdynamicpivots = DEFAULT_NUMBEROFOTHERDYNAMICPIVOTS; - - SPTAGCfg() = default; - - // bool - // CheckValid() override { - // return true; - // }; -}; -using SPTAGConfig = std::shared_ptr; - -struct KDTCfg : public SPTAGCfg { - int64_t kdtnumber = DEFAULT_KDTNUMBER; - int64_t numtopdimensionkdtsplit = DEFAULT_NUMTOPDIMENSIONKDTSPLIT; - - KDTCfg() = default; - - // bool - // CheckValid() override { - // return true; - // }; -}; -using KDTConfig = std::shared_ptr; - -struct BKTCfg : public SPTAGCfg { - int64_t bktnumber = DEFAULT_BKTNUMBER; - int64_t bktkmeansk = DEFAULT_BKTKMEANSK; - int64_t bktleafsize = DEFAULT_BKTLEAFSIZE; - - BKTCfg() = default; - - // bool - // CheckValid() override { - // return true; - // }; -}; -using BKTConfig = std::shared_ptr; - -struct BinIDMAPCfg : public Cfg { - bool - CheckValid() override { - if (metric_type == METRICTYPE::HAMMING || metric_type == METRICTYPE::TANIMOTO || - metric_type == METRICTYPE::JACCARD) { - return true; - } - std::stringstream ss; - ss << "MetricType: " << int(metric_type) << " not support!"; - KNOWHERE_THROW_MSG(ss.str()); - return false; - } -}; - -struct HNSWCfg : public Cfg { - int64_t M = DEFAULT_M; - int64_t ef = DEFAULT_EF; - - HNSWCfg() = default; -}; -using HNSWConfig = std::shared_ptr; +GetMetricType(const std::string& type); } // namespace knowhere diff --git a/core/src/index/knowhere/knowhere/index/vector_index/helpers/SPTAGParameterMgr.cpp b/core/src/index/knowhere/knowhere/index/vector_index/helpers/SPTAGParameterMgr.cpp index 36907fbf80..b32fab0f8b 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/helpers/SPTAGParameterMgr.cpp +++ b/core/src/index/knowhere/knowhere/index/vector_index/helpers/SPTAGParameterMgr.cpp @@ -15,55 +15,53 @@ namespace knowhere { -const KDTConfig& +const Config& SPTAGParameterMgr::GetKDTParameters() { return kdt_config_; } -const BKTConfig& +const Config& SPTAGParameterMgr::GetBKTParameters() { return bkt_config_; } SPTAGParameterMgr::SPTAGParameterMgr() { - kdt_config_ = std::make_shared(); - kdt_config_->kdtnumber = 1; - kdt_config_->numtopdimensionkdtsplit = 5; - kdt_config_->samples = 100; - kdt_config_->tptnumber = 1; - kdt_config_->tptleafsize = 2000; - kdt_config_->numtopdimensiontptsplit = 5; - kdt_config_->neighborhoodsize = 32; - kdt_config_->graphneighborhoodscale = 2; - kdt_config_->graphcefscale = 2; - kdt_config_->refineiterations = 0; - kdt_config_->cef = 1000; - kdt_config_->maxcheckforrefinegraph = 10000; - kdt_config_->numofthreads = 1; - kdt_config_->maxcheck = 8192; - kdt_config_->thresholdofnumberofcontinuousnobetterpropagation = 3; - kdt_config_->numberofinitialdynamicpivots = 50; - kdt_config_->numberofotherdynamicpivots = 4; + kdt_config_["kdtnumber"] = 1; + kdt_config_["numtopdimensionkdtsplit"] = 5; + kdt_config_["samples"] = 100; + kdt_config_["tptnumber"] = 1; + kdt_config_["tptleafsize"] = 2000; + kdt_config_["numtopdimensiontptsplit"] = 5; + kdt_config_["neighborhoodsize"] = 32; + kdt_config_["graphneighborhoodscale"] = 2; + kdt_config_["graphcefscale"] = 2; + kdt_config_["refineiterations"] = 0; + kdt_config_["cef"] = 1000; + kdt_config_["maxcheckforrefinegraph"] = 10000; + kdt_config_["numofthreads"] = 1; + kdt_config_["maxcheck"] = 8192; + kdt_config_["thresholdofnumberofcontinuousnobetterpropagation"] = 3; + kdt_config_["numberofinitialdynamicpivots"] = 50; + kdt_config_["numberofotherdynamicpivots"] = 4; - bkt_config_ = std::make_shared(); - bkt_config_->bktnumber = 1; - bkt_config_->bktkmeansk = 32; - bkt_config_->bktleafsize = 8; - bkt_config_->samples = 100; - bkt_config_->tptnumber = 1; - bkt_config_->tptleafsize = 2000; - bkt_config_->numtopdimensiontptsplit = 5; - bkt_config_->neighborhoodsize = 32; - bkt_config_->graphneighborhoodscale = 2; - bkt_config_->graphcefscale = 2; - bkt_config_->refineiterations = 0; - bkt_config_->cef = 1000; - bkt_config_->maxcheckforrefinegraph = 10000; - bkt_config_->numofthreads = 1; - bkt_config_->maxcheck = 8192; - bkt_config_->thresholdofnumberofcontinuousnobetterpropagation = 3; - bkt_config_->numberofinitialdynamicpivots = 50; - bkt_config_->numberofotherdynamicpivots = 4; + bkt_config_["bktnumber"] = 1; + bkt_config_["bktkmeansk"] = 32; + bkt_config_["bktleafsize"] = 8; + bkt_config_["samples"] = 100; + bkt_config_["tptnumber"] = 1; + bkt_config_["tptleafsize"] = 2000; + bkt_config_["numtopdimensiontptsplit"] = 5; + bkt_config_["neighborhoodsize"] = 32; + bkt_config_["graphneighborhoodscale"] = 2; + bkt_config_["graphcefscale"] = 2; + bkt_config_["refineiterations"] = 0; + bkt_config_["cef"] = 1000; + bkt_config_["maxcheckforrefinegraph"] = 10000; + bkt_config_["numofthreads"] = 1; + bkt_config_["maxcheck"] = 8192; + bkt_config_["thresholdofnumberofcontinuousnobetterpropagation"] = 3; + bkt_config_["numberofinitialdynamicpivots"] = 50; + bkt_config_["numberofotherdynamicpivots"] = 4; } } // namespace knowhere diff --git a/core/src/index/knowhere/knowhere/index/vector_index/helpers/SPTAGParameterMgr.h b/core/src/index/knowhere/knowhere/index/vector_index/helpers/SPTAGParameterMgr.h index b3acaa1f02..a92dfca90a 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/helpers/SPTAGParameterMgr.h +++ b/core/src/index/knowhere/knowhere/index/vector_index/helpers/SPTAGParameterMgr.h @@ -18,18 +18,15 @@ #include #include "IndexParameter.h" +#include "knowhere/common/Config.h" namespace knowhere { - -using KDTConfig = std::shared_ptr; -using BKTConfig = std::shared_ptr; - class SPTAGParameterMgr { public: - const KDTConfig& + const Config& GetKDTParameters(); - const BKTConfig& + const Config& GetBKTParameters(); public: @@ -48,8 +45,8 @@ class SPTAGParameterMgr { SPTAGParameterMgr(); private: - KDTConfig kdt_config_; - BKTConfig bkt_config_; + Config kdt_config_; + Config bkt_config_; }; } // namespace knowhere diff --git a/core/src/index/knowhere/knowhere/index/vector_index/nsg/NSG.cpp b/core/src/index/knowhere/knowhere/index/vector_index/nsg/NSG.cpp index 5f395a7f67..16d20142b3 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/nsg/NSG.cpp +++ b/core/src/index/knowhere/knowhere/index/vector_index/nsg/NSG.cpp @@ -29,16 +29,9 @@ namespace algo { unsigned int seed = 100; -NsgIndex::NsgIndex(const size_t& dimension, const size_t& n, METRICTYPE metric) +NsgIndex::NsgIndex(const size_t& dimension, const size_t& n, const std::string& metric) : dimension(dimension), ntotal(n), metric_type(metric) { - switch (metric) { - case METRICTYPE::L2: - distance_ = new DistanceL2; - break; - case METRICTYPE::IP: - distance_ = new DistanceIP; - break; - } + distance_ = new DistanceL2; // hardcode here } NsgIndex::~NsgIndex() { diff --git a/core/src/index/knowhere/knowhere/index/vector_index/nsg/NSG.h b/core/src/index/knowhere/knowhere/index/vector_index/nsg/NSG.h index 52e38c606d..fba15220a9 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/nsg/NSG.h +++ b/core/src/index/knowhere/knowhere/index/vector_index/nsg/NSG.h @@ -13,6 +13,7 @@ #include #include +#include #include #include @@ -41,8 +42,8 @@ using Graph = std::vector>; class NsgIndex { public: size_t dimension; - size_t ntotal; // totabl nb of indexed vectors - METRICTYPE metric_type; // L2 | IP + size_t ntotal; // totabl nb of indexed vectors + std::string metric_type; // todo(linxj) IP Distance* distance_; float* ori_data_; @@ -62,7 +63,7 @@ class NsgIndex { size_t out_degree; public: - explicit NsgIndex(const size_t& dimension, const size_t& n, METRICTYPE metric = METRICTYPE::L2); + explicit NsgIndex(const size_t& dimension, const size_t& n, const std::string& metric = "L2"); NsgIndex() = default; diff --git a/core/src/index/thirdparty/faiss/impl/ScalarQuantizerOp.h b/core/src/index/thirdparty/faiss/impl/ScalarQuantizerOp.h index 2991512674..c272d29bc4 100644 --- a/core/src/index/thirdparty/faiss/impl/ScalarQuantizerOp.h +++ b/core/src/index/thirdparty/faiss/impl/ScalarQuantizerOp.h @@ -14,10 +14,6 @@ #include -#ifdef __SSE__ -#include -#endif - #include #include #include diff --git a/core/src/index/unittest/CMakeLists.txt b/core/src/index/unittest/CMakeLists.txt index 0c0225e606..b898f2d702 100644 --- a/core/src/index/unittest/CMakeLists.txt +++ b/core/src/index/unittest/CMakeLists.txt @@ -30,7 +30,6 @@ set(util_srcs ${MILVUS_THIRDPARTY_SRC}/easyloggingpp/easylogging++.cc ${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_index/helpers/FaissIO.cpp ${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_index/helpers/IndexParameter.cpp - ${INDEX_SOURCE_DIR}/knowhere/knowhere/adapter/VectorAdapter.cpp ${INDEX_SOURCE_DIR}/knowhere/knowhere/common/Exception.cpp ${INDEX_SOURCE_DIR}/knowhere/knowhere/common/Timer.cpp ${INDEX_SOURCE_DIR}/unittest/utils.cpp diff --git a/core/src/index/unittest/Helper.h b/core/src/index/unittest/Helper.h index f1ee38dc7b..c56e8e272e 100644 --- a/core/src/index/unittest/Helper.h +++ b/core/src/index/unittest/Helper.h @@ -72,35 +72,32 @@ class ParamGenerator { knowhere::Config Gen(const ParameterType& type) { if (type == ParameterType::ivf) { - auto tempconf = std::make_shared(); - tempconf->d = DIM; - tempconf->gpu_id = DEVICEID; - tempconf->nlist = 100; - tempconf->nprobe = 4; - tempconf->k = K; - tempconf->metric_type = knowhere::METRICTYPE::L2; - return tempconf; + return knowhere::Config{ + {knowhere::meta::DIM, DIM}, + {knowhere::meta::TOPK, K}, + {knowhere::IndexParams::nlist, 100}, + {knowhere::IndexParams::nprobe, 4}, + {knowhere::Metric::TYPE, knowhere::Metric::L2}, + {knowhere::meta::DEVICEID, DEVICEID}, + }; } else if (type == ParameterType::ivfpq) { - auto tempconf = std::make_shared(); - tempconf->d = DIM; - tempconf->gpu_id = DEVICEID; - tempconf->nlist = 100; - tempconf->nprobe = 4; - tempconf->k = K; - tempconf->m = 4; - tempconf->nbits = 8; - tempconf->metric_type = knowhere::METRICTYPE::L2; - return tempconf; + return knowhere::Config{ + {knowhere::meta::DIM, DIM}, + {knowhere::meta::TOPK, K}, + {knowhere::IndexParams::nlist, 100}, + {knowhere::IndexParams::nprobe, 4}, + {knowhere::IndexParams::m, 4}, + {knowhere::IndexParams::nbits, 8}, + {knowhere::Metric::TYPE, knowhere::Metric::L2}, + {knowhere::meta::DEVICEID, DEVICEID}, + }; } else if (type == ParameterType::ivfsq) { - auto tempconf = std::make_shared(); - tempconf->d = DIM; - tempconf->gpu_id = DEVICEID; - tempconf->nlist = 100; - tempconf->nprobe = 4; - tempconf->k = K; - tempconf->nbits = 8; - tempconf->metric_type = knowhere::METRICTYPE::L2; - return tempconf; + return knowhere::Config{ + {knowhere::meta::DIM, DIM}, {knowhere::meta::TOPK, K}, + {knowhere::IndexParams::nlist, 100}, {knowhere::IndexParams::nprobe, 4}, + {knowhere::IndexParams::nbits, 8}, {knowhere::Metric::TYPE, knowhere::Metric::L2}, + {knowhere::meta::DEVICEID, DEVICEID}, + }; } } }; diff --git a/core/src/index/unittest/test_binaryidmap.cpp b/core/src/index/unittest/test_binaryidmap.cpp index 610fbe3ce6..a2bb7a06c9 100644 --- a/core/src/index/unittest/test_binaryidmap.cpp +++ b/core/src/index/unittest/test_binaryidmap.cpp @@ -21,7 +21,7 @@ using ::testing::Combine; using ::testing::TestWithParam; using ::testing::Values; -class BinaryIDMAPTest : public BinaryDataGen, public TestWithParam { +class BinaryIDMAPTest : public BinaryDataGen, public TestWithParam { protected: void SetUp() override { @@ -37,17 +37,17 @@ class BinaryIDMAPTest : public BinaryDataGen, public TestWithParam(); - conf->d = dim; - conf->k = k; - conf->metric_type = MetricType; + std::string MetricType = GetParam(); + knowhere::Config conf{ + {knowhere::meta::DIM, dim}, + {knowhere::meta::TOPK, k}, + {knowhere::Metric::TYPE, MetricType}, + }; index_->Train(conf); index_->Add(base_dataset, conf); @@ -88,11 +88,12 @@ TEST_P(BinaryIDMAPTest, binaryidmap_serialize) { reader(ret, bin->size); }; - knowhere::METRICTYPE MetricType = GetParam(); - auto conf = std::make_shared(); - conf->d = dim; - conf->k = k; - conf->metric_type = MetricType; + std::string MetricType = GetParam(); + knowhere::Config conf{ + {knowhere::meta::DIM, dim}, + {knowhere::meta::TOPK, k}, + {knowhere::Metric::TYPE, MetricType}, + }; { // serialize index diff --git a/core/src/index/unittest/test_binaryivf.cpp b/core/src/index/unittest/test_binaryivf.cpp index 44a7f976fd..3934a6810c 100644 --- a/core/src/index/unittest/test_binaryivf.cpp +++ b/core/src/index/unittest/test_binaryivf.cpp @@ -27,25 +27,24 @@ using ::testing::Combine; using ::testing::TestWithParam; using ::testing::Values; -class BinaryIVFTest : public BinaryDataGen, public TestWithParam { +class BinaryIVFTest : public BinaryDataGen, public TestWithParam { protected: void SetUp() override { - knowhere::METRICTYPE MetricType = GetParam(); + std::string MetricType = GetParam(); Init_with_binary_default(); // nb = 1000000; // nq = 1000; // k = 1000; // Generate(DIM, NB, NQ); index_ = std::make_shared(); - auto x_conf = std::make_shared(); - x_conf->d = dim; - x_conf->k = k; - x_conf->metric_type = MetricType; - x_conf->nlist = 100; - x_conf->nprobe = 10; - conf = x_conf; - conf->Dump(); + + knowhere::Config temp_conf{ + {knowhere::meta::DIM, dim}, {knowhere::meta::TOPK, k}, + {knowhere::IndexParams::nlist, 100}, {knowhere::IndexParams::nprobe, 10}, + {knowhere::Metric::TYPE, MetricType}, + }; + conf = temp_conf; } void @@ -59,8 +58,7 @@ class BinaryIVFTest : public BinaryDataGen, public TestWithParamDimension(), dim); auto result = index_->Search(query_dataset, conf); - AssertAnns(result, nq, conf->k); + AssertAnns(result, nq, conf[knowhere::meta::TOPK]); // PrintResult(result, nq, k); faiss::ConcurrentBitsetPtr concurrent_bitset_ptr = std::make_shared(nb); @@ -123,7 +121,7 @@ TEST_P(BinaryIVFTest, binaryivf_serialize) { // index_->set_index_model(model); // index_->Add(base_dataset, conf); // auto result = index_->Search(query_dataset, conf); - // AssertAnns(result, nq, conf->k); + // AssertAnns(result, nq, conf[knowhere::meta::TOPK]); // } { @@ -147,7 +145,7 @@ TEST_P(BinaryIVFTest, binaryivf_serialize) { EXPECT_EQ(index_->Count(), nb); EXPECT_EQ(index_->Dimension(), dim); auto result = index_->Search(query_dataset, conf); - AssertAnns(result, nq, conf->k); + AssertAnns(result, nq, conf[knowhere::meta::TOPK]); // PrintResult(result, nq, k); } } diff --git a/core/src/index/unittest/test_customized_index.cpp b/core/src/index/unittest/test_customized_index.cpp index 258af4087e..f55da9acdf 100644 --- a/core/src/index/unittest/test_customized_index.cpp +++ b/core/src/index/unittest/test_customized_index.cpp @@ -69,7 +69,7 @@ TEST_F(SingleIndexTest, IVFSQHybrid) { for (int i = 0; i < 3; ++i) { auto gpu_idx = cpu_idx->CopyCpuToGpu(DEVICEID, conf); auto result = gpu_idx->Search(query_dataset, conf); - AssertAnns(result, nq, conf->k); + AssertAnns(result, nq, conf[knowhere::meta::TOPK]); // PrintResult(result, nq, k); } } @@ -86,30 +86,18 @@ TEST_F(SingleIndexTest, IVFSQHybrid) { auto quantization = pair.second; auto result = gpu_idx->Search(query_dataset, conf); - AssertAnns(result, nq, conf->k); + AssertAnns(result, nq, conf[knowhere::meta::TOPK]); // PrintResult(result, nq, k); - auto quantizer_conf = std::make_shared(); - quantizer_conf->mode = 2; // only copy data - quantizer_conf->gpu_id = DEVICEID; + milvus::json quantizer_conf{{knowhere::meta::DEVICEID, DEVICEID}, {"mode", 2}}; for (int i = 0; i < 2; ++i) { auto hybrid_idx = std::make_shared(DEVICEID); hybrid_idx->Load(binaryset); auto new_idx = hybrid_idx->LoadData(quantization, quantizer_conf); auto result = new_idx->Search(query_dataset, conf); - AssertAnns(result, nq, conf->k); + AssertAnns(result, nq, conf[knowhere::meta::TOPK]); // PrintResult(result, nq, k); } - - { - // invalid quantizer config - quantizer_conf = std::make_shared(); - auto hybrid_idx = std::make_shared(DEVICEID); - ASSERT_ANY_THROW(hybrid_idx->LoadData(quantization, nullptr)); - ASSERT_ANY_THROW(hybrid_idx->LoadData(quantization, quantizer_conf)); - quantizer_conf->mode = 2; // only copy data - ASSERT_ANY_THROW(hybrid_idx->LoadData(quantization, quantizer_conf)); - } } { @@ -126,7 +114,7 @@ TEST_F(SingleIndexTest, IVFSQHybrid) { hybrid_idx->SetQuantizer(quantization); auto result = hybrid_idx->Search(query_dataset, conf); - AssertAnns(result, nq, conf->k); + AssertAnns(result, nq, conf[knowhere::meta::TOPK]); // PrintResult(result, nq, k); hybrid_idx->UnsetQuantizer(); } diff --git a/core/src/index/unittest/test_idmap.cpp b/core/src/index/unittest/test_idmap.cpp index 38c06ca735..69d2c0ced9 100644 --- a/core/src/index/unittest/test_idmap.cpp +++ b/core/src/index/unittest/test_idmap.cpp @@ -45,10 +45,8 @@ class IDMAPTest : public DataGen, public TestGpuIndexBase { TEST_F(IDMAPTest, idmap_basic) { ASSERT_TRUE(!xb.empty()); - auto conf = std::make_shared(); - conf->d = dim; - conf->k = k; - conf->metric_type = knowhere::METRICTYPE::L2; + knowhere::Config conf{ + {knowhere::meta::DIM, dim}, {knowhere::meta::TOPK, k}, {knowhere::Metric::TYPE, knowhere::Metric::L2}}; // null faiss index { @@ -107,10 +105,8 @@ TEST_F(IDMAPTest, idmap_serialize) { reader(ret, bin->size); }; - auto conf = std::make_shared(); - conf->d = dim; - conf->k = k; - conf->metric_type = knowhere::METRICTYPE::L2; + knowhere::Config conf{ + {knowhere::meta::DIM, dim}, {knowhere::meta::TOPK, k}, {knowhere::Metric::TYPE, knowhere::Metric::L2}}; { // serialize index @@ -146,10 +142,8 @@ TEST_F(IDMAPTest, idmap_serialize) { TEST_F(IDMAPTest, copy_test) { ASSERT_TRUE(!xb.empty()); - auto conf = std::make_shared(); - conf->d = dim; - conf->k = k; - conf->metric_type = knowhere::METRICTYPE::L2; + knowhere::Config conf{ + {knowhere::meta::DIM, dim}, {knowhere::meta::TOPK, k}, {knowhere::Metric::TYPE, knowhere::Metric::L2}}; index_->Train(conf); index_->Add(base_dataset, conf); diff --git a/core/src/index/unittest/test_ivf.cpp b/core/src/index/unittest/test_ivf.cpp index 13c50260c1..b30cffae51 100644 --- a/core/src/index/unittest/test_ivf.cpp +++ b/core/src/index/unittest/test_ivf.cpp @@ -62,7 +62,7 @@ class IVFTest : public DataGen, public TestWithParam<::std::tupleDump(); + // KNOWHERE_LOG_DEBUG << "conf: " << conf->dump(); } void @@ -109,7 +109,7 @@ TEST_P(IVFTest, ivf_basic) { EXPECT_EQ(index_->Dimension(), dim); auto result = index_->Search(query_dataset, conf); - AssertAnns(result, nq, conf->k); + AssertAnns(result, nq, conf[knowhere::meta::TOPK]); // PrintResult(result, nq, k); if (index_type.find("GPU") == std::string::npos && index_type.find("Hybrid") == std::string::npos && @@ -190,7 +190,7 @@ TEST_P(IVFTest, ivf_serialize) { index_->set_index_model(model); index_->Add(base_dataset, conf); auto result = index_->Search(query_dataset, conf); - AssertAnns(result, nq, conf->k); + AssertAnns(result, nq, conf[knowhere::meta::TOPK]); } { @@ -214,7 +214,7 @@ TEST_P(IVFTest, ivf_serialize) { EXPECT_EQ(index_->Count(), nb); EXPECT_EQ(index_->Dimension(), dim); auto result = index_->Search(query_dataset, conf); - AssertAnns(result, nq, conf->k); + AssertAnns(result, nq, conf[knowhere::meta::TOPK]); } } @@ -232,7 +232,7 @@ TEST_P(IVFTest, clone_test) { EXPECT_EQ(index_->Count(), nb); EXPECT_EQ(index_->Dimension(), dim); auto result = index_->Search(query_dataset, conf); - AssertAnns(result, nq, conf->k); + AssertAnns(result, nq, conf[knowhere::meta::TOPK]); // PrintResult(result, nq, k); auto AssertEqual = [&](knowhere::DatasetPtr p1, knowhere::DatasetPtr p2) { @@ -254,7 +254,7 @@ TEST_P(IVFTest, clone_test) { // EXPECT_NO_THROW({ // auto clone_index = index_->Clone(); // auto clone_result = clone_index->Search(query_dataset, conf); - // //AssertAnns(result, nq, conf->k); + // //AssertAnns(result, nq, conf[knowhere::meta::TOPK]); // AssertEqual(result, clone_result); // std::cout << "inplace clone [" << index_type << "] success" << std::endl; // }); @@ -339,7 +339,7 @@ TEST_P(IVFTest, gpu_seal_test) { EXPECT_EQ(index_->Count(), nb); EXPECT_EQ(index_->Dimension(), dim); auto result = index_->Search(query_dataset, conf); - AssertAnns(result, nq, conf->k); + AssertAnns(result, nq, conf[knowhere::meta::TOPK]); fiu_init(0); fiu_enable("IVF.Search.throw_std_exception", 1, nullptr, 0); @@ -374,7 +374,7 @@ TEST_P(IVFTest, invalid_gpu_source) { } auto invalid_conf = ParamGenerator::GetInstance().Gen(parameter_type_); - invalid_conf->gpu_id = -1; + invalid_conf[knowhere::meta::DEVICEID] = -1; if (index_type == "GPUIVF") { // null faiss index @@ -430,15 +430,6 @@ TEST_P(IVFTest, IVFSQHybrid_test) { ASSERT_TRUE(index != nullptr); ASSERT_ANY_THROW(index->UnsetQuantizer()); - knowhere::QuantizerConfig config = std::make_shared(); - config->gpu_id = knowhere::INVALID_VALUE; - - // mode = -1 - ASSERT_ANY_THROW(index->LoadQuantizer(config)); - config->mode = 1; - ASSERT_ANY_THROW(index->LoadQuantizer(config)); - config->gpu_id = DEVICEID; - // index->LoadQuantizer(config); ASSERT_ANY_THROW(index->SetQuantizer(nullptr)); } diff --git a/core/src/index/unittest/test_nsg/test_nsg.cpp b/core/src/index/unittest/test_nsg/test_nsg.cpp index 78499ee166..76db0cd8c2 100644 --- a/core/src/index/unittest/test_nsg/test_nsg.cpp +++ b/core/src/index/unittest/test_nsg/test_nsg.cpp @@ -46,24 +46,19 @@ class NSGInterfaceTest : public DataGen, public ::testing::Test { Generate(256, 1000000 / 100, 1); index_ = std::make_shared(); - auto tmp_conf = std::make_shared(); - tmp_conf->gpu_id = DEVICEID; - tmp_conf->d = 256; - tmp_conf->knng = 20; - tmp_conf->nprobe = 8; - tmp_conf->nlist = 163; - tmp_conf->search_length = 40; - tmp_conf->out_degree = 30; - tmp_conf->candidate_pool_size = 100; - tmp_conf->metric_type = knowhere::METRICTYPE::L2; - train_conf = tmp_conf; - train_conf->Dump(); + train_conf = knowhere::Config{{knowhere::meta::DIM, 256}, + {knowhere::IndexParams::nlist, 163}, + {knowhere::IndexParams::nprobe, 8}, + {knowhere::IndexParams::knng, 20}, + {knowhere::IndexParams::search_length, 40}, + {knowhere::IndexParams::out_degree, 30}, + {knowhere::IndexParams::candidate, 100}, + {knowhere::Metric::TYPE, knowhere::Metric::L2}}; - auto tmp2_conf = std::make_shared(); - tmp2_conf->k = k; - tmp2_conf->search_length = 30; - search_conf = tmp2_conf; - search_conf->Dump(); + search_conf = knowhere::Config{ + {knowhere::meta::TOPK, k}, + {knowhere::IndexParams::search_length, 30}, + }; } void @@ -87,9 +82,9 @@ TEST_F(NSGInterfaceTest, basic_test) { ASSERT_ANY_THROW(index_->Search(query_dataset, search_conf)); ASSERT_ANY_THROW(index_->Serialize()); } - train_conf->gpu_id = knowhere::INVALID_VALUE; - auto model_invalid_gpu = index_->Train(base_dataset, train_conf); - train_conf->gpu_id = DEVICEID; + // train_conf->gpu_id = knowhere::INVALID_VALUE; + // auto model_invalid_gpu = index_->Train(base_dataset, train_conf); + train_conf[knowhere::meta::DEVICEID] = DEVICEID; auto model = index_->Train(base_dataset, train_conf); auto result = index_->Search(query_dataset, search_conf); AssertAnns(result, nq, k); diff --git a/core/src/index/unittest/test_sptag.cpp b/core/src/index/unittest/test_sptag.cpp index d33520a759..bcaebb7da8 100644 --- a/core/src/index/unittest/test_sptag.cpp +++ b/core/src/index/unittest/test_sptag.cpp @@ -33,17 +33,17 @@ class SPTAGTest : public DataGen, public TestWithParam { Generate(128, 100, 5); index_ = std::make_shared(IndexType); if (IndexType == "KDT") { - auto tempconf = std::make_shared(); - tempconf->tptnumber = 1; - tempconf->k = 10; - tempconf->metric_type = knowhere::METRICTYPE::L2; - conf = tempconf; + conf = knowhere::Config{ + {knowhere::meta::DIM, dim}, + {knowhere::meta::TOPK, 10}, + {knowhere::Metric::TYPE, knowhere::Metric::L2}, + }; } else { - auto tempconf = std::make_shared(); - tempconf->tptnumber = 1; - tempconf->k = 10; - tempconf->metric_type = knowhere::METRICTYPE::L2; - conf = tempconf; + conf = knowhere::Config{ + {knowhere::meta::DIM, dim}, + {knowhere::meta::TOPK, 10}, + {knowhere::Metric::TYPE, knowhere::Metric::L2}, + }; } Init_with_default(); diff --git a/core/src/scheduler/job/SearchJob.cpp b/core/src/scheduler/job/SearchJob.cpp index 6d15fd71b0..55655eb8ff 100644 --- a/core/src/scheduler/job/SearchJob.cpp +++ b/core/src/scheduler/job/SearchJob.cpp @@ -16,9 +16,9 @@ namespace milvus { namespace scheduler { -SearchJob::SearchJob(const std::shared_ptr& context, uint64_t topk, uint64_t nprobe, +SearchJob::SearchJob(const std::shared_ptr& context, uint64_t topk, const milvus::json& extra_params, const engine::VectorsData& vectors) - : Job(JobType::SEARCH), context_(context), topk_(topk), nprobe_(nprobe), vectors_(vectors) { + : Job(JobType::SEARCH), context_(context), topk_(topk), extra_params_(extra_params), vectors_(vectors) { } bool @@ -72,7 +72,7 @@ SearchJob::Dump() const { json ret{ {"topk", topk_}, {"nq", vectors_.vector_count_}, - {"nprobe", nprobe_}, + {"extra_params", extra_params_.dump()}, }; auto base = Job::Dump(); ret.insert(base.begin(), base.end()); diff --git a/core/src/scheduler/job/SearchJob.h b/core/src/scheduler/job/SearchJob.h index 687c287460..8ae0fe77ba 100644 --- a/core/src/scheduler/job/SearchJob.h +++ b/core/src/scheduler/job/SearchJob.h @@ -40,7 +40,7 @@ using ResultDistances = engine::ResultDistances; class SearchJob : public Job { public: - SearchJob(const std::shared_ptr& context, uint64_t topk, uint64_t nprobe, + SearchJob(const std::shared_ptr& context, uint64_t topk, const milvus::json& extra_params, const engine::VectorsData& vectors); public: @@ -79,9 +79,9 @@ class SearchJob : public Job { return vectors_.vector_count_; } - uint64_t - nprobe() const { - return nprobe_; + const milvus::json& + extra_params() const { + return extra_params_; } const engine::VectorsData& @@ -103,7 +103,7 @@ class SearchJob : public Job { const std::shared_ptr context_; uint64_t topk_ = 0; - uint64_t nprobe_ = 0; + milvus::json extra_params_; // TODO: smart pointer const engine::VectorsData& vectors_; diff --git a/core/src/scheduler/task/BuildIndexTask.cpp b/core/src/scheduler/task/BuildIndexTask.cpp index a6bc8b88aa..13d3b4a611 100644 --- a/core/src/scheduler/task/BuildIndexTask.cpp +++ b/core/src/scheduler/task/BuildIndexTask.cpp @@ -41,8 +41,9 @@ XBuildIndexTask::XBuildIndexTask(TableFileSchemaPtr file, TaskLabelPtr label) engine_type = (EngineType)file->engine_type_; } + auto json = milvus::json::parse(file_->index_params_); to_index_engine_ = EngineFactory::Build(file_->dimension_, file_->location_, engine_type, - (MetricType)file_->metric_type_, file_->nlist_); + (MetricType)file_->metric_type_, json); } } diff --git a/core/src/scheduler/task/SearchTask.cpp b/core/src/scheduler/task/SearchTask.cpp index a3bcf27874..0b32f35ab5 100644 --- a/core/src/scheduler/task/SearchTask.cpp +++ b/core/src/scheduler/task/SearchTask.cpp @@ -115,8 +115,12 @@ XSearchTask::XSearchTask(const std::shared_ptr& context, TableF engine_type = (EngineType)file->engine_type_; } + milvus::json json_params; + if (!file_->index_params_.empty()) { + json_params = milvus::json::parse(file_->index_params_); + } index_engine_ = EngineFactory::Build(file_->dimension_, file_->location_, engine_type, - (MetricType)file_->metric_type_, file_->nlist_); + (MetricType)file_->metric_type_, json_params); } } @@ -217,7 +221,8 @@ XSearchTask::Execute() { // step 1: allocate memory uint64_t nq = search_job->nq(); uint64_t topk = search_job->topk(); - uint64_t nprobe = search_job->nprobe(); + const milvus::json& extra_params = search_job->extra_params(); + ENGINE_LOG_DEBUG << "Search job extra params: " << extra_params.dump(); const engine::VectorsData& vectors = search_job->vectors(); output_ids.resize(topk * nq); @@ -235,13 +240,13 @@ XSearchTask::Execute() { } Status s; if (!vectors.float_data_.empty()) { - s = index_engine_->Search(nq, vectors.float_data_.data(), topk, nprobe, output_distance.data(), + s = index_engine_->Search(nq, vectors.float_data_.data(), topk, extra_params, output_distance.data(), output_ids.data(), hybrid); } else if (!vectors.binary_data_.empty()) { - s = index_engine_->Search(nq, vectors.binary_data_.data(), topk, nprobe, output_distance.data(), + s = index_engine_->Search(nq, vectors.binary_data_.data(), topk, extra_params, output_distance.data(), output_ids.data(), hybrid); } else if (!vectors.id_array_.empty()) { - s = index_engine_->Search(nq, vectors.id_array_, topk, nprobe, output_distance.data(), + s = index_engine_->Search(nq, vectors.id_array_, topk, extra_params, output_distance.data(), output_ids.data(), hybrid); } diff --git a/core/src/server/Config.cpp b/core/src/server/Config.cpp index 5c519530b2..f0b88fa98c 100644 --- a/core/src/server/Config.cpp +++ b/core/src/server/Config.cpp @@ -549,9 +549,14 @@ Config::UpdateFileConfigFromMem(const std::string& parent_key, const std::string // convert value string to standard string stored in yaml file std::string value_str; if (child_key == CONFIG_CACHE_CACHE_INSERT_DATA || child_key == CONFIG_STORAGE_S3_ENABLE || - child_key == CONFIG_METRIC_ENABLE_MONITOR || child_key == CONFIG_GPU_RESOURCE_ENABLE) { - value_str = - (value == "True" || value == "true" || value == "On" || value == "on" || value == "1") ? "true" : "false"; + child_key == CONFIG_METRIC_ENABLE_MONITOR || child_key == CONFIG_GPU_RESOURCE_ENABLE || + child_key == CONFIG_WAL_ENABLE || child_key == CONFIG_WAL_RECOVERY_ERROR_IGNORE) { + bool ok = false; + status = StringHelpFunctions::ConvertToBoolean(value, ok); + if (!status.ok()) { + return status; + } + value_str = ok ? "true" : "false"; } else if (child_key == CONFIG_GPU_RESOURCE_SEARCH_RESOURCES || child_key == CONFIG_GPU_RESOURCE_BUILD_INDEX_RESOURCES) { std::vector vec; @@ -593,7 +598,6 @@ Config::UpdateFileConfigFromMem(const std::string& parent_key, const std::string } // values of gpu resources are sequences, need to remove old here - std::regex reg("\\S*"); if (child_key == CONFIG_GPU_RESOURCE_SEARCH_RESOURCES || child_key == CONFIG_GPU_RESOURCE_BUILD_INDEX_RESOURCES) { while (getline(conf_fin, line)) { if (line.find("- gpu") != std::string::npos) @@ -1009,8 +1013,7 @@ Config::CheckCacheConfigCpuCacheCapacity(const std::string& value) { std::cerr << "WARNING: cpu cache capacity value is too big" << std::endl; } - std::string str = - GetConfigStr(CONFIG_CACHE, CONFIG_CACHE_INSERT_BUFFER_SIZE, CONFIG_CACHE_INSERT_BUFFER_SIZE_DEFAULT); + std::string str = GetConfigStr(CONFIG_CACHE, CONFIG_CACHE_INSERT_BUFFER_SIZE, "0"); int64_t buffer_value = std::stoll(str); int64_t insert_buffer_size = buffer_value * GB; @@ -1059,9 +1062,8 @@ Config::CheckCacheConfigInsertBufferSize(const std::string& value) { return Status(SERVER_INVALID_ARGUMENT, msg); } - std::string str = - GetConfigStr(CONFIG_CACHE, CONFIG_CACHE_CPU_CACHE_CAPACITY, CONFIG_CACHE_CPU_CACHE_CAPACITY_DEFAULT); - int64_t cache_size = std::stoll(str); + std::string str = GetConfigStr(CONFIG_CACHE, CONFIG_CACHE_CPU_CACHE_CAPACITY, "0"); + int64_t cache_size = std::stoll(str) * GB; uint64_t total_mem = 0, free_mem = 0; CommonUtil::GetSystemMemInfo(total_mem, free_mem); @@ -1855,7 +1857,8 @@ Config::SetDBConfigBackendUrl(const std::string& value) { Status Config::SetDBConfigPreloadTable(const std::string& value) { CONFIG_CHECK(CheckDBConfigPreloadTable(value)); - return SetConfigValueInMem(CONFIG_DB, CONFIG_DB_PRELOAD_TABLE, value); + std::string cor_value = value == "*" ? "\'*\'" : value; + return SetConfigValueInMem(CONFIG_DB, CONFIG_DB_PRELOAD_TABLE, cor_value); } Status diff --git a/core/src/server/delivery/RequestHandler.cpp b/core/src/server/delivery/RequestHandler.cpp index afff3f64a5..b416e721d0 100644 --- a/core/src/server/delivery/RequestHandler.cpp +++ b/core/src/server/delivery/RequestHandler.cpp @@ -70,8 +70,8 @@ RequestHandler::DropTable(const std::shared_ptr& context, const std::st Status RequestHandler::CreateIndex(const std::shared_ptr& context, const std::string& table_name, int64_t index_type, - int64_t nlist) { - BaseRequestPtr request_ptr = CreateIndexRequest::Create(context, table_name, index_type, nlist); + const milvus::json& json_params) { + BaseRequestPtr request_ptr = CreateIndexRequest::Create(context, table_name, index_type, json_params); RequestScheduler::ExecRequest(request_ptr); return request_ptr->status(); @@ -123,11 +123,11 @@ RequestHandler::ShowTableInfo(const std::shared_ptr& context, const std Status RequestHandler::Search(const std::shared_ptr& context, const std::string& table_name, - const engine::VectorsData& vectors, int64_t topk, int64_t nprobe, + const engine::VectorsData& vectors, int64_t topk, const milvus::json& extra_params, const std::vector& partition_list, const std::vector& file_id_list, TopKQueryResult& result) { BaseRequestPtr request_ptr = - SearchRequest::Create(context, table_name, vectors, topk, nprobe, partition_list, file_id_list, result); + SearchRequest::Create(context, table_name, vectors, topk, extra_params, partition_list, file_id_list, result); RequestScheduler::ExecRequest(request_ptr); return request_ptr->status(); @@ -135,10 +135,10 @@ RequestHandler::Search(const std::shared_ptr& context, const std::strin Status RequestHandler::SearchByID(const std::shared_ptr& context, const std::string& table_name, int64_t vector_id, - int64_t topk, int64_t nprobe, const std::vector& partition_list, - TopKQueryResult& result) { + int64_t topk, const milvus::json& extra_params, + const std::vector& partition_list, TopKQueryResult& result) { BaseRequestPtr request_ptr = - SearchByIDRequest::Create(context, table_name, vector_id, topk, nprobe, partition_list, result); + SearchByIDRequest::Create(context, table_name, vector_id, topk, extra_params, partition_list, result); RequestScheduler::ExecRequest(request_ptr); return request_ptr->status(); diff --git a/core/src/server/delivery/RequestHandler.h b/core/src/server/delivery/RequestHandler.h index 804c01afea..8b9bb68b5a 100644 --- a/core/src/server/delivery/RequestHandler.h +++ b/core/src/server/delivery/RequestHandler.h @@ -38,7 +38,7 @@ class RequestHandler { Status CreateIndex(const std::shared_ptr& context, const std::string& table_name, int64_t index_type, - int64_t nlist); + const milvus::json& json_params); Status Insert(const std::shared_ptr& context, const std::string& table_name, engine::VectorsData& vectors, @@ -60,12 +60,13 @@ class RequestHandler { Status Search(const std::shared_ptr& context, const std::string& table_name, const engine::VectorsData& vectors, - int64_t topk, int64_t nprobe, const std::vector& partition_list, + int64_t topk, const milvus::json& extra_params, const std::vector& partition_list, const std::vector& file_id_list, TopKQueryResult& result); Status SearchByID(const std::shared_ptr& context, const std::string& table_name, int64_t vector_id, int64_t topk, - int64_t nprobe, const std::vector& partition_list, TopKQueryResult& result); + const milvus::json& extra_params, const std::vector& partition_list, + TopKQueryResult& result); Status DescribeTable(const std::shared_ptr& context, const std::string& table_name, TableSchema& table_schema); diff --git a/core/src/server/delivery/request/BaseRequest.h b/core/src/server/delivery/request/BaseRequest.h index a9fa921df5..7ee665695a 100644 --- a/core/src/server/delivery/request/BaseRequest.h +++ b/core/src/server/delivery/request/BaseRequest.h @@ -17,6 +17,7 @@ #include "grpc/gen-status/status.grpc.pb.h" #include "grpc/gen-status/status.pb.h" #include "server/context/Context.h" +#include "utils/Json.h" #include "utils/Status.h" #include @@ -73,17 +74,15 @@ struct TopKQueryResult { struct IndexParam { std::string table_name_; int64_t index_type_; - int64_t nlist_; + std::string extra_params_; IndexParam() { index_type_ = 0; - nlist_ = 0; } - IndexParam(const std::string& table_name, int64_t index_type, int64_t nlist) { + IndexParam(const std::string& table_name, int64_t index_type) { table_name_ = table_name; index_type_ = index_type; - nlist_ = nlist; } }; diff --git a/core/src/server/delivery/request/CreateIndexRequest.cpp b/core/src/server/delivery/request/CreateIndexRequest.cpp index c7011396a3..5836b3cc09 100644 --- a/core/src/server/delivery/request/CreateIndexRequest.cpp +++ b/core/src/server/delivery/request/CreateIndexRequest.cpp @@ -24,14 +24,17 @@ namespace milvus { namespace server { CreateIndexRequest::CreateIndexRequest(const std::shared_ptr& context, const std::string& table_name, - int64_t index_type, int64_t nlist) - : BaseRequest(context, DDL_DML_REQUEST_GROUP), table_name_(table_name), index_type_(index_type), nlist_(nlist) { + int64_t index_type, const milvus::json& json_params) + : BaseRequest(context, DDL_DML_REQUEST_GROUP), + table_name_(table_name), + index_type_(index_type), + json_params_(json_params) { } BaseRequestPtr CreateIndexRequest::Create(const std::shared_ptr& context, const std::string& table_name, int64_t index_type, - int64_t nlist) { - return std::shared_ptr(new CreateIndexRequest(context, table_name, index_type, nlist)); + const milvus::json& json_params) { + return std::shared_ptr(new CreateIndexRequest(context, table_name, index_type, json_params)); } Status @@ -69,7 +72,7 @@ CreateIndexRequest::OnExecute() { return status; } - status = ValidationUtil::ValidateTableIndexNlist(nlist_); + status = ValidationUtil::ValidateIndexParams(json_params_, table_schema, index_type_); if (!status.ok()) { return status; } @@ -109,7 +112,7 @@ CreateIndexRequest::OnExecute() { // step 3: create index engine::TableIndex index; index.engine_type_ = adapter_index_type; - index.nlist_ = nlist_; + index.extra_params_ = json_params_; status = DBWrapper::DB()->CreateIndex(table_name_, index); fiu_do_on("CreateIndexRequest.OnExecute.create_index_fail", status = Status(milvus::SERVER_UNEXPECTED_ERROR, "")); diff --git a/core/src/server/delivery/request/CreateIndexRequest.h b/core/src/server/delivery/request/CreateIndexRequest.h index 7c5f3f549c..2f52fe94f0 100644 --- a/core/src/server/delivery/request/CreateIndexRequest.h +++ b/core/src/server/delivery/request/CreateIndexRequest.h @@ -21,11 +21,12 @@ namespace server { class CreateIndexRequest : public BaseRequest { public: static BaseRequestPtr - Create(const std::shared_ptr& context, const std::string& table_name, int64_t index_type, int64_t nlist); + Create(const std::shared_ptr& context, const std::string& table_name, int64_t index_type, + const milvus::json& json_params); protected: CreateIndexRequest(const std::shared_ptr& context, const std::string& table_name, int64_t index_type, - int64_t nlist); + const milvus::json& json_params); Status OnExecute() override; @@ -33,7 +34,7 @@ class CreateIndexRequest : public BaseRequest { private: const std::string table_name_; const int64_t index_type_; - const int64_t nlist_; + milvus::json json_params_; }; } // namespace server diff --git a/core/src/server/delivery/request/DescribeIndexRequest.cpp b/core/src/server/delivery/request/DescribeIndexRequest.cpp index 17b5a4e867..fea640611b 100644 --- a/core/src/server/delivery/request/DescribeIndexRequest.cpp +++ b/core/src/server/delivery/request/DescribeIndexRequest.cpp @@ -78,7 +78,7 @@ DescribeIndexRequest::OnExecute() { index_param_.table_name_ = table_name_; index_param_.index_type_ = index.engine_type_; - index_param_.nlist_ = index.nlist_; + index_param_.extra_params_ = index.extra_params_.dump(); } catch (std::exception& ex) { return Status(SERVER_UNEXPECTED_ERROR, ex.what()); } diff --git a/core/src/server/delivery/request/SearchByIDRequest.cpp b/core/src/server/delivery/request/SearchByIDRequest.cpp index 7de23097b1..c48fe973d8 100644 --- a/core/src/server/delivery/request/SearchByIDRequest.cpp +++ b/core/src/server/delivery/request/SearchByIDRequest.cpp @@ -34,23 +34,23 @@ namespace milvus { namespace server { SearchByIDRequest::SearchByIDRequest(const std::shared_ptr& context, const std::string& table_name, - int64_t vector_id, int64_t topk, int64_t nprobe, + int64_t vector_id, int64_t topk, const milvus::json& extra_params, const std::vector& partition_list, TopKQueryResult& result) : BaseRequest(context, DQL_REQUEST_GROUP), table_name_(table_name), vector_id_(vector_id), topk_(topk), - nprobe_(nprobe), + extra_params_(extra_params), partition_list_(partition_list), result_(result) { } BaseRequestPtr SearchByIDRequest::Create(const std::shared_ptr& context, const std::string& table_name, int64_t vector_id, - int64_t topk, int64_t nprobe, const std::vector& partition_list, - TopKQueryResult& result) { + int64_t topk, const milvus::json& extra_params, + const std::vector& partition_list, TopKQueryResult& result) { return std::shared_ptr( - new SearchByIDRequest(context, table_name, vector_id, topk, nprobe, partition_list, result)); + new SearchByIDRequest(context, table_name, vector_id, topk, extra_params, partition_list, result)); } Status @@ -59,7 +59,7 @@ SearchByIDRequest::OnExecute() { auto pre_query_ctx = context_->Child("Pre query"); std::string hdr = "SearchByIDRequest(table=" + table_name_ + ", id=" + std::to_string(vector_id_) + - ", k=" + std::to_string(topk_) + ", nprob=" + std::to_string(nprobe_) + ")"; + ", k=" + std::to_string(topk_) + ", extra_params=" + extra_params_.dump() + ")"; TimeRecorder rc(hdr); @@ -88,6 +88,11 @@ SearchByIDRequest::OnExecute() { } } + status = ValidationUtil::ValidateSearchParams(extra_params_, table_schema, topk_); + if (!status.ok()) { + return status; + } + // Check whether GPU search resource is enabled #ifdef MILVUS_GPU_VERSION Config& config = Config::GetInstance(); @@ -122,11 +127,6 @@ SearchByIDRequest::OnExecute() { return status; } - status = ValidationUtil::ValidateSearchNprobe(nprobe_, table_schema); - if (!status.ok()) { - return status; - } - rc.RecordSection("check validation"); // step 5: search vectors @@ -140,8 +140,8 @@ SearchByIDRequest::OnExecute() { pre_query_ctx->GetTraceContext()->GetSpan()->Finish(); - status = DBWrapper::DB()->QueryByID(context_, table_name_, partition_list_, (size_t)topk_, nprobe_, vector_id_, - result_ids, result_distances); + status = DBWrapper::DB()->QueryByID(context_, table_name_, partition_list_, (size_t)topk_, extra_params_, + vector_id_, result_ids, result_distances); #ifdef MILVUS_ENABLE_PROFILING ProfilerStop(); diff --git a/core/src/server/delivery/request/SearchByIDRequest.h b/core/src/server/delivery/request/SearchByIDRequest.h index ff60faf90b..1a85dc5d9d 100644 --- a/core/src/server/delivery/request/SearchByIDRequest.h +++ b/core/src/server/delivery/request/SearchByIDRequest.h @@ -30,11 +30,11 @@ class SearchByIDRequest : public BaseRequest { public: static BaseRequestPtr Create(const std::shared_ptr& context, const std::string& table_name, int64_t vector_id, int64_t topk, - int64_t nprobe, const std::vector& partition_list, TopKQueryResult& result); + const milvus::json& extra_params, const std::vector& partition_list, TopKQueryResult& result); protected: SearchByIDRequest(const std::shared_ptr& context, const std::string& table_name, int64_t vector_id, - int64_t topk, int64_t nprobe, const std::vector& partition_list, + int64_t topk, const milvus::json& extra_params, const std::vector& partition_list, TopKQueryResult& result); Status @@ -44,7 +44,7 @@ class SearchByIDRequest : public BaseRequest { const std::string table_name_; const int64_t vector_id_; int64_t topk_; - int64_t nprobe_; + milvus::json extra_params_; const std::vector partition_list_; TopKQueryResult& result_; diff --git a/core/src/server/delivery/request/SearchRequest.cpp b/core/src/server/delivery/request/SearchRequest.cpp index 0e0128490c..0b12f91f5d 100644 --- a/core/src/server/delivery/request/SearchRequest.cpp +++ b/core/src/server/delivery/request/SearchRequest.cpp @@ -26,14 +26,14 @@ namespace milvus { namespace server { SearchRequest::SearchRequest(const std::shared_ptr& context, const std::string& table_name, - const engine::VectorsData& vectors, int64_t topk, int64_t nprobe, + const engine::VectorsData& vectors, int64_t topk, const milvus::json& extra_params, const std::vector& partition_list, const std::vector& file_id_list, TopKQueryResult& result) : BaseRequest(context, DQL_REQUEST_GROUP), table_name_(table_name), vectors_data_(vectors), topk_(topk), - nprobe_(nprobe), + extra_params_(extra_params), partition_list_(partition_list), file_id_list_(file_id_list), result_(result) { @@ -41,11 +41,11 @@ SearchRequest::SearchRequest(const std::shared_ptr& context, const std: BaseRequestPtr SearchRequest::Create(const std::shared_ptr& context, const std::string& table_name, - const engine::VectorsData& vectors, int64_t topk, int64_t nprobe, + const engine::VectorsData& vectors, int64_t topk, const milvus::json& extra_params, const std::vector& partition_list, const std::vector& file_id_list, TopKQueryResult& result) { return std::shared_ptr( - new SearchRequest(context, table_name, vectors, topk, nprobe, partition_list, file_id_list, result)); + new SearchRequest(context, table_name, vectors, topk, extra_params, partition_list, file_id_list, result)); } Status @@ -56,7 +56,7 @@ SearchRequest::OnExecute() { auto pre_query_ctx = context_->Child("Pre query"); std::string hdr = "SearchRequest(table=" + table_name_ + ", nq=" + std::to_string(vector_count) + - ", k=" + std::to_string(topk_) + ", nprob=" + std::to_string(nprobe_) + ")"; + ", k=" + std::to_string(topk_) + ", extra_params=" + extra_params_.dump() + ")"; TimeRecorder rc(hdr); @@ -84,13 +84,13 @@ SearchRequest::OnExecute() { } } - // step 3: check search parameter - status = ValidationUtil::ValidateSearchTopk(topk_, table_schema); + status = ValidationUtil::ValidateSearchParams(extra_params_, table_schema, topk_); if (!status.ok()) { return status; } - status = ValidationUtil::ValidateSearchNprobe(nprobe_, table_schema); + // step 3: check search parameter + status = ValidationUtil::ValidateSearchTopk(topk_, table_schema); if (!status.ok()) { return status; } @@ -150,10 +150,10 @@ SearchRequest::OnExecute() { return status; } - status = DBWrapper::DB()->Query(context_, table_name_, partition_list_, (size_t)topk_, nprobe_, + status = DBWrapper::DB()->Query(context_, table_name_, partition_list_, (size_t)topk_, extra_params_, vectors_data_, result_ids, result_distances); } else { - status = DBWrapper::DB()->QueryByFileID(context_, table_name_, file_id_list_, (size_t)topk_, nprobe_, + status = DBWrapper::DB()->QueryByFileID(context_, table_name_, file_id_list_, (size_t)topk_, extra_params_, vectors_data_, result_ids, result_distances); } diff --git a/core/src/server/delivery/request/SearchRequest.h b/core/src/server/delivery/request/SearchRequest.h index 6cbb8957f9..43db9cfff9 100644 --- a/core/src/server/delivery/request/SearchRequest.h +++ b/core/src/server/delivery/request/SearchRequest.h @@ -24,12 +24,12 @@ class SearchRequest : public BaseRequest { public: static BaseRequestPtr Create(const std::shared_ptr& context, const std::string& table_name, const engine::VectorsData& vectors, - int64_t topk, int64_t nprobe, const std::vector& partition_list, + int64_t topk, const milvus::json& extra_params, const std::vector& partition_list, const std::vector& file_id_list, TopKQueryResult& result); protected: SearchRequest(const std::shared_ptr& context, const std::string& table_name, - const engine::VectorsData& vectors, int64_t topk, int64_t nprobe, + const engine::VectorsData& vectors, int64_t topk, const milvus::json& extra_params, const std::vector& partition_list, const std::vector& file_id_list, TopKQueryResult& result); @@ -40,7 +40,7 @@ class SearchRequest : public BaseRequest { const std::string table_name_; const engine::VectorsData& vectors_data_; int64_t topk_; - int64_t nprobe_; + milvus::json extra_params_; const std::vector partition_list_; const std::vector file_id_list_; diff --git a/core/src/server/grpc_impl/GrpcRequestHandler.cpp b/core/src/server/grpc_impl/GrpcRequestHandler.cpp index a3866445a0..ed19171660 100644 --- a/core/src/server/grpc_impl/GrpcRequestHandler.cpp +++ b/core/src/server/grpc_impl/GrpcRequestHandler.cpp @@ -280,8 +280,16 @@ GrpcRequestHandler::CreateIndex(::grpc::ServerContext* context, const ::milvus:: ::milvus::grpc::Status* response) { CHECK_NULLPTR_RETURN(request); - Status status = request_handler_.CreateIndex(context_map_[context], request->table_name(), - request->index().index_type(), request->index().nlist()); + milvus::json json_params; + for (int i = 0; i < request->extra_params_size(); i++) { + const ::milvus::grpc::KeyValuePair& extra = request->extra_params(i); + if (extra.key() == EXTRA_PARAM_KEY) { + json_params = json::parse(extra.value()); + } + } + + Status status = + request_handler_.CreateIndex(context_map_[context], request->table_name(), request->index_type(), json_params); SET_RESPONSE(response, status, context); return ::grpc::Status::OK; @@ -366,14 +374,23 @@ GrpcRequestHandler::Search(::grpc::ServerContext* context, const ::milvus::grpc: partitions.emplace_back(partition); } - // step 3: search vectors + // step 3: parse extra parameters + milvus::json json_params; + for (int i = 0; i < request->extra_params_size(); i++) { + const ::milvus::grpc::KeyValuePair& extra = request->extra_params(i); + if (extra.key() == EXTRA_PARAM_KEY) { + json_params = json::parse(extra.value()); + } + } + + // step 4: search vectors std::vector file_ids; TopKQueryResult result; fiu_do_on("GrpcRequestHandler.Search.not_empty_file_ids", file_ids.emplace_back("test_file_id")); Status status = request_handler_.Search(context_map_[context], request->table_name(), vectors, request->topk(), - request->nprobe(), partitions, file_ids, result); + json_params, partitions, file_ids, result); - // step 4: construct and return result + // step 5: construct and return result ConstructResults(result, response); SET_RESPONSE(response->mutable_status(), status, context); @@ -392,12 +409,21 @@ GrpcRequestHandler::SearchByID(::grpc::ServerContext* context, const ::milvus::g partitions.emplace_back(partition); } - // step 2: search vectors + // step 2: parse extra parameters + milvus::json json_params; + for (int i = 0; i < request->extra_params_size(); i++) { + const ::milvus::grpc::KeyValuePair& extra = request->extra_params(i); + if (extra.key() == EXTRA_PARAM_KEY) { + json_params = json::parse(extra.value()); + } + } + + // step 3: search vectors TopKQueryResult result; Status status = request_handler_.SearchByID(context_map_[context], request->table_name(), request->id(), - request->topk(), request->nprobe(), partitions, result); + request->topk(), json_params, partitions, result); - // step 3: construct and return result + // step 4: construct and return result ConstructResults(result, response); SET_RESPONSE(response->mutable_status(), status, context); @@ -429,13 +455,21 @@ GrpcRequestHandler::SearchInFiles(::grpc::ServerContext* context, const ::milvus partitions.emplace_back(partition); } - // step 4: search vectors - TopKQueryResult result; - Status status = - request_handler_.Search(context_map_[context], search_request->table_name(), vectors, search_request->topk(), - search_request->nprobe(), partitions, file_ids, result); + // step 4: parse extra parameters + milvus::json json_params; + for (int i = 0; i < search_request->extra_params_size(); i++) { + const ::milvus::grpc::KeyValuePair& extra = search_request->extra_params(i); + if (extra.key() == EXTRA_PARAM_KEY) { + json_params = json::parse(extra.value()); + } + } - // step 5: construct and return result + // step 5: search vectors + TopKQueryResult result; + Status status = request_handler_.Search(context_map_[context], search_request->table_name(), vectors, + search_request->topk(), json_params, partitions, file_ids, result); + + // step 6: construct and return result ConstructResults(result, response); SET_RESPONSE(response->mutable_status(), status, context); @@ -549,8 +583,10 @@ GrpcRequestHandler::DescribeIndex(::grpc::ServerContext* context, const ::milvus IndexParam param; Status status = request_handler_.DescribeIndex(context_map_[context], request->table_name(), param); response->set_table_name(param.table_name_); - response->mutable_index()->set_index_type(param.index_type_); - response->mutable_index()->set_nlist(param.nlist_); + response->set_index_type(param.index_type_); + ::milvus::grpc::KeyValuePair* kv = response->add_extra_params(); + kv->set_key(EXTRA_PARAM_KEY); + kv->set_value(param.extra_params_); SET_RESPONSE(response->mutable_status(), status, context); return ::grpc::Status::OK; diff --git a/core/src/server/grpc_impl/GrpcRequestHandler.h b/core/src/server/grpc_impl/GrpcRequestHandler.h index 8ebdcc849e..a75fde16d9 100644 --- a/core/src/server/grpc_impl/GrpcRequestHandler.h +++ b/core/src/server/grpc_impl/GrpcRequestHandler.h @@ -56,6 +56,8 @@ namespace grpc { ::milvus::grpc::ErrorCode ErrorMap(ErrorCode code); +static const char* EXTRA_PARAM_KEY = "params"; + class GrpcRequestHandler final : public ::milvus::grpc::MilvusService::Service, public GrpcInterceptorHookHandler { public: explicit GrpcRequestHandler(const std::shared_ptr& tracer); diff --git a/core/src/server/web_impl/controller/WebController.hpp b/core/src/server/web_impl/controller/WebController.hpp index 7b74c86e64..ab2a26c2ca 100644 --- a/core/src/server/web_impl/controller/WebController.hpp +++ b/core/src/server/web_impl/controller/WebController.hpp @@ -208,14 +208,14 @@ class WebController : public oatpp::web::server::api::ApiController { ADD_CORS(TablesOptions) - ENDPOINT("OPTIONS", "/tables", TablesOptions) { + ENDPOINT("OPTIONS", "/collections", TablesOptions) { return createResponse(Status::CODE_204, "No Content"); } ADD_CORS(CreateTable) - ENDPOINT("POST", "/tables", CreateTable, BODY_DTO(TableRequestDto::ObjectWrapper, body)) { - TimeRecorder tr(std::string(WEB_LOG_PREFIX) + "POST \'/tables\'"); + ENDPOINT("POST", "/collections", CreateTable, BODY_DTO(TableRequestDto::ObjectWrapper, body)) { + TimeRecorder tr(std::string(WEB_LOG_PREFIX) + "POST \'/collections\'"); tr.RecordSection("Received request."); WebRequestHandler handler = WebRequestHandler(); @@ -238,8 +238,8 @@ class WebController : public oatpp::web::server::api::ApiController { ADD_CORS(ShowTables) - ENDPOINT("GET", "/tables", ShowTables, QUERIES(const QueryParams&, query_params)) { - TimeRecorder tr(std::string(WEB_LOG_PREFIX) + "GET \'/tables\'"); + ENDPOINT("GET", "/collections", ShowTables, QUERIES(const QueryParams&, query_params)) { + TimeRecorder tr(std::string(WEB_LOG_PREFIX) + "GET \'/collections\'"); tr.RecordSection("Received request."); WebRequestHandler handler = WebRequestHandler(); @@ -265,21 +265,21 @@ class WebController : public oatpp::web::server::api::ApiController { ADD_CORS(TableOptions) - ENDPOINT("OPTIONS", "/tables/{table_name}", TableOptions) { + ENDPOINT("OPTIONS", "/collections/{collection_name}", TableOptions) { return createResponse(Status::CODE_204, "No Content"); } ADD_CORS(GetTable) - ENDPOINT("GET", "/tables/{table_name}", GetTable, - PATH(String, table_name), QUERIES(const QueryParams&, query_params)) { - TimeRecorder tr(std::string(WEB_LOG_PREFIX) + "GET \'/tables/" + table_name->std_str() + "\'"); + ENDPOINT("GET", "/collections/{collection_name}", GetTable, + PATH(String, collection_name), QUERIES(const QueryParams&, query_params)) { + TimeRecorder tr(std::string(WEB_LOG_PREFIX) + "GET \'/collections/" + collection_name->std_str() + "\'"); tr.RecordSection("Received request."); WebRequestHandler handler = WebRequestHandler(); String response_str; - auto status_dto = handler.GetTable(table_name, query_params, response_str); + auto status_dto = handler.GetTable(collection_name, query_params, response_str); std::shared_ptr response; switch (status_dto->code->getValue()) { @@ -302,14 +302,14 @@ class WebController : public oatpp::web::server::api::ApiController { ADD_CORS(DropTable) - ENDPOINT("DELETE", "/tables/{table_name}", DropTable, PATH(String, table_name)) { - TimeRecorder tr(std::string(WEB_LOG_PREFIX) + "DELETE \'/tables/" + table_name->std_str() + "\'"); + ENDPOINT("DELETE", "/collections/{collection_name}", DropTable, PATH(String, collection_name)) { + TimeRecorder tr(std::string(WEB_LOG_PREFIX) + "DELETE \'/collections/" + collection_name->std_str() + "\'"); tr.RecordSection("Received request."); WebRequestHandler handler = WebRequestHandler(); std::shared_ptr response; - auto status_dto = handler.DropTable(table_name); + auto status_dto = handler.DropTable(collection_name); switch (status_dto->code->getValue()) { case StatusCode::SUCCESS: response = createDtoResponse(Status::CODE_204, status_dto); @@ -330,21 +330,21 @@ class WebController : public oatpp::web::server::api::ApiController { ADD_CORS(IndexOptions) - ENDPOINT("OPTIONS", "/tables/{table_name}/indexes", IndexOptions) { + ENDPOINT("OPTIONS", "/collections/{collection_name}/indexes", IndexOptions) { return createResponse(Status::CODE_204, "No Content"); } ADD_CORS(CreateIndex) - ENDPOINT("POST", "/tables/{table_name}/indexes", CreateIndex, - PATH(String, table_name), BODY_DTO(IndexRequestDto::ObjectWrapper, body)) { - TimeRecorder tr(std::string(WEB_LOG_PREFIX) + "POST \'/tables/" + table_name->std_str() + "/indexes\'"); + ENDPOINT("POST", "/tables/{collection_name}/indexes", CreateIndex, + PATH(String, collection_name), BODY_STRING(String, body)) { + TimeRecorder tr(std::string(WEB_LOG_PREFIX) + "POST \'/tables/" + collection_name->std_str() + "/indexes\'"); tr.RecordSection("Received request."); auto handler = WebRequestHandler(); std::shared_ptr response; - auto status_dto = handler.CreateIndex(table_name, body); + auto status_dto = handler.CreateIndex(collection_name, body); switch (status_dto->code->getValue()) { case StatusCode::SUCCESS: response = createDtoResponse(Status::CODE_201, status_dto); @@ -365,19 +365,19 @@ class WebController : public oatpp::web::server::api::ApiController { ADD_CORS(GetIndex) - ENDPOINT("GET", "/tables/{table_name}/indexes", GetIndex, PATH(String, table_name)) { - TimeRecorder tr(std::string(WEB_LOG_PREFIX) + "GET \'/tables/" + table_name->std_str() + "/indexes\'"); + ENDPOINT("GET", "/collections/{collection_name}/indexes", GetIndex, PATH(String, collection_name)) { + TimeRecorder tr(std::string(WEB_LOG_PREFIX) + "GET \'/collections/" + collection_name->std_str() + "/indexes\'"); tr.RecordSection("Received request."); - auto index_dto = IndexDto::createShared(); auto handler = WebRequestHandler(); - auto status_dto = handler.GetIndex(table_name, index_dto); + OString result; + auto status_dto = handler.GetIndex(collection_name, result); std::shared_ptr response; switch (status_dto->code->getValue()) { case StatusCode::SUCCESS: - response = createDtoResponse(Status::CODE_200, index_dto); + response = createResponse(Status::CODE_200, result); break; case StatusCode::TABLE_NOT_EXISTS: response = createDtoResponse(Status::CODE_404, status_dto); @@ -395,14 +395,14 @@ class WebController : public oatpp::web::server::api::ApiController { ADD_CORS(DropIndex) - ENDPOINT("DELETE", "/tables/{table_name}/indexes", DropIndex, PATH(String, table_name)) { - TimeRecorder tr(std::string(WEB_LOG_PREFIX) + "DELETE \'/tables/" + table_name->std_str() + "/indexes\'"); + ENDPOINT("DELETE", "/collections/{collection_name}/indexes", DropIndex, PATH(String, collection_name)) { + TimeRecorder tr(std::string(WEB_LOG_PREFIX) + "DELETE \'/collections/" + collection_name->std_str() + "/indexes\'"); tr.RecordSection("Received request."); auto handler = WebRequestHandler(); std::shared_ptr response; - auto status_dto = handler.DropIndex(table_name); + auto status_dto = handler.DropIndex(collection_name); switch (status_dto->code->getValue()) { case StatusCode::SUCCESS: response = createDtoResponse(Status::CODE_204, status_dto); @@ -423,21 +423,21 @@ class WebController : public oatpp::web::server::api::ApiController { ADD_CORS(PartitionsOptions) - ENDPOINT("OPTIONS", "/tables/{table_name}/partitions", PartitionsOptions) { + ENDPOINT("OPTIONS", "/collections/{collection_name}/partitions", PartitionsOptions) { return createResponse(Status::CODE_204, "No Content"); } ADD_CORS(CreatePartition) - ENDPOINT("POST", "/tables/{table_name}/partitions", - CreatePartition, PATH(String, table_name), BODY_DTO(PartitionRequestDto::ObjectWrapper, body)) { - TimeRecorder tr(std::string(WEB_LOG_PREFIX) + "POST \'/tables/" + table_name->std_str() + "/partitions\'"); + ENDPOINT("POST", "/collections/{collection_name}/partitions", + CreatePartition, PATH(String, collection_name), BODY_DTO(PartitionRequestDto::ObjectWrapper, body)) { + TimeRecorder tr(std::string(WEB_LOG_PREFIX) + "POST \'/collections/" + collection_name->std_str() + "/partitions\'"); tr.RecordSection("Received request."); auto handler = WebRequestHandler(); std::shared_ptr response; - auto status_dto = handler.CreatePartition(table_name, body); + auto status_dto = handler.CreatePartition(collection_name, body); switch (status_dto->code->getValue()) { case StatusCode::SUCCESS: response = createDtoResponse(Status::CODE_201, status_dto); @@ -457,9 +457,9 @@ class WebController : public oatpp::web::server::api::ApiController { ADD_CORS(ShowPartitions) - ENDPOINT("GET", "/tables/{table_name}/partitions", ShowPartitions, - PATH(String, table_name), QUERIES(const QueryParams&, query_params)) { - TimeRecorder tr(std::string(WEB_LOG_PREFIX) + "GET \'/tables/" + table_name->std_str() + "/partitions\'"); + ENDPOINT("GET", "/collections/{collection_name}/partitions", ShowPartitions, + PATH(String, collection_name), QUERIES(const QueryParams&, query_params)) { + TimeRecorder tr(std::string(WEB_LOG_PREFIX) + "GET \'/collections/" + collection_name->std_str() + "/partitions\'"); tr.RecordSection("Received request."); auto offset = query_params.get("offset"); @@ -469,7 +469,7 @@ class WebController : public oatpp::web::server::api::ApiController { auto handler = WebRequestHandler(); std::shared_ptr response; - auto status_dto = handler.ShowPartitions(table_name, query_params, partition_list_dto); + auto status_dto = handler.ShowPartitions(collection_name, query_params, partition_list_dto); switch (status_dto->code->getValue()) { case StatusCode::SUCCESS: response = createDtoResponse(Status::CODE_200, partition_list_dto); @@ -488,16 +488,16 @@ class WebController : public oatpp::web::server::api::ApiController { ADD_CORS(DropPartition) - ENDPOINT("DELETE", "/tables/{table_name}/partitions", DropPartition, - PATH(String, table_name), BODY_STRING(String, body)) { + ENDPOINT("DELETE", "/collections/{collection_name}/partitions", DropPartition, + PATH(String, collection_name), BODY_STRING(String, body)) { TimeRecorder tr(std::string(WEB_LOG_PREFIX) + - "DELETE \'/tables/" + table_name->std_str() + "/partitions\'"); + "DELETE \'/collections/" + collection_name->std_str() + "/partitions\'"); tr.RecordSection("Received request."); auto handler = WebRequestHandler(); std::shared_ptr response; - auto status_dto = handler.DropPartition(table_name, body); + auto status_dto = handler.DropPartition(collection_name, body); switch (status_dto->code->getValue()) { case StatusCode::SUCCESS: response = createDtoResponse(Status::CODE_204, status_dto); @@ -517,14 +517,14 @@ class WebController : public oatpp::web::server::api::ApiController { ADD_CORS(ShowSegments) - ENDPOINT("GET", "/tables/{table_name}/segments", ShowSegments, - PATH(String, table_name), QUERIES(const QueryParams&, query_params)) { + ENDPOINT("GET", "/collections/{collection_name}/segments", ShowSegments, + PATH(String, collection_name), QUERIES(const QueryParams&, query_params)) { auto offset = query_params.get("offset"); auto page_size = query_params.get("page_size"); auto handler = WebRequestHandler(); String response; - auto status_dto = handler.ShowSegments(table_name, query_params, response); + auto status_dto = handler.ShowSegments(collection_name, query_params, response); switch (status_dto->code->getValue()) { case StatusCode::SUCCESS: @@ -541,14 +541,14 @@ class WebController : public oatpp::web::server::api::ApiController { * * GetSegmentVector */ - ENDPOINT("GET", "/tables/{table_name}/segments/{segment_name}/{info}", GetSegmentInfo, - PATH(String, table_name), PATH(String, segment_name), PATH(String, info), QUERIES(const QueryParams&, query_params)) { + ENDPOINT("GET", "/collections/{collection_name}/segments/{segment_name}/{info}", GetSegmentInfo, + PATH(String, collection_name), PATH(String, segment_name), PATH(String, info), QUERIES(const QueryParams&, query_params)) { auto offset = query_params.get("offset"); auto page_size = query_params.get("page_size"); auto handler = WebRequestHandler(); String response; - auto status_dto = handler.GetSegmentInfo(table_name, segment_name, info, query_params, response); + auto status_dto = handler.GetSegmentInfo(collection_name, segment_name, info, query_params, response); switch (status_dto->code->getValue()) { case StatusCode::SUCCESS: @@ -562,7 +562,7 @@ class WebController : public oatpp::web::server::api::ApiController { ADD_CORS(VectorsOptions) - ENDPOINT("OPTIONS", "/tables/{table_name}/vectors", VectorsOptions) { + ENDPOINT("OPTIONS", "/collections/{collection_name}/vectors", VectorsOptions) { return createResponse(Status::CODE_204, "No Content"); } @@ -571,11 +571,11 @@ class WebController : public oatpp::web::server::api::ApiController { * * GetVectorByID ?id= */ - ENDPOINT("GET", "/tables/{table_name}/vectors", GetVectors, - PATH(String, table_name), QUERIES(const QueryParams&, query_params)) { + ENDPOINT("GET", "/collections/{collection_name}/vectors", GetVectors, + PATH(String, collection_name), QUERIES(const QueryParams&, query_params)) { auto handler = WebRequestHandler(); String response; - auto status_dto = handler.GetVector(table_name, query_params, response); + auto status_dto = handler.GetVector(collection_name, query_params, response); switch (status_dto->code->getValue()) { case StatusCode::SUCCESS: @@ -589,16 +589,16 @@ class WebController : public oatpp::web::server::api::ApiController { ADD_CORS(Insert) - ENDPOINT("POST", "/tables/{table_name}/vectors", Insert, - PATH(String, table_name), BODY_STRING(String, body)) { - TimeRecorder tr(std::string(WEB_LOG_PREFIX) + "POST \'/tables/" + table_name->std_str() + "/vectors\'"); + ENDPOINT("POST", "/collections/{collection_name}/vectors", Insert, + PATH(String, collection_name), BODY_STRING(String, body)) { + TimeRecorder tr(std::string(WEB_LOG_PREFIX) + "POST \'/collections/" + collection_name->std_str() + "/vectors\'"); tr.RecordSection("Received request."); auto ids_dto = VectorIdsDto::createShared(); WebRequestHandler handler = WebRequestHandler(); std::shared_ptr response; - auto status_dto = handler.Insert(table_name, body, ids_dto); + auto status_dto = handler.Insert(collection_name, body, ids_dto); switch (status_dto->code->getValue()) { case StatusCode::SUCCESS: response = createDtoResponse(Status::CODE_201, ids_dto); @@ -621,16 +621,16 @@ class WebController : public oatpp::web::server::api::ApiController { * Search * Delete by ID * */ - ENDPOINT("PUT", "/tables/{table_name}/vectors", VectorsOp, - PATH(String, table_name), BODY_STRING(String, body)) { - TimeRecorder tr(std::string(WEB_LOG_PREFIX) + "PUT \'/tables/" + table_name->std_str() + "/vectors\'"); + ENDPOINT("PUT", "/collections/{collection_name}/vectors", VectorsOp, + PATH(String, collection_name), BODY_STRING(String, body)) { + TimeRecorder tr(std::string(WEB_LOG_PREFIX) + "PUT \'/collections/" + collection_name->std_str() + "/vectors\'"); tr.RecordSection("Received request."); WebRequestHandler handler = WebRequestHandler(); OString result; std::shared_ptr response; - auto status_dto = handler.VectorsOp(table_name, body, result); + auto status_dto = handler.VectorsOp(collection_name, body, result); switch (status_dto->code->getValue()) { case StatusCode::SUCCESS: response = createResponse(Status::CODE_200, result); @@ -674,11 +674,6 @@ class WebController : public oatpp::web::server::api::ApiController { ADD_CORS(SystemOp) - /** - * Load - * Compact - * Flush - */ ENDPOINT("PUT", "/system/{Op}", SystemOp, PATH(String, Op), BODY_STRING(String, body_str)) { TimeRecorder tr(std::string(WEB_LOG_PREFIX) + "PUT \'/system/" + Op->std_str() + "\'"); tr.RecordSection("Received request."); diff --git a/core/src/server/web_impl/dto/IndexDto.hpp b/core/src/server/web_impl/dto/IndexDto.hpp index e4d8d9acc6..9b2176561f 100644 --- a/core/src/server/web_impl/dto/IndexDto.hpp +++ b/core/src/server/web_impl/dto/IndexDto.hpp @@ -26,7 +26,7 @@ class IndexRequestDto : public oatpp::data::mapping::type::Object { DTO_FIELD(String, index_type) = VALUE_INDEX_INDEX_TYPE_DEFAULT; - DTO_FIELD(Int64, nlist) = VALUE_INDEX_NLIST_DEFAULT; + DTO_FIELD(String, params) = VALUE_INDEX_NLIST_DEFAULT; }; using IndexDto = IndexRequestDto; diff --git a/core/src/server/web_impl/dto/TableDto.hpp b/core/src/server/web_impl/dto/TableDto.hpp index 12ff874cd1..d5e533c737 100644 --- a/core/src/server/web_impl/dto/TableDto.hpp +++ b/core/src/server/web_impl/dto/TableDto.hpp @@ -25,7 +25,7 @@ namespace web { class TableRequestDto : public oatpp::data::mapping::type::Object { DTO_INIT(TableRequestDto, Object) - DTO_FIELD(String, table_name, "table_name"); + DTO_FIELD(String, collection_name, "collection_name"); DTO_FIELD(Int64, dimension, "dimension"); DTO_FIELD(Int64, index_file_size, "index_file_size") = VALUE_TABLE_INDEX_FILE_SIZE_DEFAULT; DTO_FIELD(String, metric_type, "metric_type") = VALUE_TABLE_METRIC_TYPE_DEFAULT; @@ -34,25 +34,25 @@ class TableRequestDto : public oatpp::data::mapping::type::Object { class TableFieldsDto : public oatpp::data::mapping::type::Object { DTO_INIT(TableFieldsDto, Object) - DTO_FIELD(String, table_name); + DTO_FIELD(String, collection_name); DTO_FIELD(Int64, dimension); DTO_FIELD(Int64, index_file_size); DTO_FIELD(String, metric_type); DTO_FIELD(Int64, count); DTO_FIELD(String, index); - DTO_FIELD(Int64, nlist); + DTO_FIELD(String, index_params); }; class TableListDto : public OObject { DTO_INIT(TableListDto, Object) - DTO_FIELD(List::ObjectWrapper, table_names); + DTO_FIELD(List::ObjectWrapper, collection_names); }; class TableListFieldsDto : public OObject { DTO_INIT(TableListFieldsDto, Object) - DTO_FIELD(List::ObjectWrapper, tables); + DTO_FIELD(List::ObjectWrapper, collections); DTO_FIELD(Int64, count) = 0; }; diff --git a/core/src/server/web_impl/dto/VectorDto.hpp b/core/src/server/web_impl/dto/VectorDto.hpp index 7bb39b2e54..01b73fdc51 100644 --- a/core/src/server/web_impl/dto/VectorDto.hpp +++ b/core/src/server/web_impl/dto/VectorDto.hpp @@ -20,26 +20,6 @@ namespace web { #include OATPP_CODEGEN_BEGIN(DTO) -class SearchRequestDto : public OObject { - DTO_INIT(SearchRequestDto, Object) - - DTO_FIELD(Int64, topk); - DTO_FIELD(Int64, nprobe); - DTO_FIELD(List::ObjectWrapper, tags); - DTO_FIELD(List::ObjectWrapper, file_ids); - DTO_FIELD(List::ObjectWrapper>::ObjectWrapper, records); - DTO_FIELD(List::ObjectWrapper>::ObjectWrapper, records_bin); -}; - -class InsertRequestDto : public oatpp::data::mapping::type::Object { - DTO_INIT(InsertRequestDto, Object) - - DTO_FIELD(String, tag) = VALUE_PARTITION_TAG_DEFAULT; - DTO_FIELD(List::ObjectWrapper>::ObjectWrapper, records); - DTO_FIELD(List::ObjectWrapper>::ObjectWrapper, records_bin); - DTO_FIELD(List::ObjectWrapper, ids); -}; - class VectorIdsDto : public oatpp::data::mapping::type::Object { DTO_INIT(VectorIdsDto, Object) diff --git a/core/src/server/web_impl/handler/WebRequestHandler.cpp b/core/src/server/web_impl/handler/WebRequestHandler.cpp index 7abf2c0784..92d68e77ca 100644 --- a/core/src/server/web_impl/handler/WebRequestHandler.cpp +++ b/core/src/server/web_impl/handler/WebRequestHandler.cpp @@ -109,6 +109,26 @@ WebRequestHandler::ParseQueryStr(const OQueryParams& query_params, const std::st return Status::OK(); } +Status +WebRequestHandler::ParseQueryBool(const OQueryParams& query_params, const std::string& key, bool& value, + bool nullable) { + auto query = query_params.get(key.c_str()); + if (nullptr != query.get() && query->getSize() > 0) { + std::string value_str = query->std_str(); + if (!ValidationUtil::ValidateStringIsBool(value_str).ok()) { + return Status(ILLEGAL_QUERY_PARAM, "Query param \'all_required\' must be a bool"); + } + value = value_str == "True" || value_str == "true"; + return Status::OK(); + } + + if (!nullable) { + return Status(QUERY_PARAM_LOSS, "Query param \"" + key + "\" is required"); + } + + return Status::OK(); +} + void WebRequestHandler::AddStatusToJson(nlohmann::json& json, int64_t code, const std::string& msg) { json["code"] = (int64_t)code; @@ -142,9 +162,9 @@ WebRequestHandler::ParsePartitionStat(const milvus::server::PartitionStat& par_s } Status -WebRequestHandler::IsBinaryTable(const std::string& table_name, bool& bin) { +WebRequestHandler::IsBinaryTable(const std::string& collection_name, bool& bin) { TableSchema schema; - auto status = request_handler_.DescribeTable(context_ptr_, table_name, schema); + auto status = request_handler_.DescribeTable(context_ptr_, collection_name, schema); if (status.ok()) { auto metric = engine::MetricType(schema.metric_type_); bin = engine::MetricType::HAMMING == metric || engine::MetricType::JACCARD == metric || @@ -187,30 +207,30 @@ WebRequestHandler::CopyRecordsFromJson(const nlohmann::json& json, engine::Vecto ///////////////////////// WebRequestHandler methods /////////////////////////////////////// Status -WebRequestHandler::GetTableMetaInfo(const std::string& table_name, nlohmann::json& json_out) { +WebRequestHandler::GetTableMetaInfo(const std::string& collection_name, nlohmann::json& json_out) { TableSchema schema; - auto status = request_handler_.DescribeTable(context_ptr_, table_name, schema); + auto status = request_handler_.DescribeTable(context_ptr_, collection_name, schema); if (!status.ok()) { return status; } int64_t count; - status = request_handler_.CountTable(context_ptr_, table_name, count); + status = request_handler_.CountTable(context_ptr_, collection_name, count); if (!status.ok()) { return status; } IndexParam index_param; - status = request_handler_.DescribeIndex(context_ptr_, table_name, index_param); + status = request_handler_.DescribeIndex(context_ptr_, collection_name, index_param); if (!status.ok()) { return status; } - json_out["table_name"] = schema.table_name_; + json_out["collection_name"] = schema.table_name_; json_out["dimension"] = schema.dimension_; json_out["index_file_size"] = schema.index_file_size_; json_out["index"] = IndexMap.at(engine::EngineType(index_param.index_type_)); - json_out["nlist"] = index_param.nlist_; + json_out["index_params"] = index_param.extra_params_; json_out["metric_type"] = MetricMap.at(engine::MetricType(schema.metric_type_)); json_out["count"] = count; @@ -218,15 +238,15 @@ WebRequestHandler::GetTableMetaInfo(const std::string& table_name, nlohmann::jso } Status -WebRequestHandler::GetTableStat(const std::string& table_name, nlohmann::json& json_out) { - struct TableInfo table_info; - auto status = request_handler_.ShowTableInfo(context_ptr_, table_name, table_info); +WebRequestHandler::GetTableStat(const std::string& collection_name, nlohmann::json& json_out) { + struct TableInfo collection_info; + auto status = request_handler_.ShowTableInfo(context_ptr_, collection_name, collection_info); if (status.ok()) { - json_out["count"] = table_info.total_row_num_; + json_out["count"] = collection_info.total_row_num_; std::vector par_stat_json; - for (auto& par : table_info.partitions_stat_) { + for (auto& par : collection_info.partitions_stat_) { nlohmann::json par_json; ParsePartitionStat(par, par_json); par_stat_json.push_back(par_json); @@ -238,10 +258,10 @@ WebRequestHandler::GetTableStat(const std::string& table_name, nlohmann::json& j } Status -WebRequestHandler::GetSegmentVectors(const std::string& table_name, const std::string& segment_name, int64_t page_size, - int64_t offset, nlohmann::json& json_out) { +WebRequestHandler::GetSegmentVectors(const std::string& collection_name, const std::string& segment_name, + int64_t page_size, int64_t offset, nlohmann::json& json_out) { std::vector vector_ids; - auto status = request_handler_.GetVectorIDs(context_ptr_, table_name, segment_name, vector_ids); + auto status = request_handler_.GetVectorIDs(context_ptr_, collection_name, segment_name, vector_ids); if (!status.ok()) { return status; } @@ -251,7 +271,7 @@ WebRequestHandler::GetSegmentVectors(const std::string& table_name, const std::s auto ids = std::vector(vector_ids.begin() + ids_begin, vector_ids.begin() + ids_end); nlohmann::json vectors_json; - status = GetVectorsByIDs(table_name, ids, vectors_json); + status = GetVectorsByIDs(collection_name, ids, vectors_json); nlohmann::json result_json; if (vectors_json.empty()) { @@ -267,10 +287,10 @@ WebRequestHandler::GetSegmentVectors(const std::string& table_name, const std::s } Status -WebRequestHandler::GetSegmentIds(const std::string& table_name, const std::string& segment_name, int64_t page_size, +WebRequestHandler::GetSegmentIds(const std::string& collection_name, const std::string& segment_name, int64_t page_size, int64_t offset, nlohmann::json& json_out) { std::vector vector_ids; - auto status = request_handler_.GetVectorIDs(context_ptr_, table_name, segment_name, vector_ids); + auto status = request_handler_.GetVectorIDs(context_ptr_, collection_name, segment_name, vector_ids); if (status.ok()) { auto ids_begin = std::min(vector_ids.size(), (size_t)offset); auto ids_end = std::min(vector_ids.size(), (size_t)(offset + page_size)); @@ -310,12 +330,12 @@ WebRequestHandler::Cmd(const std::string& cmd, std::string& result_str) { Status WebRequestHandler::PreLoadTable(const nlohmann::json& json, std::string& result_str) { - if (!json.contains("table_name")) { - return Status(BODY_FIELD_LOSS, "Field \"load\" must contains table_name"); + if (!json.contains("collection_name")) { + return Status(BODY_FIELD_LOSS, "Field \"load\" must contains collection_name"); } - auto table_name = json["table_name"]; - auto status = request_handler_.PreloadTable(context_ptr_, table_name.get()); + auto collection_name = json["collection_name"]; + auto status = request_handler_.PreloadTable(context_ptr_, collection_name.get()); if (status.ok()) { nlohmann::json result; AddStatusToJson(result, status.code(), status.message()); @@ -327,17 +347,17 @@ WebRequestHandler::PreLoadTable(const nlohmann::json& json, std::string& result_ Status WebRequestHandler::Flush(const nlohmann::json& json, std::string& result_str) { - if (!json.contains("table_names")) { - return Status(BODY_FIELD_LOSS, "Field \"flush\" must contains table_names"); + if (!json.contains("collection_names")) { + return Status(BODY_FIELD_LOSS, "Field \"flush\" must contains collection_names"); } - auto table_names = json["table_names"]; - if (!table_names.is_array()) { - return Status(BODY_FIELD_LOSS, "Field \"table_names\" must be and array"); + auto collection_names = json["collection_names"]; + if (!collection_names.is_array()) { + return Status(BODY_FIELD_LOSS, "Field \"collection_names\" must be and array"); } std::vector names; - for (auto& name : table_names) { + for (auto& name : collection_names) { names.emplace_back(name.get()); } @@ -353,16 +373,16 @@ WebRequestHandler::Flush(const nlohmann::json& json, std::string& result_str) { Status WebRequestHandler::Compact(const nlohmann::json& json, std::string& result_str) { - if (!json.contains("table_name")) { - return Status(BODY_FIELD_LOSS, "Field \"compact\" must contains table_names"); + if (!json.contains("collection_name")) { + return Status(BODY_FIELD_LOSS, "Field \"compact\" must contains collection_names"); } - auto table_name = json["table_name"]; - if (!table_name.is_string()) { - return Status(BODY_FIELD_LOSS, "Field \"table_names\" must be a string"); + auto collection_name = json["collection_name"]; + if (!collection_name.is_string()) { + return Status(BODY_FIELD_LOSS, "Field \"collection_names\" must be a string"); } - auto name = table_name.get(); + auto name = collection_name.get(); auto status = request_handler_.Compact(context_ptr_, name); @@ -463,17 +483,12 @@ WebRequestHandler::SetConfig(const nlohmann::json& json, std::string& result_str } Status -WebRequestHandler::Search(const std::string& table_name, const nlohmann::json& json, std::string& result_str) { +WebRequestHandler::Search(const std::string& collection_name, const nlohmann::json& json, std::string& result_str) { if (!json.contains("topk")) { return Status(BODY_FIELD_LOSS, "Field \'topk\' is required"); } int64_t topk = json["topk"]; - if (!json.contains("nprobe")) { - return Status(BODY_FIELD_LOSS, "Field \'nprobe\' is required"); - } - int64_t nprobe = json["nprobe"]; - std::vector partition_tags; if (json.contains("partition_tags")) { auto tags = json["partition_tags"]; @@ -497,8 +512,12 @@ WebRequestHandler::Search(const std::string& table_name, const nlohmann::json& j } } + if (!json.contains("params")) { + return Status(BODY_FIELD_LOSS, "Field \'params\' is required"); + } + bool bin_flag = false; - auto status = IsBinaryTable(table_name, bin_flag); + auto status = IsBinaryTable(collection_name, bin_flag); if (!status.ok()) { return status; } @@ -514,8 +533,9 @@ WebRequestHandler::Search(const std::string& table_name, const nlohmann::json& j } TopKQueryResult result; - status = request_handler_.Search(context_ptr_, table_name, vectors_data, topk, nprobe, partition_tags, file_id_vec, - result); + status = request_handler_.Search(context_ptr_, collection_name, vectors_data, topk, json["params"], partition_tags, + file_id_vec, result); + if (!status.ok()) { return status; } @@ -547,7 +567,8 @@ WebRequestHandler::Search(const std::string& table_name, const nlohmann::json& j } Status -WebRequestHandler::DeleteByIDs(const std::string& table_name, const nlohmann::json& json, std::string& result_str) { +WebRequestHandler::DeleteByIDs(const std::string& collection_name, const nlohmann::json& json, + std::string& result_str) { std::vector vector_ids; if (!json.contains("ids")) { return Status(BODY_FIELD_LOSS, "Field \"delete\" must contains \"ids\""); @@ -565,7 +586,7 @@ WebRequestHandler::DeleteByIDs(const std::string& table_name, const nlohmann::js vector_ids.emplace_back(std::stol(id_str)); } - auto status = request_handler_.DeleteByID(context_ptr_, table_name, vector_ids); + auto status = request_handler_.DeleteByID(context_ptr_, collection_name, vector_ids); nlohmann::json result_json; AddStatusToJson(result_json, status.code(), status.message()); @@ -575,13 +596,13 @@ WebRequestHandler::DeleteByIDs(const std::string& table_name, const nlohmann::js } Status -WebRequestHandler::GetVectorsByIDs(const std::string& table_name, const std::vector& ids, +WebRequestHandler::GetVectorsByIDs(const std::string& collection_name, const std::vector& ids, nlohmann::json& json_out) { std::vector vector_batch; for (size_t i = 0; i < ids.size(); i++) { auto vec_ids = std::vector(ids.begin() + i, ids.begin() + i + 1); engine::VectorsData vectors_data; - auto status = request_handler_.GetVectorByID(context_ptr_, table_name, vec_ids, vectors_data); + auto status = request_handler_.GetVectorByID(context_ptr_, collection_name, vec_ids, vectors_data); if (!status.ok()) { return status; } @@ -589,7 +610,7 @@ WebRequestHandler::GetVectorsByIDs(const std::string& table_name, const std::vec } bool bin; - auto status = IsBinaryTable(table_name, bin); + auto status = IsBinaryTable(collection_name, bin); if (!status.ok()) { return status; } @@ -879,30 +900,31 @@ WebRequestHandler::SetGpuConfig(const GPUConfigDto::ObjectWrapper& gpu_config_dt * Table { */ StatusDto::ObjectWrapper -WebRequestHandler::CreateTable(const TableRequestDto::ObjectWrapper& table_schema) { - if (nullptr == table_schema->table_name.get()) { - RETURN_STATUS_DTO(BODY_FIELD_LOSS, "Field \'table_name\' is missing") +WebRequestHandler::CreateTable(const TableRequestDto::ObjectWrapper& collection_schema) { + if (nullptr == collection_schema->collection_name.get()) { + RETURN_STATUS_DTO(BODY_FIELD_LOSS, "Field \'collection_name\' is missing") } - if (nullptr == table_schema->dimension.get()) { + if (nullptr == collection_schema->dimension.get()) { RETURN_STATUS_DTO(BODY_FIELD_LOSS, "Field \'dimension\' is missing") } - if (nullptr == table_schema->index_file_size.get()) { + if (nullptr == collection_schema->index_file_size.get()) { RETURN_STATUS_DTO(BODY_FIELD_LOSS, "Field \'index_file_size\' is missing") } - if (nullptr == table_schema->metric_type.get()) { + if (nullptr == collection_schema->metric_type.get()) { RETURN_STATUS_DTO(BODY_FIELD_LOSS, "Field \'metric_type\' is missing") } - if (MetricNameMap.find(table_schema->metric_type->std_str()) == MetricNameMap.end()) { + if (MetricNameMap.find(collection_schema->metric_type->std_str()) == MetricNameMap.end()) { RETURN_STATUS_DTO(ILLEGAL_METRIC_TYPE, "metric_type is illegal") } - auto status = request_handler_.CreateTable( - context_ptr_, table_schema->table_name->std_str(), table_schema->dimension, table_schema->index_file_size, - static_cast(MetricNameMap.at(table_schema->metric_type->std_str()))); + auto status = + request_handler_.CreateTable(context_ptr_, collection_schema->collection_name->std_str(), + collection_schema->dimension, collection_schema->index_file_size, + static_cast(MetricNameMap.at(collection_schema->metric_type->std_str()))); ASSIGN_RETURN_STATUS_DTO(status) } @@ -926,45 +948,41 @@ WebRequestHandler::ShowTables(const OQueryParams& query_params, OString& result) } bool all_required = false; - auto required = query_params.get("all_required"); - if (nullptr != required.get()) { - auto required_str = required->std_str(); - if (!ValidationUtil::ValidateStringIsBool(required_str).ok()) { - RETURN_STATUS_DTO(ILLEGAL_QUERY_PARAM, "Query param \'all_required\' must be a bool") - } - all_required = required_str == "True" || required_str == "true"; + ParseQueryBool(query_params, "all_required", all_required); + if (!status.ok()) { + RETURN_STATUS_DTO(status.code(), status.message().c_str()); } - std::vector tables; - status = request_handler_.ShowTables(context_ptr_, tables); + std::vector collections; + status = request_handler_.ShowTables(context_ptr_, collections); if (!status.ok()) { ASSIGN_RETURN_STATUS_DTO(status) } if (all_required) { offset = 0; - page_size = tables.size(); + page_size = collections.size(); } else { - offset = std::min((size_t)offset, tables.size()); - page_size = std::min(tables.size() - offset, (size_t)page_size); + offset = std::min((size_t)offset, collections.size()); + page_size = std::min(collections.size() - offset, (size_t)page_size); } - nlohmann::json tables_json; + nlohmann::json collections_json; for (int64_t i = offset; i < page_size + offset; i++) { - nlohmann::json table_json; - status = GetTableMetaInfo(tables.at(i), table_json); + nlohmann::json collection_json; + status = GetTableMetaInfo(collections.at(i), collection_json); if (!status.ok()) { ASSIGN_RETURN_STATUS_DTO(status) } - tables_json.push_back(table_json); + collections_json.push_back(collection_json); } nlohmann::json result_json; - result_json["count"] = tables.size(); - if (tables_json.empty()) { - result_json["tables"] = std::vector(); + result_json["count"] = collections.size(); + if (collections_json.empty()) { + result_json["collections"] = std::vector(); } else { - result_json["tables"] = tables_json; + result_json["collections"] = collections_json; } result = result_json.dump().c_str(); @@ -973,9 +991,9 @@ WebRequestHandler::ShowTables(const OQueryParams& query_params, OString& result) } StatusDto::ObjectWrapper -WebRequestHandler::GetTable(const OString& table_name, const OQueryParams& query_params, OString& result) { - if (nullptr == table_name.get()) { - RETURN_STATUS_DTO(PATH_PARAM_LOSS, "Path param \'table_name\' is required!"); +WebRequestHandler::GetTable(const OString& collection_name, const OQueryParams& query_params, OString& result) { + if (nullptr == collection_name.get()) { + RETURN_STATUS_DTO(PATH_PARAM_LOSS, "Path param \'collection_name\' is required!"); } std::string stat; @@ -986,11 +1004,11 @@ WebRequestHandler::GetTable(const OString& table_name, const OQueryParams& query if (!stat.empty() && stat == "stat") { nlohmann::json json; - status = GetTableStat(table_name->std_str(), json); + status = GetTableStat(collection_name->std_str(), json); result = status.ok() ? json.dump().c_str() : "NULL"; } else { nlohmann::json json; - status = GetTableMetaInfo(table_name->std_str(), json); + status = GetTableMetaInfo(collection_name->std_str(), json); result = status.ok() ? json.dump().c_str() : "NULL"; } @@ -998,8 +1016,8 @@ WebRequestHandler::GetTable(const OString& table_name, const OQueryParams& query } StatusDto::ObjectWrapper -WebRequestHandler::DropTable(const OString& table_name) { - auto status = request_handler_.DropTable(context_ptr_, table_name->std_str()); +WebRequestHandler::DropTable(const OString& collection_name) { + auto status = request_handler_.DropTable(context_ptr_, collection_name->std_str()); ASSIGN_RETURN_STATUS_DTO(status) } @@ -1010,41 +1028,49 @@ WebRequestHandler::DropTable(const OString& table_name) { */ StatusDto::ObjectWrapper -WebRequestHandler::CreateIndex(const OString& table_name, const IndexRequestDto::ObjectWrapper& index_param) { - if (nullptr == index_param->index_type.get()) { - RETURN_STATUS_DTO(BODY_FIELD_LOSS, "Field \'index_type\' is required") - } - std::string index_type = index_param->index_type->std_str(); - if (IndexNameMap.find(index_type) == IndexNameMap.end()) { - RETURN_STATUS_DTO(ILLEGAL_INDEX_TYPE, "The index type is invalid.") +WebRequestHandler::CreateIndex(const OString& table_name, const OString& body) { + try { + auto request_json = nlohmann::json::parse(body->std_str()); + if (!request_json.contains("index_type")) { + RETURN_STATUS_DTO(BODY_FIELD_LOSS, "Field \'index_type\' is required"); + } + + std::string index_type = request_json["index_type"]; + if (IndexNameMap.find(index_type) == IndexNameMap.end()) { + RETURN_STATUS_DTO(ILLEGAL_INDEX_TYPE, "The index type is invalid.") + } + auto index = static_cast(IndexNameMap.at(index_type)); + if (!request_json.contains("params")) { + RETURN_STATUS_DTO(BODY_FIELD_LOSS, "Field \'params\' is required") + } + auto status = request_handler_.CreateIndex(context_ptr_, table_name->std_str(), index, request_json["params"]); + ASSIGN_RETURN_STATUS_DTO(status); + } catch (nlohmann::detail::parse_error& e) { + } catch (nlohmann::detail::type_error& e) { } - if (nullptr == index_param->nlist.get()) { - RETURN_STATUS_DTO(BODY_FIELD_LOSS, "Field \'nlist\' is required") - } - - auto status = - request_handler_.CreateIndex(context_ptr_, table_name->std_str(), - static_cast(IndexNameMap.at(index_type)), index_param->nlist->getValue()); - ASSIGN_RETURN_STATUS_DTO(status) + ASSIGN_RETURN_STATUS_DTO(Status::OK()) } StatusDto::ObjectWrapper -WebRequestHandler::GetIndex(const OString& table_name, IndexDto::ObjectWrapper& index_dto) { +WebRequestHandler::GetIndex(const OString& collection_name, OString& result) { IndexParam param; - auto status = request_handler_.DescribeIndex(context_ptr_, table_name->std_str(), param); + auto status = request_handler_.DescribeIndex(context_ptr_, collection_name->std_str(), param); if (status.ok()) { - index_dto->index_type = IndexMap.at(engine::EngineType(param.index_type_)).c_str(); - index_dto->nlist = param.nlist_; + nlohmann::json json_out; + auto index_type = IndexMap.at(engine::EngineType(param.index_type_)); + json_out["index_type"] = index_type; + json_out["params"] = nlohmann::json::parse(param.extra_params_); + result = json_out.dump().c_str(); } ASSIGN_RETURN_STATUS_DTO(status) } StatusDto::ObjectWrapper -WebRequestHandler::DropIndex(const OString& table_name) { - auto status = request_handler_.DropIndex(context_ptr_, table_name->std_str()); +WebRequestHandler::DropIndex(const OString& collection_name) { + auto status = request_handler_.DropIndex(context_ptr_, collection_name->std_str()); ASSIGN_RETURN_STATUS_DTO(status) } @@ -1054,19 +1080,19 @@ WebRequestHandler::DropIndex(const OString& table_name) { * Partition { */ StatusDto::ObjectWrapper -WebRequestHandler::CreatePartition(const OString& table_name, const PartitionRequestDto::ObjectWrapper& param) { +WebRequestHandler::CreatePartition(const OString& collection_name, const PartitionRequestDto::ObjectWrapper& param) { if (nullptr == param->partition_tag.get()) { RETURN_STATUS_DTO(BODY_FIELD_LOSS, "Field \'partition_tag\' is required") } auto status = - request_handler_.CreatePartition(context_ptr_, table_name->std_str(), param->partition_tag->std_str()); + request_handler_.CreatePartition(context_ptr_, collection_name->std_str(), param->partition_tag->std_str()); ASSIGN_RETURN_STATUS_DTO(status) } StatusDto::ObjectWrapper -WebRequestHandler::ShowPartitions(const OString& table_name, const OQueryParams& query_params, +WebRequestHandler::ShowPartitions(const OString& collection_name, const OQueryParams& query_params, PartitionListDto::ObjectWrapper& partition_list_dto) { int64_t offset = 0; auto status = ParseQueryInteger(query_params, "offset", offset); @@ -1096,7 +1122,7 @@ WebRequestHandler::ShowPartitions(const OString& table_name, const OQueryParams& } std::vector partitions; - status = request_handler_.ShowPartitions(context_ptr_, table_name->std_str(), partitions); + status = request_handler_.ShowPartitions(context_ptr_, collection_name->std_str(), partitions); if (!status.ok()) { ASSIGN_RETURN_STATUS_DTO(status) } @@ -1124,7 +1150,7 @@ WebRequestHandler::ShowPartitions(const OString& table_name, const OQueryParams& } StatusDto::ObjectWrapper -WebRequestHandler::DropPartition(const OString& table_name, const OString& body) { +WebRequestHandler::DropPartition(const OString& collection_name, const OString& body) { std::string tag; try { auto json = nlohmann::json::parse(body->std_str()); @@ -1134,7 +1160,7 @@ WebRequestHandler::DropPartition(const OString& table_name, const OString& body) } catch (nlohmann::detail::type_error& e) { RETURN_STATUS_DTO(BODY_PARSE_FAIL, e.what()) } - auto status = request_handler_.DropPartition(context_ptr_, table_name->std_str(), tag); + auto status = request_handler_.DropPartition(context_ptr_, collection_name->std_str(), tag); ASSIGN_RETURN_STATUS_DTO(status) } @@ -1144,7 +1170,7 @@ WebRequestHandler::DropPartition(const OString& table_name, const OString& body) * Segment { */ StatusDto::ObjectWrapper -WebRequestHandler::ShowSegments(const OString& table_name, const OQueryParams& query_params, OString& response) { +WebRequestHandler::ShowSegments(const OString& collection_name, const OQueryParams& query_params, OString& response) { int64_t offset = 0; auto status = ParseQueryInteger(query_params, "offset", offset); if (!status.ok()) { @@ -1177,7 +1203,7 @@ WebRequestHandler::ShowSegments(const OString& table_name, const OQueryParams& q } TableInfo info; - status = request_handler_.ShowTableInfo(context_ptr_, table_name->std_str(), info); + status = request_handler_.ShowTableInfo(context_ptr_, collection_name->std_str(), info); if (!status.ok()) { ASSIGN_RETURN_STATUS_DTO(status) } @@ -1228,7 +1254,7 @@ WebRequestHandler::ShowSegments(const OString& table_name, const OQueryParams& q } StatusDto::ObjectWrapper -WebRequestHandler::GetSegmentInfo(const OString& table_name, const OString& segment_name, const OString& info, +WebRequestHandler::GetSegmentInfo(const OString& collection_name, const OString& segment_name, const OString& info, const OQueryParams& query_params, OString& result) { int64_t offset = 0; auto status = ParseQueryInteger(query_params, "offset", offset); @@ -1252,10 +1278,10 @@ WebRequestHandler::GetSegmentInfo(const OString& table_name, const OString& segm nlohmann::json json; // Get vectors if (re == "vectors") { - status = GetSegmentVectors(table_name->std_str(), segment_name->std_str(), page_size, offset, json); + status = GetSegmentVectors(collection_name->std_str(), segment_name->std_str(), page_size, offset, json); // Get vector ids } else if (re == "ids") { - status = GetSegmentIds(table_name->std_str(), segment_name->std_str(), page_size, offset, json); + status = GetSegmentIds(collection_name->std_str(), segment_name->std_str(), page_size, offset, json); } result = status.ok() ? json.dump().c_str() : "NULL"; @@ -1268,14 +1294,14 @@ WebRequestHandler::GetSegmentInfo(const OString& table_name, const OString& segm * Vector { */ StatusDto::ObjectWrapper -WebRequestHandler::Insert(const OString& table_name, const OString& body, VectorIdsDto::ObjectWrapper& ids_dto) { +WebRequestHandler::Insert(const OString& collection_name, const OString& body, VectorIdsDto::ObjectWrapper& ids_dto) { if (nullptr == body.get() || body->getSize() == 0) { RETURN_STATUS_DTO(BODY_FIELD_LOSS, "Request payload is required.") } // step 1: copy vectors bool bin_flag; - auto status = IsBinaryTable(table_name->std_str(), bin_flag); + auto status = IsBinaryTable(collection_name->std_str(), bin_flag); if (!status.ok()) { ASSIGN_RETURN_STATUS_DTO(status) } @@ -1310,7 +1336,7 @@ WebRequestHandler::Insert(const OString& table_name, const OString& body, Vector } // step 4: construct result - status = request_handler_.Insert(context_ptr_, table_name->std_str(), vectors, tag); + status = request_handler_.Insert(context_ptr_, collection_name->std_str(), vectors, tag); if (status.ok()) { ids_dto->ids = ids_dto->ids->createShared(); for (auto& id : vectors.id_array_) { @@ -1322,7 +1348,7 @@ WebRequestHandler::Insert(const OString& table_name, const OString& body, Vector } StatusDto::ObjectWrapper -WebRequestHandler::GetVector(const OString& table_name, const OQueryParams& query_params, OString& response) { +WebRequestHandler::GetVector(const OString& collection_name, const OQueryParams& query_params, OString& response) { int64_t id = 0; auto status = ParseQueryInteger(query_params, "id", id, false); if (!status.ok()) { @@ -1332,7 +1358,7 @@ WebRequestHandler::GetVector(const OString& table_name, const OQueryParams& quer std::vector ids = {id}; engine::VectorsData vectors; nlohmann::json vectors_json; - status = GetVectorsByIDs(table_name->std_str(), ids, vectors_json); + status = GetVectorsByIDs(collection_name->std_str(), ids, vectors_json); if (!status.ok()) { response = "NULL"; ASSIGN_RETURN_STATUS_DTO(status) @@ -1352,7 +1378,7 @@ WebRequestHandler::GetVector(const OString& table_name, const OQueryParams& quer } StatusDto::ObjectWrapper -WebRequestHandler::VectorsOp(const OString& table_name, const OString& payload, OString& response) { +WebRequestHandler::VectorsOp(const OString& collection_name, const OString& payload, OString& response) { auto status = Status::OK(); std::string result_str; @@ -1360,9 +1386,9 @@ WebRequestHandler::VectorsOp(const OString& table_name, const OString& payload, nlohmann::json payload_json = nlohmann::json::parse(payload->std_str()); if (payload_json.contains("delete")) { - status = DeleteByIDs(table_name->std_str(), payload_json["delete"], result_str); + status = DeleteByIDs(collection_name->std_str(), payload_json["delete"], result_str); } else if (payload_json.contains("search")) { - status = Search(table_name->std_str(), payload_json["search"], result_str); + status = Search(collection_name->std_str(), payload_json["search"], result_str); } else { status = Status(ILLEGAL_BODY, "Unknown body"); } diff --git a/core/src/server/web_impl/handler/WebRequestHandler.h b/core/src/server/web_impl/handler/WebRequestHandler.h index 7444caa4dc..d3e47affb9 100644 --- a/core/src/server/web_impl/handler/WebRequestHandler.h +++ b/core/src/server/web_impl/handler/WebRequestHandler.h @@ -84,6 +84,9 @@ class WebRequestHandler { Status ParseQueryStr(const OQueryParams& query_params, const std::string& key, std::string& value, bool nullable = true); + Status + ParseQueryBool(const OQueryParams& query_params, const std::string& key, bool& value, bool nullable = true); + private: void AddStatusToJson(nlohmann::json& json, int64_t code, const std::string& msg); @@ -189,10 +192,10 @@ class WebRequestHandler { * Index */ StatusDto::ObjectWrapper - CreateIndex(const OString& table_name, const IndexRequestDto::ObjectWrapper& index_param); + CreateIndex(const OString& table_name, const OString& body); StatusDto::ObjectWrapper - GetIndex(const OString& table_name, IndexDto::ObjectWrapper& index_dto); + GetIndex(const OString& table_name, OString& result); StatusDto::ObjectWrapper DropIndex(const OString& table_name); diff --git a/core/src/utils/StringHelpFunctions.cpp b/core/src/utils/StringHelpFunctions.cpp index 05f7679363..cd2a74712b 100644 --- a/core/src/utils/StringHelpFunctions.cpp +++ b/core/src/utils/StringHelpFunctions.cpp @@ -12,9 +12,12 @@ #include "utils/StringHelpFunctions.h" #include +#include #include #include +#include "utils/ValidationUtil.h" + namespace milvus { namespace server { @@ -148,11 +151,21 @@ StringHelpFunctions::IsRegexMatch(const std::string& target_str, const std::stri // regex match std::regex pattern(pattern_str); std::smatch results; - if (std::regex_match(target_str, results, pattern)) { - return true; - } else { - return false; + return std::regex_match(target_str, results, pattern); +} + +Status +StringHelpFunctions::ConvertToBoolean(const std::string& str, bool& value) { + auto status = ValidationUtil::ValidateStringIsBool(str); + if (!status.ok()) { + return status; } + + std::string s = str; + std::transform(s.begin(), s.end(), s.begin(), ::tolower); + value = s == "true" || s == "on" || s == "yes" || s == "1"; + + return Status::OK(); } } // namespace server diff --git a/core/src/utils/StringHelpFunctions.h b/core/src/utils/StringHelpFunctions.h index 85dc5b9555..0f1226a8c2 100644 --- a/core/src/utils/StringHelpFunctions.h +++ b/core/src/utils/StringHelpFunctions.h @@ -64,6 +64,12 @@ class StringHelpFunctions { // regex grammar reference: http://www.cplusplus.com/reference/regex/ECMAScript/ static bool IsRegexMatch(const std::string& target_str, const std::string& pattern); + + // conversion rules refer to ValidationUtil::ValidateStringIsBool() + // "true", "on", "yes", "1" ==> true + // "false", "off", "no", "0", "" ==> false + static Status + ConvertToBoolean(const std::string& str, bool& value); }; } // namespace server diff --git a/core/src/utils/ValidationUtil.cpp b/core/src/utils/ValidationUtil.cpp index ce93c41171..84f260c508 100644 --- a/core/src/utils/ValidationUtil.cpp +++ b/core/src/utils/ValidationUtil.cpp @@ -12,6 +12,7 @@ #include "utils/ValidationUtil.h" #include "Log.h" #include "db/engine/ExecutionEngine.h" +#include "index/knowhere/knowhere/index/vector_index/helpers/IndexParameter.h" #include "utils/StringHelpFunctions.h" #include @@ -32,10 +33,62 @@ namespace milvus { namespace server { +namespace { constexpr size_t TABLE_NAME_SIZE_LIMIT = 255; constexpr int64_t TABLE_DIMENSION_LIMIT = 32768; constexpr int32_t INDEX_FILE_SIZE_LIMIT = 4096; // index trigger size max = 4096 MB +Status +CheckParameterRange(const milvus::json& json_params, const std::string& param_name, int64_t min, int64_t max, + bool min_close = true, bool max_closed = true) { + if (json_params.find(param_name) == json_params.end()) { + std::string msg = "Parameter list must contain: "; + return Status(SERVER_INVALID_ARGUMENT, msg + param_name); + } + + try { + int64_t value = json_params[param_name]; + bool min_err = min_close ? value < min : value <= min; + bool max_err = max_closed ? value > max : value >= max; + if (min_err || max_err) { + std::string msg = "Invalid " + param_name + " value: " + std::to_string(value) + ". Valid range is " + + (min_close ? "[" : "(") + std::to_string(min) + ", " + std::to_string(max) + + (max_closed ? "]" : ")"); + SERVER_LOG_ERROR << msg; + return Status(SERVER_INVALID_ARGUMENT, msg); + } + } catch (std::exception& e) { + std::string msg = "Invalid " + param_name + ": "; + return Status(SERVER_INVALID_ARGUMENT, msg + e.what()); + } + + return Status::OK(); +} + +Status +CheckParameterExistence(const milvus::json& json_params, const std::string& param_name) { + if (json_params.find(param_name) == json_params.end()) { + std::string msg = "Parameter list must contain: "; + return Status(SERVER_INVALID_ARGUMENT, msg + param_name); + } + + try { + int64_t value = json_params[param_name]; + if (value < 0) { + std::string msg = "Invalid " + param_name + " value: " + std::to_string(value); + SERVER_LOG_ERROR << msg; + return Status(SERVER_INVALID_ARGUMENT, msg); + } + } catch (std::exception& e) { + std::string msg = "Invalid " + param_name + ": "; + return Status(SERVER_INVALID_ARGUMENT, msg + e.what()); + } + + return Status::OK(); +} + +} // namespace + Status ValidationUtil::ValidateTableName(const std::string& table_name) { // Table name shouldn't be empty. @@ -109,24 +162,114 @@ ValidationUtil::ValidateTableIndexType(int32_t index_type) { return Status::OK(); } +Status +ValidationUtil::ValidateIndexParams(const milvus::json& index_params, const engine::meta::TableSchema& table_schema, + int32_t index_type) { + switch (index_type) { + case (int32_t)engine::EngineType::FAISS_IDMAP: + case (int32_t)engine::EngineType::FAISS_BIN_IDMAP: { + break; + } + case (int32_t)engine::EngineType::FAISS_IVFFLAT: + case (int32_t)engine::EngineType::FAISS_IVFSQ8: + case (int32_t)engine::EngineType::FAISS_IVFSQ8H: + case (int32_t)engine::EngineType::FAISS_BIN_IVFFLAT: { + auto status = CheckParameterRange(index_params, knowhere::IndexParams::nlist, 0, 999999, false); + if (!status.ok()) { + return status; + } + break; + } + case (int32_t)engine::EngineType::FAISS_PQ: { + auto status = CheckParameterRange(index_params, knowhere::IndexParams::nlist, 0, 999999, false); + if (!status.ok()) { + return status; + } + + status = CheckParameterExistence(index_params, knowhere::IndexParams::m); + if (!status.ok()) { + return status; + } + + break; + } + case (int32_t)engine::EngineType::NSG_MIX: { + auto status = CheckParameterRange(index_params, knowhere::IndexParams::search_length, 10, 300); + if (!status.ok()) { + return status; + } + status = CheckParameterRange(index_params, knowhere::IndexParams::out_degree, 5, 300); + if (!status.ok()) { + return status; + } + status = CheckParameterRange(index_params, knowhere::IndexParams::candidate, 50, 1000); + if (!status.ok()) { + return status; + } + status = CheckParameterRange(index_params, knowhere::IndexParams::knng, 5, 300); + if (!status.ok()) { + return status; + } + break; + } + case (int32_t)engine::EngineType::HNSW: { + auto status = CheckParameterRange(index_params, knowhere::IndexParams::M, 5, 48); + if (!status.ok()) { + return status; + } + status = CheckParameterRange(index_params, knowhere::IndexParams::efConstruction, 100, 500); + if (!status.ok()) { + return status; + } + break; + } + } + return Status::OK(); +} + +Status +ValidationUtil::ValidateSearchParams(const milvus::json& search_params, const engine::meta::TableSchema& table_schema, + int64_t topk) { + switch (table_schema.engine_type_) { + case (int32_t)engine::EngineType::FAISS_IDMAP: + case (int32_t)engine::EngineType::FAISS_BIN_IDMAP: { + break; + } + case (int32_t)engine::EngineType::FAISS_IVFFLAT: + case (int32_t)engine::EngineType::FAISS_IVFSQ8: + case (int32_t)engine::EngineType::FAISS_IVFSQ8H: + case (int32_t)engine::EngineType::FAISS_BIN_IVFFLAT: + case (int32_t)engine::EngineType::FAISS_PQ: { + auto status = CheckParameterRange(search_params, knowhere::IndexParams::nprobe, 1, 999999); + if (!status.ok()) { + return status; + } + break; + } + case (int32_t)engine::EngineType::NSG_MIX: { + auto status = CheckParameterRange(search_params, knowhere::IndexParams::search_length, 10, 300); + if (!status.ok()) { + return status; + } + break; + } + case (int32_t)engine::EngineType::HNSW: { + auto status = CheckParameterRange(search_params, knowhere::IndexParams::ef, topk, 1000); + if (!status.ok()) { + return status; + } + break; + } + } + return Status::OK(); +} + bool ValidationUtil::IsBinaryIndexType(int32_t index_type) { return (index_type == static_cast(engine::EngineType::FAISS_BIN_IDMAP)) || (index_type == static_cast(engine::EngineType::FAISS_BIN_IVFFLAT)); } -Status -ValidationUtil::ValidateTableIndexNlist(int32_t nlist) { - if (nlist <= 0) { - std::string msg = - "Invalid index nlist: " + std::to_string(nlist) + ". " + "The index nlist must be greater than 0."; - SERVER_LOG_ERROR << msg; - return Status(SERVER_INVALID_INDEX_NLIST, msg); - } - - return Status::OK(); -} - Status ValidationUtil::ValidateTableIndexFileSize(int64_t index_file_size) { if (index_file_size <= 0 || index_file_size > INDEX_FILE_SIZE_LIMIT) { @@ -170,18 +313,6 @@ ValidationUtil::ValidateSearchTopk(int64_t top_k, const engine::meta::TableSchem return Status::OK(); } -Status -ValidationUtil::ValidateSearchNprobe(int64_t nprobe, const engine::meta::TableSchema& table_schema) { - if (nprobe <= 0 || nprobe > table_schema.nlist_) { - std::string msg = "Invalid nprobe: " + std::to_string(nprobe) + ". " + - "The nprobe must be within the range of 1 ~ index nlist."; - SERVER_LOG_ERROR << msg; - return Status(SERVER_INVALID_NPROBE, msg); - } - - return Status::OK(); -} - Status ValidationUtil::ValidatePartitionName(const std::string& partition_name) { if (partition_name.empty()) { diff --git a/core/src/utils/ValidationUtil.h b/core/src/utils/ValidationUtil.h index 8b3537cc1b..4fa2df1619 100644 --- a/core/src/utils/ValidationUtil.h +++ b/core/src/utils/ValidationUtil.h @@ -12,6 +12,7 @@ #pragma once #include "db/meta/MetaTypes.h" +#include "utils/Json.h" #include "utils/Status.h" #include @@ -34,11 +35,16 @@ class ValidationUtil { static Status ValidateTableIndexType(int32_t index_type); - static bool - IsBinaryIndexType(int32_t index_type); + static Status + ValidateIndexParams(const milvus::json& index_params, const engine::meta::TableSchema& table_schema, + int32_t index_type); static Status - ValidateTableIndexNlist(int32_t nlist); + ValidateSearchParams(const milvus::json& search_params, const engine::meta::TableSchema& table_schema, + int64_t topk); + + static bool + IsBinaryIndexType(int32_t index_type); static Status ValidateTableIndexFileSize(int64_t index_file_size); @@ -52,9 +58,6 @@ class ValidationUtil { static Status ValidateSearchTopk(int64_t top_k, const engine::meta::TableSchema& table_schema); - static Status - ValidateSearchNprobe(int64_t nprobe, const engine::meta::TableSchema& table_schema); - static Status ValidatePartitionName(const std::string& partition_name); diff --git a/core/src/wrapper/BinVecImpl.cpp b/core/src/wrapper/BinVecImpl.cpp index 8d29e1e7a6..ee219bfb4f 100644 --- a/core/src/wrapper/BinVecImpl.cpp +++ b/core/src/wrapper/BinVecImpl.cpp @@ -25,7 +25,7 @@ Status BinVecImpl::BuildAll(const int64_t& nb, const uint8_t* xb, const int64_t* ids, const Config& cfg, const int64_t& nt, const uint8_t* xt) { try { - dim = cfg->d; + dim = cfg[knowhere::meta::DIM]; auto ret_ds = std::make_shared(); ret_ds->Set(knowhere::meta::ROWS, nb); @@ -47,15 +47,13 @@ BinVecImpl::BuildAll(const int64_t& nb, const uint8_t* xb, const int64_t* ids, c Status BinVecImpl::Search(const int64_t& nq, const uint8_t* xq, float* dist, int64_t* ids, const Config& cfg) { try { - auto k = cfg->k; + int64_t k = cfg[knowhere::meta::TOPK]; auto ret_ds = std::make_shared(); ret_ds->Set(knowhere::meta::ROWS, nq); ret_ds->Set(knowhere::meta::DIM, dim); ret_ds->Set(knowhere::meta::TENSOR, xq); - Config search_cfg = cfg; - - auto res = index_->Search(ret_ds, search_cfg); + auto res = index_->Search(ret_ds, cfg); //{ // auto& ids = ids_array; // auto& dists = dis_array; @@ -150,9 +148,7 @@ BinVecImpl::GetVectorById(const int64_t n, const int64_t* ids, uint8_t* x, const ret_ds->Set(knowhere::meta::DIM, dim); ret_ds->Set(knowhere::meta::IDS, ids); - Config search_cfg = cfg; - - auto res = index_->GetVectorById(ret_ds, search_cfg); + auto res = index_->GetVectorById(ret_ds, cfg); // TODO(linxj): avoid copy here. auto res_x = res->Get(knowhere::meta::TENSOR); @@ -176,15 +172,13 @@ BinVecImpl::SearchById(const int64_t& nq, const int64_t* xq, float* dist, int64_ throw WrapperException("not support"); } try { - auto k = cfg->k; + int64_t k = cfg[knowhere::meta::TOPK]; auto ret_ds = std::make_shared(); ret_ds->Set(knowhere::meta::ROWS, nq); ret_ds->Set(knowhere::meta::DIM, dim); ret_ds->Set(knowhere::meta::IDS, xq); - Config search_cfg = cfg; - - auto res = index_->SearchById(ret_ds, search_cfg); + auto res = index_->SearchById(ret_ds, cfg); //{ // auto& ids = ids_array; // auto& dists = dis_array; @@ -235,7 +229,7 @@ BinVecImpl::GetBlacklist(faiss::ConcurrentBitsetPtr& list) { ErrorCode BinBFIndex::Build(const Config& cfg) { try { - dim = cfg->d; + dim = cfg[knowhere::meta::DIM]; std::static_pointer_cast(index_)->Train(cfg); } catch (knowhere::KnowhereException& e) { WRAPPER_LOG_ERROR << e.what(); @@ -251,7 +245,7 @@ Status BinBFIndex::BuildAll(const int64_t& nb, const uint8_t* xb, const int64_t* ids, const Config& cfg, const int64_t& nt, const uint8_t* xt) { try { - dim = cfg->d; + dim = cfg[knowhere::meta::DIM]; auto ret_ds = std::make_shared(); ret_ds->Set(knowhere::meta::ROWS, nb); ret_ds->Set(knowhere::meta::DIM, dim); diff --git a/core/src/wrapper/ConfAdapter.cpp b/core/src/wrapper/ConfAdapter.cpp index 4d184f566b..7742f3bab7 100644 --- a/core/src/wrapper/ConfAdapter.cpp +++ b/core/src/wrapper/ConfAdapter.cpp @@ -8,12 +8,13 @@ // Unless required by applicable law or agreed to in writing, software distributed under the License // is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express // or implied. See the License for the specific language governing permissions and limitations under the License. - #include "wrapper/ConfAdapter.h" #include + #include #include +#include #include #include "WrapperException.h" @@ -21,8 +22,6 @@ #include "server/Config.h" #include "utils/Log.h" -// TODO(lxj): add conf checker - namespace milvus { namespace engine { @@ -32,45 +31,64 @@ namespace engine { #define GPU_MAX_NRPOBE 1024 #endif -void -ConfAdapter::MatchBase(knowhere::Config conf, knowhere::METRICTYPE default_metric) { - if (conf->metric_type == knowhere::DEFAULT_TYPE) - conf->metric_type = default_metric; +#define DEFAULT_MAX_DIM 16384 +#define DEFAULT_MIN_DIM 1 +#define DEFAULT_MAX_K 16384 +#define DEFAULT_MIN_K 1 +#define DEFAULT_MIN_ROWS 1 // minimum size for build index +#define DEFAULT_MAX_ROWS 50000000 + +#define CheckIntByRange(key, min, max) \ + if (!oricfg.contains(key) || !oricfg[key].is_number_integer() || oricfg[key].get() > max || \ + oricfg[key].get() < min) { \ + return false; \ + } + +// #define checkfloat(key, min, max) \ +// if (!oricfg.contains(key) || !oricfg[key].is_number_float() || oricfg[key] >= max || oricfg[key] <= min) { \ +// return false; \ +// } + +#define CheckIntByValues(key, container) \ + if (!oricfg.contains(key) || !oricfg[key].is_number_integer()) { \ + return false; \ + } else { \ + auto finder = std::find(std::begin(container), std::end(container), oricfg[key].get()); \ + if (finder == std::end(container)) { \ + return false; \ + } \ + } + +#define CheckStrByValues(key, container) \ + if (!oricfg.contains(key) || !oricfg[key].is_string()) { \ + return false; \ + } else { \ + auto finder = std::find(std::begin(container), std::end(container), oricfg[key].get()); \ + if (finder == std::end(container)) { \ + return false; \ + } \ + } + +bool +ConfAdapter::CheckTrain(milvus::json& oricfg) { + static std::vector METRICS{knowhere::Metric::L2, knowhere::Metric::IP}; + + CheckIntByRange(knowhere::meta::DIM, DEFAULT_MIN_DIM, DEFAULT_MAX_DIM); + CheckStrByValues(knowhere::Metric::TYPE, METRICS); + + return true; } -knowhere::Config -ConfAdapter::Match(const TempMetaConf& metaconf) { - auto conf = std::make_shared(); - conf->d = metaconf.dim; - conf->metric_type = metaconf.metric_type; - conf->gpu_id = metaconf.gpu_id; - conf->k = metaconf.k; - MatchBase(conf); - return conf; -} +bool +ConfAdapter::CheckSearch(milvus::json& oricfg, const IndexType& type) { + CheckIntByRange(knowhere::meta::TOPK, DEFAULT_MIN_K - 1, DEFAULT_MAX_K); -knowhere::Config -ConfAdapter::MatchSearch(const TempMetaConf& metaconf, const IndexType& type) { - auto conf = std::make_shared(); - conf->k = metaconf.k; - return conf; + return true; } -knowhere::Config -IVFConfAdapter::Match(const TempMetaConf& metaconf) { - auto conf = std::make_shared(); - conf->nlist = MatchNlist(metaconf.size, metaconf.nlist, 16384); - conf->d = metaconf.dim; - conf->metric_type = metaconf.metric_type; - conf->gpu_id = metaconf.gpu_id; - MatchBase(conf); - return conf; -} - -static constexpr float TYPICAL_COUNT = 1000000.0; - int64_t -IVFConfAdapter::MatchNlist(const int64_t& size, const int64_t& nlist, const int64_t& per_nlist) { +MatchNlist(const int64_t& size, const int64_t& nlist, const int64_t& per_nlist) { + static float TYPICAL_COUNT = 1000000.0; if (size <= TYPICAL_COUNT / per_nlist + 1) { // handle less row count, avoid nlist set to 0 return 1; @@ -81,62 +99,88 @@ IVFConfAdapter::MatchNlist(const int64_t& size, const int64_t& nlist, const int6 return nlist; } -knowhere::Config -IVFConfAdapter::MatchSearch(const TempMetaConf& metaconf, const IndexType& type) { - auto conf = std::make_shared(); - conf->k = metaconf.k; +bool +IVFConfAdapter::CheckTrain(milvus::json& oricfg) { + static int64_t MAX_NLIST = 999999; + static int64_t MIN_NLIST = 1; - if (metaconf.nprobe <= 0) - conf->nprobe = 16; // hardcode here - else - conf->nprobe = metaconf.nprobe; + CheckIntByRange(knowhere::IndexParams::nlist, MIN_NLIST, MAX_NLIST); + CheckIntByRange(knowhere::meta::ROWS, DEFAULT_MIN_ROWS, DEFAULT_MAX_ROWS); - switch (type) { - case IndexType::FAISS_IVFFLAT_GPU: - case IndexType::FAISS_IVFSQ8_GPU: - case IndexType::FAISS_IVFPQ_GPU: - if (conf->nprobe > GPU_MAX_NRPOBE) { - WRAPPER_LOG_WARNING << "When search with GPU, nprobe shoud be no more than " << GPU_MAX_NRPOBE - << ", but you passed " << conf->nprobe << ". Search with " << GPU_MAX_NRPOBE - << " instead"; - conf->nprobe = GPU_MAX_NRPOBE; - } + // int64_t nlist = oricfg[knowhere::IndexParams::nlist]; + // CheckIntByRange(knowhere::meta::ROWS, nlist, DEFAULT_MAX_ROWS); + + // auto tune params + oricfg[knowhere::IndexParams::nlist] = MatchNlist(oricfg[knowhere::meta::ROWS].get(), + oricfg[knowhere::IndexParams::nlist].get(), 16384); + + // Best Practice + // static int64_t MIN_POINTS_PER_CENTROID = 40; + // static int64_t MAX_POINTS_PER_CENTROID = 256; + // CheckIntByRange(knowhere::meta::ROWS, MIN_POINTS_PER_CENTROID * nlist, MAX_POINTS_PER_CENTROID * nlist); + + return ConfAdapter::CheckTrain(oricfg); +} + +bool +IVFConfAdapter::CheckSearch(milvus::json& oricfg, const IndexType& type) { + static int64_t MIN_NPROBE = 1; + static int64_t MAX_NPROBE = 999999; // todo(linxj): [1, nlist] + + if (type == IndexType::FAISS_IVFPQ_GPU || type == IndexType::FAISS_IVFSQ8_GPU || + type == IndexType::FAISS_IVFSQ8_HYBRID || type == IndexType::FAISS_IVFFLAT_GPU) { + CheckIntByRange(knowhere::IndexParams::nprobe, MIN_NPROBE, GPU_MAX_NRPOBE); + } else { + CheckIntByRange(knowhere::IndexParams::nprobe, MIN_NPROBE, MAX_NPROBE); } - return conf; + + return ConfAdapter::CheckSearch(oricfg, type); } -knowhere::Config -IVFSQConfAdapter::Match(const TempMetaConf& metaconf) { - auto conf = std::make_shared(); - conf->nlist = MatchNlist(metaconf.size, metaconf.nlist, 16384); - conf->d = metaconf.dim; - conf->metric_type = metaconf.metric_type; - conf->gpu_id = metaconf.gpu_id; - conf->nbits = 8; - MatchBase(conf); - return conf; +bool +IVFSQConfAdapter::CheckTrain(milvus::json& oricfg) { + static int64_t DEFAULT_NBITS = 8; + oricfg[knowhere::IndexParams::nbits] = DEFAULT_NBITS; + + return IVFConfAdapter::CheckTrain(oricfg); } -knowhere::Config -IVFPQConfAdapter::Match(const TempMetaConf& metaconf) { - auto conf = std::make_shared(); - conf->nlist = MatchNlist(metaconf.size, metaconf.nlist); - conf->d = metaconf.dim; - conf->metric_type = metaconf.metric_type; - conf->gpu_id = metaconf.gpu_id; - conf->nbits = 8; - MatchBase(conf); +bool +IVFPQConfAdapter::CheckTrain(milvus::json& oricfg) { + static int64_t DEFAULT_NBITS = 8; + static int64_t MAX_NLIST = 999999; + static int64_t MIN_NLIST = 1; + static std::vector CPU_METRICS{knowhere::Metric::L2, knowhere::Metric::IP}; + static std::vector GPU_METRICS{knowhere::Metric::L2}; + + oricfg[knowhere::IndexParams::nbits] = DEFAULT_NBITS; #ifdef MILVUS_GPU_VERSION Status s; bool enable_gpu = false; server::Config& config = server::Config::GetInstance(); s = config.GetGpuResourceConfigEnable(enable_gpu); - if (s.ok() && conf->metric_type == knowhere::METRICTYPE::IP) { - WRAPPER_LOG_ERROR << "PQ not support IP in GPU version!"; - throw WrapperException("PQ not support IP in GPU version!"); + if (s.ok()) { + CheckStrByValues(knowhere::Metric::TYPE, GPU_METRICS); + } else { + CheckStrByValues(knowhere::Metric::TYPE, CPU_METRICS); } #endif + CheckIntByRange(knowhere::meta::DIM, DEFAULT_MIN_DIM, DEFAULT_MAX_DIM); + CheckIntByRange(knowhere::meta::ROWS, DEFAULT_MIN_ROWS, DEFAULT_MAX_ROWS); + CheckIntByRange(knowhere::IndexParams::nlist, MIN_NLIST, MAX_NLIST); + + // int64_t nlist = oricfg[knowhere::IndexParams::nlist]; + // CheckIntByRange(knowhere::meta::ROWS, nlist, DEFAULT_MAX_ROWS); + + // auto tune params + oricfg[knowhere::IndexParams::nlist] = MatchNlist(oricfg[knowhere::meta::ROWS].get(), + oricfg[knowhere::IndexParams::nlist].get(), 16384); + + // Best Practice + // static int64_t MIN_POINTS_PER_CENTROID = 40; + // static int64_t MAX_POINTS_PER_CENTROID = 256; + // CheckIntByRange(knowhere::meta::ROWS, MIN_POINTS_PER_CENTROID * nlist, MAX_POINTS_PER_CENTROID * nlist); /* * Faiss 1.6 @@ -147,165 +191,110 @@ IVFPQConfAdapter::Match(const TempMetaConf& metaconf) { static std::vector support_subquantizer{96, 64, 56, 48, 40, 32, 28, 24, 20, 16, 12, 8, 4, 3, 2, 1}; std::vector resset; for (const auto& dimperquantizer : support_dim_per_subquantizer) { - if (!(conf->d % dimperquantizer)) { - auto subquantzier_num = conf->d / dimperquantizer; + if (!(oricfg[knowhere::meta::DIM].get() % dimperquantizer)) { + auto subquantzier_num = oricfg[knowhere::meta::DIM].get() / dimperquantizer; auto finder = std::find(support_subquantizer.begin(), support_subquantizer.end(), subquantzier_num); if (finder != support_subquantizer.end()) { resset.push_back(subquantzier_num); } } } - fiu_do_on("IVFPQConfAdapter.Match.empty_resset", resset.clear()); - if (resset.empty()) { - // todo(linxj): throw exception here. - WRAPPER_LOG_ERROR << "The dims of PQ is wrong : only 1, 2, 3, 4, 6, 8, 10, 12, 16, 20, 24, 28, 32 dims per sub-" - "quantizer are currently supported with no precomputed codes."; - throw WrapperException( - "The dims of PQ is wrong : only 1, 2, 3, 4, 6, 8, 10, 12, 16, 20, 24, 28, 32 dims " - "per sub-quantizer are currently supported with no precomputed codes."); - // return nullptr; - } - static int64_t compression_level = 1; // 1:low, 2:high - if (compression_level == 1) { - conf->m = resset[int(resset.size() / 2)]; - WRAPPER_LOG_DEBUG << "PQ m = " << conf->m << ", compression radio = " << conf->d / conf->m * 4; - } - return conf; + CheckIntByValues(knowhere::IndexParams::m, resset); + + return true; } -knowhere::Config -IVFPQConfAdapter::MatchSearch(const TempMetaConf& metaconf, const IndexType& type) { - auto conf = std::make_shared(); - conf->k = metaconf.k; +bool +NSGConfAdapter::CheckTrain(milvus::json& oricfg) { + static int64_t MIN_KNNG = 5; + static int64_t MAX_KNNG = 300; + static int64_t MIN_SEARCH_LENGTH = 10; + static int64_t MAX_SEARCH_LENGTH = 300; + static int64_t MIN_OUT_DEGREE = 5; + static int64_t MAX_OUT_DEGREE = 300; + static int64_t MIN_CANDIDATE_POOL_SIZE = 50; + static int64_t MAX_CANDIDATE_POOL_SIZE = 1000; + static std::vector METRICS{knowhere::Metric::L2}; - if (metaconf.nprobe <= 0) { - WRAPPER_LOG_ERROR << "The nprobe of PQ is wrong!"; - throw WrapperException("The nprobe of PQ is wrong!"); - } else { - conf->nprobe = metaconf.nprobe; - } + CheckStrByValues(knowhere::Metric::TYPE, METRICS); + CheckIntByRange(knowhere::meta::ROWS, DEFAULT_MIN_ROWS, DEFAULT_MAX_ROWS); + CheckIntByRange(knowhere::IndexParams::knng, MIN_KNNG, MAX_KNNG); + CheckIntByRange(knowhere::IndexParams::search_length, MIN_SEARCH_LENGTH, MAX_SEARCH_LENGTH); + CheckIntByRange(knowhere::IndexParams::out_degree, MIN_OUT_DEGREE, MAX_OUT_DEGREE); + CheckIntByRange(knowhere::IndexParams::candidate, MIN_CANDIDATE_POOL_SIZE, MAX_CANDIDATE_POOL_SIZE); - return conf; + // auto tune params + oricfg[knowhere::IndexParams::nlist] = MatchNlist(oricfg[knowhere::meta::ROWS].get(), 8192, 8192); + oricfg[knowhere::IndexParams::nprobe] = int(oricfg[knowhere::IndexParams::nlist].get() * 0.01); + + return true; } -int64_t -IVFPQConfAdapter::MatchNlist(const int64_t& size, const int64_t& nlist) { - if (size <= TYPICAL_COUNT / 16384 + 1) { - // handle less row count, avoid nlist set to 0 - return 1; - } else if (int(size / TYPICAL_COUNT) * nlist <= 0) { - // calculate a proper nlist if nlist not specified or size less than TYPICAL_COUNT - return int(size / TYPICAL_COUNT * 16384); - } - return nlist; +bool +NSGConfAdapter::CheckSearch(milvus::json& oricfg, const IndexType& type) { + static int64_t MIN_SEARCH_LENGTH = 1; + static int64_t MAX_SEARCH_LENGTH = 300; + + CheckIntByRange(knowhere::IndexParams::search_length, MIN_SEARCH_LENGTH, MAX_SEARCH_LENGTH); + + return ConfAdapter::CheckSearch(oricfg, type); } -knowhere::Config -NSGConfAdapter::Match(const TempMetaConf& metaconf) { - auto conf = std::make_shared(); - conf->nlist = MatchNlist(metaconf.size, metaconf.nlist, 16384); - conf->d = metaconf.dim; - conf->metric_type = metaconf.metric_type; - conf->gpu_id = metaconf.gpu_id; - conf->k = metaconf.k; +bool +HNSWConfAdapter::CheckTrain(milvus::json& oricfg) { + static int64_t MIN_EFCONSTRUCTION = 100; + static int64_t MAX_EFCONSTRUCTION = 500; + static int64_t MIN_M = 5; + static int64_t MAX_M = 48; - auto scale_factor = round(metaconf.dim / 128.0); - scale_factor = scale_factor >= 4 ? 4 : scale_factor; - conf->nprobe = int64_t(conf->nlist * 0.01); - // conf->knng = 40 + 10 * scale_factor; // the size of knng - conf->knng = 50; - conf->search_length = 50 + 5 * scale_factor; - conf->out_degree = 50 + 5 * scale_factor; - conf->candidate_pool_size = 300; - MatchBase(conf); - return conf; + CheckIntByRange(knowhere::meta::ROWS, DEFAULT_MIN_ROWS, DEFAULT_MAX_ROWS); + CheckIntByRange(knowhere::IndexParams::efConstruction, MIN_EFCONSTRUCTION, MAX_EFCONSTRUCTION); + CheckIntByRange(knowhere::IndexParams::M, MIN_M, MAX_M); + + return ConfAdapter::CheckTrain(oricfg); } -knowhere::Config -NSGConfAdapter::MatchSearch(const TempMetaConf& metaconf, const IndexType& type) { - auto conf = std::make_shared(); - conf->k = metaconf.k; - conf->search_length = metaconf.search_length; - if (metaconf.search_length == TEMPMETA_DEFAULT_VALUE) { - conf->search_length = 30; // TODO(linxj): hardcode here. - } - return conf; +bool +HNSWConfAdapter::CheckSearch(milvus::json& oricfg, const IndexType& type) { + static int64_t MAX_EF = 4096; + + CheckIntByRange(knowhere::IndexParams::ef, oricfg[knowhere::meta::TOPK], MAX_EF); + + return ConfAdapter::CheckSearch(oricfg, type); } -knowhere::Config -SPTAGKDTConfAdapter::Match(const TempMetaConf& metaconf) { - auto conf = std::make_shared(); - conf->d = metaconf.dim; - conf->metric_type = metaconf.metric_type; - return conf; +bool +BinIDMAPConfAdapter::CheckTrain(milvus::json& oricfg) { + static std::vector METRICS{knowhere::Metric::HAMMING, knowhere::Metric::JACCARD, + knowhere::Metric::TANIMOTO}; + + CheckIntByRange(knowhere::meta::DIM, DEFAULT_MIN_DIM, DEFAULT_MAX_DIM); + CheckStrByValues(knowhere::Metric::TYPE, METRICS); + + return true; } -knowhere::Config -SPTAGKDTConfAdapter::MatchSearch(const TempMetaConf& metaconf, const IndexType& type) { - auto conf = std::make_shared(); - conf->k = metaconf.k; - return conf; -} +bool +BinIVFConfAdapter::CheckTrain(milvus::json& oricfg) { + static std::vector METRICS{knowhere::Metric::HAMMING, knowhere::Metric::JACCARD, + knowhere::Metric::TANIMOTO}; + static int64_t MAX_NLIST = 999999; + static int64_t MIN_NLIST = 1; -knowhere::Config -SPTAGBKTConfAdapter::Match(const TempMetaConf& metaconf) { - auto conf = std::make_shared(); - conf->d = metaconf.dim; - conf->metric_type = metaconf.metric_type; - return conf; -} + CheckIntByRange(knowhere::meta::ROWS, DEFAULT_MIN_ROWS, DEFAULT_MAX_ROWS); + CheckIntByRange(knowhere::meta::DIM, DEFAULT_MIN_DIM, DEFAULT_MAX_DIM); + CheckIntByRange(knowhere::IndexParams::nlist, MIN_NLIST, MAX_NLIST); + CheckStrByValues(knowhere::Metric::TYPE, METRICS); -knowhere::Config -SPTAGBKTConfAdapter::MatchSearch(const TempMetaConf& metaconf, const IndexType& type) { - auto conf = std::make_shared(); - conf->k = metaconf.k; - return conf; -} + int64_t nlist = oricfg[knowhere::IndexParams::nlist]; + CheckIntByRange(knowhere::meta::ROWS, nlist, DEFAULT_MAX_ROWS); -knowhere::Config -HNSWConfAdapter::Match(const TempMetaConf& metaconf) { - auto conf = std::make_shared(); - conf->d = metaconf.dim; - conf->metric_type = metaconf.metric_type; + // Best Practice + // static int64_t MIN_POINTS_PER_CENTROID = 40; + // static int64_t MAX_POINTS_PER_CENTROID = 256; + // CheckIntByRange(knowhere::meta::ROWS, MIN_POINTS_PER_CENTROID * nlist, MAX_POINTS_PER_CENTROID * nlist); - conf->ef = 500; // ef can be auto-configured by using sample data. - conf->M = 24; // A reasonable range of M is from 5 to 48. - return conf; -} - -knowhere::Config -HNSWConfAdapter::MatchSearch(const TempMetaConf& metaconf, const IndexType& type) { - auto conf = std::make_shared(); - conf->k = metaconf.k; - - if (metaconf.nprobe < metaconf.k) { - conf->ef = metaconf.k + 32; - } else { - conf->ef = metaconf.nprobe; - } - return conf; -} - -knowhere::Config -BinIDMAPConfAdapter::Match(const TempMetaConf& metaconf) { - auto conf = std::make_shared(); - conf->d = metaconf.dim; - conf->metric_type = metaconf.metric_type; - conf->gpu_id = metaconf.gpu_id; - conf->k = metaconf.k; - MatchBase(conf, knowhere::METRICTYPE::HAMMING); - return conf; -} - -knowhere::Config -BinIVFConfAdapter::Match(const TempMetaConf& metaconf) { - auto conf = std::make_shared(); - conf->nlist = MatchNlist(metaconf.size, metaconf.nlist, 2048); - conf->d = metaconf.dim; - conf->metric_type = metaconf.metric_type; - conf->gpu_id = metaconf.gpu_id; - MatchBase(conf, knowhere::METRICTYPE::HAMMING); - return conf; + return true; } } // namespace engine } // namespace milvus diff --git a/core/src/wrapper/ConfAdapter.h b/core/src/wrapper/ConfAdapter.h index 11086075cf..7d42598ecd 100644 --- a/core/src/wrapper/ConfAdapter.h +++ b/core/src/wrapper/ConfAdapter.h @@ -14,117 +14,78 @@ #include #include "VecIndex.h" -#include "knowhere/common/Config.h" +#include "utils/Json.h" namespace milvus { namespace engine { -// TODO(linxj): remove later, replace with real metaconf -constexpr int64_t TEMPMETA_DEFAULT_VALUE = -1; -struct TempMetaConf { - int64_t size = TEMPMETA_DEFAULT_VALUE; - int64_t nlist = TEMPMETA_DEFAULT_VALUE; - int64_t dim = TEMPMETA_DEFAULT_VALUE; - int64_t gpu_id = TEMPMETA_DEFAULT_VALUE; - int64_t k = TEMPMETA_DEFAULT_VALUE; - int64_t nprobe = TEMPMETA_DEFAULT_VALUE; - int64_t search_length = TEMPMETA_DEFAULT_VALUE; - knowhere::METRICTYPE metric_type = knowhere::DEFAULT_TYPE; -}; - class ConfAdapter { public: - virtual knowhere::Config - Match(const TempMetaConf& metaconf); + virtual bool + CheckTrain(milvus::json& oricfg); - virtual knowhere::Config - MatchSearch(const TempMetaConf& metaconf, const IndexType& type); + virtual bool + CheckSearch(milvus::json& oricfg, const IndexType& type); - protected: - static void - MatchBase(knowhere::Config conf, knowhere::METRICTYPE defalut_metric = knowhere::METRICTYPE::L2); + // todo(linxj): refactor in next release. + // + // virtual bool + // CheckTrain(milvus::json&, IndexMode&) = 0; + // + // virtual bool + // CheckSearch(milvus::json&, const IndexType&, IndexMode&) = 0; }; - using ConfAdapterPtr = std::shared_ptr; class IVFConfAdapter : public ConfAdapter { public: - knowhere::Config - Match(const TempMetaConf& metaconf) override; + bool + CheckTrain(milvus::json& oricfg) override; - knowhere::Config - MatchSearch(const TempMetaConf& metaconf, const IndexType& type) override; - - protected: - static int64_t - MatchNlist(const int64_t& size, const int64_t& nlist, const int64_t& per_nlist); + bool + CheckSearch(milvus::json& oricfg, const IndexType& type) override; }; class IVFSQConfAdapter : public IVFConfAdapter { public: - knowhere::Config - Match(const TempMetaConf& metaconf) override; + bool + CheckTrain(milvus::json& oricfg) override; }; class IVFPQConfAdapter : public IVFConfAdapter { public: - knowhere::Config - Match(const TempMetaConf& metaconf) override; - - knowhere::Config - MatchSearch(const TempMetaConf& metaconf, const IndexType& type) override; - - protected: - static int64_t - MatchNlist(const int64_t& size, const int64_t& nlist); + bool + CheckTrain(milvus::json& oricfg) override; }; class NSGConfAdapter : public IVFConfAdapter { public: - knowhere::Config - Match(const TempMetaConf& metaconf) override; + bool + CheckTrain(milvus::json& oricfg) override; - knowhere::Config - MatchSearch(const TempMetaConf& metaconf, const IndexType& type) final; -}; - -class SPTAGKDTConfAdapter : public ConfAdapter { - public: - knowhere::Config - Match(const TempMetaConf& metaconf) override; - - knowhere::Config - MatchSearch(const TempMetaConf& metaconf, const IndexType& type) override; -}; - -class SPTAGBKTConfAdapter : public ConfAdapter { - public: - knowhere::Config - Match(const TempMetaConf& metaconf) override; - - knowhere::Config - MatchSearch(const TempMetaConf& metaconf, const IndexType& type) override; + bool + CheckSearch(milvus::json& oricfg, const IndexType& type) override; }; class BinIDMAPConfAdapter : public ConfAdapter { public: - knowhere::Config - Match(const TempMetaConf& metaconf) override; + bool + CheckTrain(milvus::json& oricfg) override; }; class BinIVFConfAdapter : public IVFConfAdapter { public: - knowhere::Config - Match(const TempMetaConf& metaconf) override; + bool + CheckTrain(milvus::json& oricfg) override; }; class HNSWConfAdapter : public ConfAdapter { public: - knowhere::Config - Match(const TempMetaConf& metaconf) override; + bool + CheckTrain(milvus::json& oricfg) override; - knowhere::Config - MatchSearch(const TempMetaConf& metaconf, const IndexType& type) override; + bool + CheckSearch(milvus::json& oricfg, const IndexType& type) override; }; } // namespace engine diff --git a/core/src/wrapper/ConfAdapterMgr.cpp b/core/src/wrapper/ConfAdapterMgr.cpp index cabf113b9d..298a3a588e 100644 --- a/core/src/wrapper/ConfAdapterMgr.cpp +++ b/core/src/wrapper/ConfAdapterMgr.cpp @@ -54,8 +54,8 @@ AdapterMgr::RegisterAdapter() { REGISTER_CONF_ADAPTER(NSGConfAdapter, IndexType::NSG_MIX, nsg_mix); - REGISTER_CONF_ADAPTER(SPTAGKDTConfAdapter, IndexType::SPTAG_KDT_RNT_CPU, sptag_kdt); - REGISTER_CONF_ADAPTER(SPTAGBKTConfAdapter, IndexType::SPTAG_BKT_RNT_CPU, sptag_bkt); + REGISTER_CONF_ADAPTER(ConfAdapter, IndexType::SPTAG_KDT_RNT_CPU, sptag_kdt); + REGISTER_CONF_ADAPTER(ConfAdapter, IndexType::SPTAG_BKT_RNT_CPU, sptag_bkt); REGISTER_CONF_ADAPTER(HNSWConfAdapter, IndexType::HNSW, hnsw); } diff --git a/core/src/wrapper/VecImpl.cpp b/core/src/wrapper/VecImpl.cpp index 68aa920029..71fc59c083 100644 --- a/core/src/wrapper/VecImpl.cpp +++ b/core/src/wrapper/VecImpl.cpp @@ -40,7 +40,7 @@ Status VecIndexImpl::BuildAll(const int64_t& nb, const float* xb, const int64_t* ids, const Config& cfg, const int64_t& nt, const float* xt) { try { - dim = cfg->d; + dim = cfg[knowhere::meta::DIM]; auto dataset = GenDatasetWithIds(nb, dim, xb, ids); fiu_do_on("VecIndexImpl.BuildAll.throw_knowhere_exception", throw knowhere::KnowhereException("")); fiu_do_on("VecIndexImpl.BuildAll.throw_std_exception", throw std::exception()); @@ -80,15 +80,13 @@ VecIndexImpl::Add(const int64_t& nb, const float* xb, const int64_t* ids, const Status VecIndexImpl::Search(const int64_t& nq, const float* xq, float* dist, int64_t* ids, const Config& cfg) { try { - auto k = cfg->k; + int64_t k = cfg[knowhere::meta::TOPK]; auto dataset = GenDataset(nq, dim, xq); - Config search_cfg = cfg; - fiu_do_on("VecIndexImpl.Search.throw_knowhere_exception", throw knowhere::KnowhereException("")); fiu_do_on("VecIndexImpl.Search.throw_std_exception", throw std::exception()); - auto res = index_->Search(dataset, search_cfg); + auto res = index_->Search(dataset, cfg); //{ // auto& ids = ids_array; // auto& dists = dis_array; @@ -216,8 +214,7 @@ VecIndexImpl::GetVectorById(const int64_t n, const int64_t* xid, float* x, const dataset->Set(knowhere::meta::DIM, dim); dataset->Set(knowhere::meta::IDS, xid); - Config search_cfg = cfg; - auto res = index_->GetVectorById(dataset, search_cfg); + auto res = index_->GetVectorById(dataset, cfg); // TODO(linxj): avoid copy here. auto res_x = res->Get(knowhere::meta::TENSOR); @@ -242,14 +239,13 @@ VecIndexImpl::SearchById(const int64_t& nq, const int64_t* xq, float* dist, int6 } try { - auto k = cfg->k; + int64_t k = cfg[knowhere::meta::TOPK]; auto dataset = std::make_shared(); dataset->Set(knowhere::meta::ROWS, nq); dataset->Set(knowhere::meta::DIM, dim); dataset->Set(knowhere::meta::IDS, xq); - Config search_cfg = cfg; - auto res = index_->SearchById(dataset, search_cfg); + auto res = index_->SearchById(dataset, cfg); //{ // auto& ids = ids_array; // auto& dists = dis_array; @@ -337,7 +333,7 @@ BFIndex::Build(const Config& cfg) { try { fiu_do_on("BFIndex.Build.throw_knowhere_exception", throw knowhere::KnowhereException("")); fiu_do_on("BFIndex.Build.throw_std_exception", throw std::exception()); - dim = cfg->d; + dim = cfg[knowhere::meta::DIM]; std::static_pointer_cast(index_)->Train(cfg); } catch (knowhere::KnowhereException& e) { WRAPPER_LOG_ERROR << e.what(); @@ -353,7 +349,7 @@ Status BFIndex::BuildAll(const int64_t& nb, const float* xb, const int64_t* ids, const Config& cfg, const int64_t& nt, const float* xt) { try { - dim = cfg->d; + dim = cfg[knowhere::meta::DIM]; auto dataset = GenDatasetWithIds(nb, dim, xb, ids); fiu_do_on("BFIndex.BuildAll.throw_knowhere_exception", throw knowhere::KnowhereException("")); fiu_do_on("BFIndex.BuildAll.throw_std_exception", throw std::exception()); diff --git a/core/src/wrapper/VecIndex.h b/core/src/wrapper/VecIndex.h index 9916459a04..f7cf50b5bc 100644 --- a/core/src/wrapper/VecIndex.h +++ b/core/src/wrapper/VecIndex.h @@ -12,6 +12,7 @@ #pragma once #include +#include #include #include @@ -29,7 +30,8 @@ namespace milvus { namespace engine { -using Config = knowhere::Config; +using json = nlohmann::json; +using Config = json; // TODO(linxj): replace with string, Do refactor serialization enum class IndexType { diff --git a/core/src/wrapper/gpu/GPUVecImpl.cpp b/core/src/wrapper/gpu/GPUVecImpl.cpp index 197e47749a..edf8f05c85 100644 --- a/core/src/wrapper/gpu/GPUVecImpl.cpp +++ b/core/src/wrapper/gpu/GPUVecImpl.cpp @@ -38,7 +38,7 @@ IVFMixIndex::BuildAll(const int64_t& nb, const float* xb, const int64_t* ids, co fiu_do_on("IVFMixIndex.BuildAll.throw_knowhere_exception", throw knowhere::KnowhereException("")); fiu_do_on("IVFMixIndex.BuildAll.throw_std_exception", throw std::exception()); - dim = cfg->d; + dim = cfg[knowhere::meta::DIM]; auto dataset = GenDatasetWithIds(nb, dim, xb, ids); auto preprocessor = index_->BuildPreprocessor(dataset, cfg); index_->set_preprocessor(preprocessor); diff --git a/core/unittest/db/test_db.cpp b/core/unittest/db/test_db.cpp index 003ad332ca..459fa24aed 100644 --- a/core/unittest/db/test_db.cpp +++ b/core/unittest/db/test_db.cpp @@ -186,16 +186,20 @@ TEST_F(DBTest, DB_TEST) { std::stringstream ss; uint64_t count = 0; uint64_t prev_count = 0; + milvus::json json_params = {{"nprobe", 10}}; for (auto j = 0; j < 10; ++j) { ss.str(""); db_->Size(count); prev_count = count; + if (count == 0) { + continue; + } START_TIMER; std::vector tags; - stat = db_->Query(dummy_context_, TABLE_NAME, tags, k, 10, qxb, result_ids, result_distances); + stat = db_->Query(dummy_context_, TABLE_NAME, tags, k, json_params, qxb, result_ids, result_distances); ss << "Search " << j << " With Size " << count / milvus::engine::M << " M"; STOP_TIMER(ss.str()); @@ -306,37 +310,41 @@ TEST_F(DBTest, SEARCH_TEST) { stat = db_->InsertVectors(TABLE_NAME, "", xb); ASSERT_TRUE(stat.ok()); + milvus::json json_params = {{"nprobe", 10}}; milvus::engine::TableIndex index; index.engine_type_ = (int)milvus::engine::EngineType::FAISS_IDMAP; - db_->CreateIndex(TABLE_NAME, index); // wait until build index finish - - { - std::vector tags; - milvus::engine::ResultIds result_ids; - milvus::engine::ResultDistances result_distances; - stat = db_->Query(dummy_context_, TABLE_NAME, tags, k, 10, xq, result_ids, result_distances); - ASSERT_TRUE(stat.ok()); - } - - index.engine_type_ = (int)milvus::engine::EngineType::FAISS_IVFFLAT; - db_->CreateIndex(TABLE_NAME, index); // wait until build index finish - - { - std::vector tags; - milvus::engine::ResultIds result_ids; - milvus::engine::ResultDistances result_distances; - stat = db_->Query(dummy_context_, TABLE_NAME, tags, k, 10, xq, result_ids, result_distances); - ASSERT_TRUE(stat.ok()); - } + index.extra_params_ = {{"nlist", 16384}}; +// db_->CreateIndex(TABLE_NAME, index); // wait until build index finish +// +// { +// std::vector tags; +// milvus::engine::ResultIds result_ids; +// milvus::engine::ResultDistances result_distances; +// stat = db_->Query(dummy_context_, TABLE_NAME, tags, k, json_params, xq, result_ids, result_distances); +// ASSERT_TRUE(stat.ok()); +// } +// +// index.engine_type_ = (int)milvus::engine::EngineType::FAISS_IVFFLAT; +// index.extra_params_ = {{"nlist", 16384}}; +// db_->CreateIndex(TABLE_NAME, index); // wait until build index finish +// +// { +// std::vector tags; +// milvus::engine::ResultIds result_ids; +// milvus::engine::ResultDistances result_distances; +// stat = db_->Query(dummy_context_, TABLE_NAME, tags, k, json_params, xq, result_ids, result_distances); +// ASSERT_TRUE(stat.ok()); +// } index.engine_type_ = (int)milvus::engine::EngineType::FAISS_IVFSQ8; + index.extra_params_ = {{"nlist", 16384}}; db_->CreateIndex(TABLE_NAME, index); // wait until build index finish { std::vector tags; milvus::engine::ResultIds result_ids; milvus::engine::ResultDistances result_distances; - stat = db_->Query(dummy_context_, TABLE_NAME, tags, k, 10, xq, result_ids, result_distances); + stat = db_->Query(dummy_context_, TABLE_NAME, tags, k, json_params, xq, result_ids, result_distances); ASSERT_TRUE(stat.ok()); } @@ -349,7 +357,7 @@ TEST_F(DBTest, SEARCH_TEST) { std::vector tags; milvus::engine::ResultIds result_ids; milvus::engine::ResultDistances result_distances; - stat = db_->Query(dummy_context_, TABLE_NAME, tags, k, 10, xq, result_ids, result_distances); + stat = db_->Query(dummy_context_, TABLE_NAME, tags, k, json_params, xq, result_ids, result_distances); ASSERT_TRUE(stat.ok()); } #endif @@ -365,16 +373,22 @@ TEST_F(DBTest, SEARCH_TEST) { } milvus::engine::ResultIds result_ids; milvus::engine::ResultDistances result_distances; - stat = db_->QueryByFileID(dummy_context_, TABLE_NAME, file_ids, k, 10, xq, result_ids, result_distances); + stat = db_->QueryByFileID(dummy_context_, TABLE_NAME, file_ids, k, + json_params, + xq, + result_ids, + result_distances); ASSERT_TRUE(stat.ok()); FIU_ENABLE_FIU("SqliteMetaImpl.FilesToSearch.throw_exception"); - stat = db_->QueryByFileID(dummy_context_, TABLE_NAME, file_ids, k, 10, xq, result_ids, result_distances); + stat = + db_->QueryByFileID(dummy_context_, TABLE_NAME, file_ids, k, json_params, xq, result_ids, result_distances); ASSERT_FALSE(stat.ok()); fiu_disable("SqliteMetaImpl.FilesToSearch.throw_exception"); FIU_ENABLE_FIU("DBImpl.QueryByFileID.empty_files_array"); - stat = db_->QueryByFileID(dummy_context_, TABLE_NAME, file_ids, k, 10, xq, result_ids, result_distances); + stat = + db_->QueryByFileID(dummy_context_, TABLE_NAME, file_ids, k, json_params, xq, result_ids, result_distances); ASSERT_FALSE(stat.ok()); fiu_disable("DBImpl.QueryByFileID.empty_files_array"); } @@ -385,13 +399,13 @@ TEST_F(DBTest, SEARCH_TEST) { std::vector tags; milvus::engine::ResultIds result_ids; milvus::engine::ResultDistances result_distances; - stat = db_->Query(dummy_context_, TABLE_NAME, tags, k, 10, xq, result_ids, result_distances); + stat = db_->Query(dummy_context_, TABLE_NAME, tags, k, json_params, xq, result_ids, result_distances); ASSERT_TRUE(stat.ok()); - stat = db_->Query(dummy_context_, TABLE_NAME, tags, k, 10, xq, result_ids, result_distances); + stat = db_->Query(dummy_context_, TABLE_NAME, tags, k, json_params, xq, result_ids, result_distances); ASSERT_TRUE(stat.ok()); FIU_ENABLE_FIU("SqliteMetaImpl.FilesToSearch.throw_exception"); - stat = db_->Query(dummy_context_, TABLE_NAME, tags, k, 10, xq, result_ids, result_distances); + stat = db_->Query(dummy_context_, TABLE_NAME, tags, k, json_params, xq, result_ids, result_distances); ASSERT_FALSE(stat.ok()); fiu_disable("SqliteMetaImpl.FilesToSearch.throw_exception"); } @@ -409,7 +423,7 @@ TEST_F(DBTest, SEARCH_TEST) { { result_ids.clear(); result_dists.clear(); - stat = db_->Query(dummy_context_, TABLE_NAME, partition_tag, k, 10, xq, result_ids, result_dists); + stat = db_->Query(dummy_context_, TABLE_NAME, partition_tag, k, json_params, xq, result_ids, result_dists); ASSERT_TRUE(stat.ok()); } @@ -423,7 +437,7 @@ TEST_F(DBTest, SEARCH_TEST) { } result_ids.clear(); result_dists.clear(); - stat = db_->QueryByFileID(dummy_context_, TABLE_NAME, file_ids, k, 10, xq, result_ids, result_dists); + stat = db_->QueryByFileID(dummy_context_, TABLE_NAME, file_ids, k, json_params, xq, result_ids, result_dists); ASSERT_TRUE(stat.ok()); } #endif @@ -564,13 +578,27 @@ TEST_F(DBTest, SHUTDOWN_TEST) { std::vector tags; milvus::engine::ResultIds result_ids; milvus::engine::ResultDistances result_distances; - stat = db_->Query(dummy_context_, table_info.table_id_, tags, 1, 1, xb, result_ids, result_distances); + milvus::json json_params = {{"nprobe", 1}}; + stat = db_->Query(dummy_context_, table_info.table_id_, tags, 1, json_params, xb, result_ids, result_distances); ASSERT_FALSE(stat.ok()); std::vector file_ids; - stat = db_->QueryByFileID(dummy_context_, table_info.table_id_, file_ids, 1, 1, xb, result_ids, result_distances); + stat = db_->QueryByFileID(dummy_context_, + table_info.table_id_, + file_ids, + 1, + json_params, + xb, + result_ids, + result_distances); ASSERT_FALSE(stat.ok()); - stat = db_->Query(dummy_context_, table_info.table_id_, tags, 1, 1, milvus::engine::VectorsData(), result_ids, + stat = db_->Query(dummy_context_, + table_info.table_id_, + tags, + 1, + json_params, + milvus::engine::VectorsData(), + result_ids, result_distances); ASSERT_FALSE(stat.ok()); @@ -731,7 +759,7 @@ TEST_F(DBTest, INDEX_TEST) { stat = db_->DescribeIndex(table_info.table_id_, index_out); ASSERT_TRUE(stat.ok()); ASSERT_EQ(index.engine_type_, index_out.engine_type_); - ASSERT_EQ(index.nlist_, index_out.nlist_); + ASSERT_EQ(index.extra_params_, index_out.extra_params_); ASSERT_EQ(table_info.metric_type_, index_out.metric_type_); stat = db_->DropIndex(table_info.table_id_); @@ -845,7 +873,9 @@ TEST_F(DBTest, PARTITION_TEST) { std::vector tags = {"0", std::to_string(PARTITION_COUNT - 1)}; milvus::engine::ResultIds result_ids; milvus::engine::ResultDistances result_distances; - stat = db_->Query(dummy_context_, TABLE_NAME, tags, topk, nprobe, xq, result_ids, result_distances); + milvus::json json_params = {{"nprobe", nprobe}}; + + stat = db_->Query(dummy_context_, TABLE_NAME, tags, topk, json_params, xq, result_ids, result_distances); ASSERT_TRUE(stat.ok()); ASSERT_EQ(result_ids.size() / topk, nq); @@ -853,7 +883,7 @@ TEST_F(DBTest, PARTITION_TEST) { tags.clear(); result_ids.clear(); result_distances.clear(); - stat = db_->Query(dummy_context_, TABLE_NAME, tags, topk, nprobe, xq, result_ids, result_distances); + stat = db_->Query(dummy_context_, TABLE_NAME, tags, topk, json_params, xq, result_ids, result_distances); ASSERT_TRUE(stat.ok()); ASSERT_EQ(result_ids.size() / topk, nq); @@ -861,7 +891,7 @@ TEST_F(DBTest, PARTITION_TEST) { tags.push_back("\\d"); result_ids.clear(); result_distances.clear(); - stat = db_->Query(dummy_context_, TABLE_NAME, tags, topk, nprobe, xq, result_ids, result_distances); + stat = db_->Query(dummy_context_, TABLE_NAME, tags, topk, json_params, xq, result_ids, result_distances); ASSERT_TRUE(stat.ok()); ASSERT_EQ(result_ids.size() / topk, nq); } @@ -1074,11 +1104,12 @@ TEST_F(DBTestWAL, DB_STOP_TEST) { const int64_t topk = 10; const int64_t nprobe = 10; + milvus::json json_params = {{"nprobe", nprobe}}; milvus::engine::ResultIds result_ids; milvus::engine::ResultDistances result_distances; milvus::engine::VectorsData qxb; BuildVectors(qb, 0, qxb); - stat = db_->Query(dummy_context_, table_info.table_id_, {}, topk, nprobe, qxb, result_ids, result_distances); + stat = db_->Query(dummy_context_, table_info.table_id_, {}, topk, json_params, qxb, result_ids, result_distances); ASSERT_TRUE(stat.ok()); ASSERT_EQ(result_ids.size() / topk, qb); @@ -1102,11 +1133,12 @@ TEST_F(DBTestWALRecovery, RECOVERY_WITH_NO_ERROR) { const int64_t topk = 10; const int64_t nprobe = 10; + milvus::json json_params = {{"nprobe", nprobe}}; milvus::engine::ResultIds result_ids; milvus::engine::ResultDistances result_distances; milvus::engine::VectorsData qxb; BuildVectors(qb, 0, qxb); - stat = db_->Query(dummy_context_, table_info.table_id_, {}, topk, nprobe, qxb, result_ids, result_distances); + stat = db_->Query(dummy_context_, table_info.table_id_, {}, topk, json_params, qxb, result_ids, result_distances); ASSERT_TRUE(stat.ok()); ASSERT_NE(result_ids.size() / topk, qb); @@ -1119,14 +1151,14 @@ TEST_F(DBTestWALRecovery, RECOVERY_WITH_NO_ERROR) { result_ids.clear(); result_distances.clear(); - stat = db_->Query(dummy_context_, table_info.table_id_, {}, topk, nprobe, qxb, result_ids, result_distances); + stat = db_->Query(dummy_context_, table_info.table_id_, {}, topk, json_params, qxb, result_ids, result_distances); ASSERT_TRUE(stat.ok()); ASSERT_EQ(result_ids.size(), 0); db_->Flush(); result_ids.clear(); result_distances.clear(); - stat = db_->Query(dummy_context_, table_info.table_id_, {}, topk, nprobe, qxb, result_ids, result_distances); + stat = db_->Query(dummy_context_, table_info.table_id_, {}, topk, json_params, qxb, result_ids, result_distances); ASSERT_TRUE(stat.ok()); ASSERT_EQ(result_ids.size() / topk, qb); } @@ -1312,6 +1344,7 @@ TEST_F(DBTest2, SEARCH_WITH_DIFFERENT_INDEX) { ASSERT_TRUE(stat.ok()); int topk = 10, nprobe = 10; + milvus::json json_params = {{"nprobe", nprobe}}; for (auto id : ids_to_search) { // std::cout << "xxxxxxxxxxxxxxxxxxxx " << i << std::endl; @@ -1319,7 +1352,7 @@ TEST_F(DBTest2, SEARCH_WITH_DIFFERENT_INDEX) { milvus::engine::ResultIds result_ids; milvus::engine::ResultDistances result_distances; - stat = db_->QueryByID(dummy_context_, table_info.table_id_, tags, topk, nprobe, id, result_ids, + stat = db_->QueryByID(dummy_context_, table_info.table_id_, tags, topk, json_params, id, result_ids, result_distances); ASSERT_TRUE(stat.ok()); ASSERT_EQ(result_ids[0], id); @@ -1341,7 +1374,7 @@ result_distances); milvus::engine::ResultIds result_ids; milvus::engine::ResultDistances result_distances; - stat = db_->QueryByID(dummy_context_, table_info.table_id_, tags, topk, nprobe, id, result_ids, + stat = db_->QueryByID(dummy_context_, table_info.table_id_, tags, topk, json_params, id, result_ids, result_distances); ASSERT_TRUE(stat.ok()); ASSERT_EQ(result_ids[0], id); diff --git a/core/unittest/db/test_db_mysql.cpp b/core/unittest/db/test_db_mysql.cpp index 9ec1304c0b..0a5a8ce84c 100644 --- a/core/unittest/db/test_db_mysql.cpp +++ b/core/unittest/db/test_db_mysql.cpp @@ -78,16 +78,20 @@ TEST_F(MySqlDBTest, DB_TEST) { std::stringstream ss; uint64_t count = 0; uint64_t prev_count = 0; + milvus::json json_params = {{"nprobe", 10}}; for (auto j = 0; j < 10; ++j) { ss.str(""); db_->Size(count); prev_count = count; + if (count == 0) { + continue; + } START_TIMER; std::vector tags; - stat = db_->Query(dummy_context_, TABLE_NAME, tags, k, 10, qxb, result_ids, result_distances); + stat = db_->Query(dummy_context_, TABLE_NAME, tags, k, json_params, qxb, result_ids, result_distances); ss << "Search " << j << " With Size " << count / milvus::engine::M << " M"; STOP_TIMER(ss.str()); @@ -186,7 +190,8 @@ TEST_F(MySqlDBTest, SEARCH_TEST) { std::vector tags; milvus::engine::ResultIds result_ids; milvus::engine::ResultDistances result_distances; - stat = db_->Query(dummy_context_, TABLE_NAME, tags, k, 10, xq, result_ids, result_distances); + milvus::json json_params = {{"nprobe", 10}}; + stat = db_->Query(dummy_context_, TABLE_NAME, tags, k, json_params, xq, result_ids, result_distances); ASSERT_TRUE(stat.ok()); } @@ -377,7 +382,8 @@ TEST_F(MySqlDBTest, PARTITION_TEST) { std::vector tags = {"0", std::to_string(PARTITION_COUNT - 1)}; milvus::engine::ResultIds result_ids; milvus::engine::ResultDistances result_distances; - stat = db_->Query(dummy_context_, TABLE_NAME, tags, 10, 10, xq, result_ids, result_distances); + milvus::json json_params = {{"nprobe", nprobe}}; + stat = db_->Query(dummy_context_, TABLE_NAME, tags, topk, json_params, xq, result_ids, result_distances); ASSERT_TRUE(stat.ok()); ASSERT_EQ(result_ids.size() / topk, nq); @@ -385,7 +391,7 @@ TEST_F(MySqlDBTest, PARTITION_TEST) { tags.clear(); result_ids.clear(); result_distances.clear(); - stat = db_->Query(dummy_context_, TABLE_NAME, tags, 10, 10, xq, result_ids, result_distances); + stat = db_->Query(dummy_context_, TABLE_NAME, tags, topk, json_params, xq, result_ids, result_distances); ASSERT_TRUE(stat.ok()); ASSERT_EQ(result_ids.size() / topk, nq); @@ -393,7 +399,7 @@ TEST_F(MySqlDBTest, PARTITION_TEST) { tags.push_back("\\d"); result_ids.clear(); result_distances.clear(); - stat = db_->Query(dummy_context_, TABLE_NAME, tags, 10, 10, xq, result_ids, result_distances); + stat = db_->Query(dummy_context_, TABLE_NAME, tags, topk, json_params, xq, result_ids, result_distances); ASSERT_TRUE(stat.ok()); ASSERT_EQ(result_ids.size() / topk, nq); } diff --git a/core/unittest/db/test_delete.cpp b/core/unittest/db/test_delete.cpp index 26441de971..4d6051455d 100644 --- a/core/unittest/db/test_delete.cpp +++ b/core/unittest/db/test_delete.cpp @@ -301,6 +301,7 @@ TEST_F(DeleteTest, delete_with_index) { milvus::engine::TableIndex index; index.engine_type_ = (int)milvus::engine::EngineType::FAISS_IVFSQ8; + index.extra_params_ = {{"nlist", 100}}; stat = db_->CreateIndex(GetTableName(), index); ASSERT_TRUE(stat.ok()); @@ -368,12 +369,13 @@ TEST_F(DeleteTest, delete_single_vector) { ASSERT_TRUE(stat.ok()); ASSERT_EQ(row_count, 0); - int topk = 1, nprobe = 1; + const int topk = 1, nprobe = 1; + milvus::json json_params = {{"nprobe", nprobe}}; std::vector tags; milvus::engine::ResultIds result_ids; milvus::engine::ResultDistances result_distances; - stat = db_->Query(dummy_context_, GetTableName(), tags, topk, nprobe, xb, result_ids, result_distances); + stat = db_->Query(dummy_context_, GetTableName(), tags, topk, json_params, xb, result_ids, result_distances); ASSERT_TRUE(result_ids.empty()); ASSERT_TRUE(result_distances.empty()); // ASSERT_EQ(result_ids[0], -1); @@ -402,6 +404,7 @@ TEST_F(DeleteTest, delete_add_create_index) { // ASSERT_TRUE(stat.ok()); milvus::engine::TableIndex index; index.engine_type_ = (int)milvus::engine::EngineType::FAISS_IVFSQ8; + index.extra_params_ = {{"nlist", 100}}; stat = db_->CreateIndex(GetTableName(), index); ASSERT_TRUE(stat.ok()); @@ -426,7 +429,8 @@ TEST_F(DeleteTest, delete_add_create_index) { ASSERT_TRUE(stat.ok()); ASSERT_EQ(row_count, nb * 2 - 1); - int topk = 10, nprobe = 10; + const int topk = 10, nprobe = 10; + milvus::json json_params = {{"nprobe", nprobe}}; std::vector tags; milvus::engine::ResultIds result_ids; @@ -435,14 +439,14 @@ TEST_F(DeleteTest, delete_add_create_index) { qb.float_data_.resize(TABLE_DIM); qb.vector_count_ = 1; qb.id_array_.clear(); - stat = db_->Query(dummy_context_, GetTableName(), tags, topk, nprobe, qb, result_ids, result_distances); + stat = db_->Query(dummy_context_, GetTableName(), tags, topk, json_params, qb, result_ids, result_distances); ASSERT_EQ(result_ids[0], xb2.id_array_.front()); ASSERT_LT(result_distances[0], 1e-4); result_ids.clear(); result_distances.clear(); - stat = db_->QueryByID(dummy_context_, GetTableName(), tags, topk, nprobe, ids_to_delete.front(), result_ids, + stat = db_->QueryByID(dummy_context_, GetTableName(), tags, topk, json_params, ids_to_delete.front(), result_ids, result_distances); ASSERT_EQ(result_ids[0], -1); ASSERT_EQ(result_distances[0], std::numeric_limits::max()); @@ -496,7 +500,8 @@ TEST_F(DeleteTest, delete_add_auto_flush) { ASSERT_TRUE(stat.ok()); ASSERT_EQ(row_count, nb * 2 - 1); - int topk = 10, nprobe = 10; + const int topk = 10, nprobe = 10; + milvus::json json_params = {{"nprobe", nprobe}}; std::vector tags; milvus::engine::ResultIds result_ids; @@ -505,7 +510,7 @@ TEST_F(DeleteTest, delete_add_auto_flush) { qb.float_data_.resize(TABLE_DIM); qb.vector_count_ = 1; qb.id_array_.clear(); - stat = db_->Query(dummy_context_, GetTableName(), tags, topk, nprobe, qb, result_ids, result_distances); + stat = db_->Query(dummy_context_, GetTableName(), tags, topk, json_params, qb, result_ids, result_distances); ASSERT_EQ(result_ids[0], xb2.id_array_.front()); ASSERT_LT(result_distances[0], 1e-4); @@ -555,7 +560,8 @@ TEST_F(CompactTest, compact_basic) { stat = db_->Compact(GetTableName()); ASSERT_TRUE(stat.ok()); - int topk = 1, nprobe = 1; + const int topk = 1, nprobe = 1; + milvus::json json_params = {{"nprobe", nprobe}}; std::vector tags; milvus::engine::ResultIds result_ids; @@ -563,7 +569,8 @@ TEST_F(CompactTest, compact_basic) { milvus::engine::VectorsData qb = xb; for (auto& id : ids_to_delete) { - stat = db_->QueryByID(dummy_context_, GetTableName(), tags, topk, nprobe, id, result_ids, result_distances); + stat = + db_->QueryByID(dummy_context_, GetTableName(), tags, topk, json_params, id, result_ids, result_distances); ASSERT_EQ(result_ids[0], -1); ASSERT_EQ(result_distances[0], std::numeric_limits::max()); } @@ -643,14 +650,17 @@ TEST_F(CompactTest, compact_with_index) { ASSERT_TRUE(stat.ok()); ASSERT_FLOAT_EQ(table_index.engine_type_, index.engine_type_); - int topk = 10, nprobe = 10; + const int topk = 10, nprobe = 10; + milvus::json json_params = {{"nprobe", nprobe}}; + for (auto& pair : search_vectors) { auto& search = pair.second; std::vector tags; milvus::engine::ResultIds result_ids; milvus::engine::ResultDistances result_distances; - stat = db_->Query(dummy_context_, GetTableName(), tags, topk, nprobe, search, result_ids, result_distances); + stat = + db_->Query(dummy_context_, GetTableName(), tags, topk, json_params, search, result_ids, result_distances); ASSERT_NE(result_ids[0], pair.first); // ASSERT_LT(result_distances[0], 1e-4); ASSERT_GT(result_distances[0], 1); diff --git a/core/unittest/db/test_engine.cpp b/core/unittest/db/test_engine.cpp index 4760d843ab..4a99b8fb9a 100644 --- a/core/unittest/db/test_engine.cpp +++ b/core/unittest/db/test_engine.cpp @@ -19,17 +19,58 @@ #include #include +namespace { + +static constexpr uint16_t DIMENSION = 64; +static constexpr int64_t ROW_COUNT = 1000; +static const char* INIT_PATH = "/tmp/milvus_index_1"; + +milvus::engine::ExecutionEnginePtr +CreateExecEngine(const milvus::json& json_params, milvus::engine::MetricType metric = milvus::engine::MetricType::IP) { + auto engine_ptr = milvus::engine::EngineFactory::Build( + DIMENSION, + INIT_PATH, + milvus::engine::EngineType::FAISS_IDMAP, + metric, + json_params); + + std::vector data; + std::vector ids; + data.reserve(ROW_COUNT * DIMENSION); + ids.reserve(ROW_COUNT); + for (int64_t i = 0; i < ROW_COUNT; i++) { + ids.push_back(i); + for (uint16_t k = 0; k < DIMENSION; k++) { + data.push_back(i * DIMENSION + k); + } + } + + auto status = engine_ptr->AddWithIds((int64_t)ids.size(), data.data(), ids.data()); + return engine_ptr; +} + +} // namespace + TEST_F(EngineTest, FACTORY_TEST) { + const milvus::json index_params = {{"nlist", 1024}}; { auto engine_ptr = milvus::engine::EngineFactory::Build( - 512, "/tmp/milvus_index_1", milvus::engine::EngineType::INVALID, milvus::engine::MetricType::IP, 1024); + 512, + "/tmp/milvus_index_1", + milvus::engine::EngineType::INVALID, + milvus::engine::MetricType::IP, + index_params); ASSERT_TRUE(engine_ptr == nullptr); } { auto engine_ptr = milvus::engine::EngineFactory::Build( - 512, "/tmp/milvus_index_1", milvus::engine::EngineType::FAISS_IDMAP, milvus::engine::MetricType::IP, 1024); + 512, + "/tmp/milvus_index_1", + milvus::engine::EngineType::FAISS_IDMAP, + milvus::engine::MetricType::IP, + index_params); ASSERT_TRUE(engine_ptr != nullptr); } @@ -37,28 +78,40 @@ TEST_F(EngineTest, FACTORY_TEST) { { auto engine_ptr = milvus::engine::EngineFactory::Build(512, "/tmp/milvus_index_1", milvus::engine::EngineType::FAISS_IVFFLAT, - milvus::engine::MetricType::IP, 1024); + milvus::engine::MetricType::IP, index_params); ASSERT_TRUE(engine_ptr != nullptr); } { auto engine_ptr = milvus::engine::EngineFactory::Build( - 512, "/tmp/milvus_index_1", milvus::engine::EngineType::FAISS_IVFSQ8, milvus::engine::MetricType::IP, 1024); + 512, + "/tmp/milvus_index_1", + milvus::engine::EngineType::FAISS_IVFSQ8, + milvus::engine::MetricType::IP, + index_params); ASSERT_TRUE(engine_ptr != nullptr); } { auto engine_ptr = milvus::engine::EngineFactory::Build( - 512, "/tmp/milvus_index_1", milvus::engine::EngineType::NSG_MIX, milvus::engine::MetricType::IP, 1024); + 512, + "/tmp/milvus_index_1", + milvus::engine::EngineType::NSG_MIX, + milvus::engine::MetricType::IP, + index_params); ASSERT_TRUE(engine_ptr != nullptr); } { auto engine_ptr = milvus::engine::EngineFactory::Build( - 512, "/tmp/milvus_index_1", milvus::engine::EngineType::FAISS_PQ, milvus::engine::MetricType::IP, 1024); + 512, + "/tmp/milvus_index_1", + milvus::engine::EngineType::FAISS_PQ, + milvus::engine::MetricType::IP, + index_params); ASSERT_TRUE(engine_ptr != nullptr); } @@ -66,7 +119,7 @@ TEST_F(EngineTest, FACTORY_TEST) { { auto engine_ptr = milvus::engine::EngineFactory::Build( 512, "/tmp/milvus_index_1", milvus::engine::EngineType::SPTAG_KDT, - milvus::engine::MetricType::L2, 1024); + milvus::engine::MetricType::L2, index_params); ASSERT_TRUE(engine_ptr != nullptr); } @@ -74,7 +127,7 @@ TEST_F(EngineTest, FACTORY_TEST) { { auto engine_ptr = milvus::engine::EngineFactory::Build( 512, "/tmp/milvus_index_1", milvus::engine::EngineType::SPTAG_KDT, - milvus::engine::MetricType::L2, 1024); + milvus::engine::MetricType::L2, index_params); ASSERT_TRUE(engine_ptr != nullptr); } @@ -85,7 +138,7 @@ TEST_F(EngineTest, FACTORY_TEST) { FIU_ENABLE_FIU("ExecutionEngineImpl.CreatetVecIndex.invalid_type"); ASSERT_ANY_THROW(milvus::engine::EngineFactory::Build( 512, "/tmp/milvus_index_1", milvus::engine::EngineType::SPTAG_KDT, - milvus::engine::MetricType::L2, 1024)); + milvus::engine::MetricType::L2, index_params)); fiu_disable("ExecutionEngineImpl.CreatetVecIndex.invalid_type"); } @@ -94,91 +147,88 @@ TEST_F(EngineTest, FACTORY_TEST) { FIU_ENABLE_FIU("BFIndex.Build.throw_knowhere_exception"); ASSERT_ANY_THROW(milvus::engine::EngineFactory::Build( 512, "/tmp/milvus_index_1", milvus::engine::EngineType::SPTAG_KDT, - milvus::engine::MetricType::L2, 1024)); + milvus::engine::MetricType::L2, index_params)); fiu_disable("BFIndex.Build.throw_knowhere_exception"); } } TEST_F(EngineTest, ENGINE_IMPL_TEST) { fiu_init(0); - uint16_t dimension = 64; - std::string file_path = "/tmp/milvus_index_1"; - auto engine_ptr = milvus::engine::EngineFactory::Build( - dimension, file_path, milvus::engine::EngineType::FAISS_IVFFLAT, milvus::engine::MetricType::IP, 1024); - std::vector data; - std::vector ids; - const int row_count = 500; - data.reserve(row_count * dimension); - ids.reserve(row_count); - for (int64_t i = 0; i < row_count; i++) { - ids.push_back(i); - for (uint16_t k = 0; k < dimension; k++) { - data.push_back(i * dimension + k); - } + { + milvus::json index_params = {{"nlist", 10}}; + auto engine_ptr = CreateExecEngine(index_params); + + ASSERT_EQ(engine_ptr->Dimension(), DIMENSION); + ASSERT_EQ(engine_ptr->Count(), ROW_COUNT); + ASSERT_EQ(engine_ptr->GetLocation(), INIT_PATH); + ASSERT_EQ(engine_ptr->IndexMetricType(), milvus::engine::MetricType::IP); + + ASSERT_ANY_THROW(engine_ptr->BuildIndex(INIT_PATH, milvus::engine::EngineType::INVALID)); + FIU_ENABLE_FIU("VecIndexImpl.BuildAll.throw_knowhere_exception"); + ASSERT_ANY_THROW(engine_ptr->BuildIndex(INIT_PATH, milvus::engine::EngineType::SPTAG_KDT)); + fiu_disable("VecIndexImpl.BuildAll.throw_knowhere_exception"); + + auto engine_build = engine_ptr->BuildIndex("/tmp/milvus_index_2", milvus::engine::EngineType::FAISS_IVFSQ8); + ASSERT_NE(engine_build, nullptr); } - auto status = engine_ptr->AddWithIds((int64_t)ids.size(), data.data(), ids.data()); - ASSERT_TRUE(status.ok()); - - ASSERT_EQ(engine_ptr->Dimension(), dimension); - ASSERT_EQ(engine_ptr->Count(), ids.size()); - - ASSERT_EQ(engine_ptr->GetLocation(), file_path); - ASSERT_EQ(engine_ptr->IndexMetricType(), milvus::engine::MetricType::IP); - - ASSERT_ANY_THROW(engine_ptr->BuildIndex(file_path, milvus::engine::EngineType::INVALID)); - FIU_ENABLE_FIU("VecIndexImpl.BuildAll.throw_knowhere_exception"); - ASSERT_ANY_THROW(engine_ptr->BuildIndex(file_path, milvus::engine::EngineType::SPTAG_KDT)); - fiu_disable("VecIndexImpl.BuildAll.throw_knowhere_exception"); - - auto engine_build = engine_ptr->BuildIndex("/tmp/milvus_index_2", milvus::engine::EngineType::FAISS_IVFSQ8); + { #ifndef MILVUS_GPU_VERSION - //PQ don't support IP In gpu version - engine_build = engine_ptr->BuildIndex("/tmp/milvus_index_3", milvus::engine::EngineType::FAISS_PQ); + milvus::json index_params = {{"nlist", 10}, {"m", 16}}; + auto engine_ptr = CreateExecEngine(index_params); + //PQ don't support IP In gpu version + auto engine_build = engine_ptr->BuildIndex("/tmp/milvus_index_3", milvus::engine::EngineType::FAISS_PQ); + ASSERT_NE(engine_build, nullptr); #endif - engine_build = engine_ptr->BuildIndex("/tmp/milvus_index_4", milvus::engine::EngineType::SPTAG_KDT); - engine_build = engine_ptr->BuildIndex("/tmp/milvus_index_5", milvus::engine::EngineType::SPTAG_BKT); - engine_ptr->BuildIndex("/tmp/milvus_index_SPTAG_BKT", milvus::engine::EngineType::SPTAG_BKT); + } + + { + milvus::json index_params = {{"nlist", 10}}; + auto engine_ptr = CreateExecEngine(index_params); + auto engine_build = engine_ptr->BuildIndex("/tmp/milvus_index_4", milvus::engine::EngineType::SPTAG_KDT); + engine_build = engine_ptr->BuildIndex("/tmp/milvus_index_5", milvus::engine::EngineType::SPTAG_BKT); + engine_ptr->BuildIndex("/tmp/milvus_index_SPTAG_BKT", milvus::engine::EngineType::SPTAG_BKT); + + //CPU version invoke CopyToCpu will fail + auto status = engine_ptr->CopyToCpu(); + ASSERT_FALSE(status.ok()); + } #ifdef MILVUS_GPU_VERSION - FIU_ENABLE_FIU("ExecutionEngineImpl.CreatetVecIndex.gpu_res_disabled"); - engine_ptr->BuildIndex("/tmp/milvus_index_NSG_MIX", milvus::engine::EngineType::NSG_MIX); - engine_ptr->BuildIndex("/tmp/milvus_index_6", milvus::engine::EngineType::FAISS_IVFFLAT); - engine_ptr->BuildIndex("/tmp/milvus_index_7", milvus::engine::EngineType::FAISS_IVFSQ8); - ASSERT_ANY_THROW(engine_ptr->BuildIndex("/tmp/milvus_index_8", milvus::engine::EngineType::FAISS_IVFSQ8H)); - ASSERT_ANY_THROW(engine_ptr->BuildIndex("/tmp/milvus_index_9", milvus::engine::EngineType::FAISS_PQ)); - fiu_disable("ExecutionEngineImpl.CreatetVecIndex.gpu_res_disabled"); -#endif + { + FIU_ENABLE_FIU("ExecutionEngineImpl.CreatetVecIndex.gpu_res_disabled"); + milvus::json index_params = {{"search_length", 100}, {"out_degree", 40}, {"pool_size", 100}, {"knng", 200}, + {"candidate_pool_size", 500}}; + auto engine_ptr = CreateExecEngine(index_params, milvus::engine::MetricType::L2); + engine_ptr->BuildIndex("/tmp/milvus_index_NSG_MIX", milvus::engine::EngineType::NSG_MIX); + fiu_disable("ExecutionEngineImpl.CreatetVecIndex.gpu_res_disabled"); - //CPU version invoke CopyToCpu will fail - status = engine_ptr->CopyToCpu(); - ASSERT_FALSE(status.ok()); + auto status = engine_ptr->CopyToGpu(0, false); + ASSERT_TRUE(status.ok()); + status = engine_ptr->GpuCache(0); + ASSERT_TRUE(status.ok()); + status = engine_ptr->CopyToGpu(0, false); + ASSERT_TRUE(status.ok()); -#ifdef MILVUS_GPU_VERSION - status = engine_ptr->CopyToGpu(0, false); - ASSERT_TRUE(status.ok()); - status = engine_ptr->GpuCache(0); - ASSERT_TRUE(status.ok()); - status = engine_ptr->CopyToGpu(0, false); - ASSERT_TRUE(status.ok()); + // auto new_engine = engine_ptr->Clone(); + // ASSERT_EQ(new_engine->Dimension(), dimension); + // ASSERT_EQ(new_engine->Count(), ids.size()); -// auto new_engine = engine_ptr->Clone(); -// ASSERT_EQ(new_engine->Dimension(), dimension); -// ASSERT_EQ(new_engine->Count(), ids.size()); - - status = engine_ptr->CopyToCpu(); - ASSERT_TRUE(status.ok()); - engine_ptr->CopyToCpu(); - ASSERT_TRUE(status.ok()); + status = engine_ptr->CopyToCpu(); + ASSERT_TRUE(status.ok()); + engine_ptr->CopyToCpu(); + ASSERT_TRUE(status.ok()); + } #endif } TEST_F(EngineTest, ENGINE_IMPL_NULL_INDEX_TEST) { uint16_t dimension = 64; std::string file_path = "/tmp/milvus_index_1"; + milvus::json index_params = {{"nlist", 1024}}; auto engine_ptr = milvus::engine::EngineFactory::Build( - dimension, file_path, milvus::engine::EngineType::FAISS_IVFFLAT, milvus::engine::MetricType::IP, 1024); + dimension, file_path, milvus::engine::EngineType::FAISS_IVFFLAT, milvus::engine::MetricType::IP, index_params); fiu_init(0); // init fiu_enable("read_null_index", 1, NULL, 0); @@ -209,11 +259,13 @@ TEST_F(EngineTest, ENGINE_IMPL_NULL_INDEX_TEST) { TEST_F(EngineTest, ENGINE_IMPL_THROW_EXCEPTION_TEST) { uint16_t dimension = 64; std::string file_path = "/tmp/invalid_file"; + milvus::json index_params = {{"nlist", 1024}}; + fiu_init(0); // init fiu_enable("ValidateStringNotBool", 1, NULL, 0); auto engine_ptr = milvus::engine::EngineFactory::Build( - dimension, file_path, milvus::engine::EngineType::FAISS_IVFFLAT, milvus::engine::MetricType::IP, 1024); + dimension, file_path, milvus::engine::EngineType::FAISS_IVFFLAT, milvus::engine::MetricType::IP, index_params); fiu_disable("ValidateStringNotBool"); diff --git a/core/unittest/db/test_mem.cpp b/core/unittest/db/test_mem.cpp index a5a0aa026b..ebd9fcf548 100644 --- a/core/unittest/db/test_mem.cpp +++ b/core/unittest/db/test_mem.cpp @@ -285,7 +285,8 @@ TEST_F(MemManagerTest2, SERIAL_INSERT_SEARCH_TEST) { search_vectors.insert(std::make_pair(xb.id_array_[index], search)); } - int topk = 10, nprobe = 10; + const int topk = 10, nprobe = 10; + milvus::json json_params = {{"nprobe", nprobe}}; for (auto& pair : search_vectors) { auto& search = pair.second; @@ -293,7 +294,8 @@ TEST_F(MemManagerTest2, SERIAL_INSERT_SEARCH_TEST) { milvus::engine::ResultIds result_ids; milvus::engine::ResultDistances result_distances; - stat = db_->Query(dummy_context_, GetTableName(), tags, topk, nprobe, search, result_ids, result_distances); + stat = + db_->Query(dummy_context_, GetTableName(), tags, topk, json_params, search, result_ids, result_distances); ASSERT_EQ(result_ids[0], pair.first); ASSERT_LT(result_distances[0], 1e-4); } @@ -388,6 +390,7 @@ TEST_F(MemManagerTest2, INSERT_BINARY_TEST) { // std::stringstream ss; // uint64_t count = 0; // uint64_t prev_count = 0; +// milvus::json json_params = {{"nprobe", 10}}; // // for (auto j = 0; j < 10; ++j) { // ss.str(""); @@ -397,7 +400,8 @@ TEST_F(MemManagerTest2, INSERT_BINARY_TEST) { // START_TIMER; // // std::vector tags; -// stat = db_->Query(dummy_context_, GetTableName(), tags, k, 10, qxb, result_ids, result_distances); +// stat = +// db_->Query(dummy_context_, GetTableName(), tags, k, json_params, qxb, result_ids, result_distances); // ss << "Search " << j << " With Size " << count / milvus::engine::M << " M"; // STOP_TIMER(ss.str()); // diff --git a/core/unittest/db/test_meta.cpp b/core/unittest/db/test_meta.cpp index 519a3c13a8..820e459670 100644 --- a/core/unittest/db/test_meta.cpp +++ b/core/unittest/db/test_meta.cpp @@ -647,7 +647,7 @@ TEST_F(MetaTest, INDEX_TEST) { milvus::engine::TableIndex index; index.metric_type_ = 2; - index.nlist_ = 1234; + index.extra_params_ = {{"nlist", 1234}}; index.engine_type_ = 3; status = impl_->UpdateTableIndex(table_id, index); ASSERT_TRUE(status.ok()); @@ -664,14 +664,13 @@ TEST_F(MetaTest, INDEX_TEST) { milvus::engine::TableIndex index_out; status = impl_->DescribeTableIndex(table_id, index_out); ASSERT_EQ(index_out.metric_type_, index.metric_type_); - ASSERT_EQ(index_out.nlist_, index.nlist_); + ASSERT_EQ(index_out.extra_params_, index.extra_params_); ASSERT_EQ(index_out.engine_type_, index.engine_type_); status = impl_->DropTableIndex(table_id); ASSERT_TRUE(status.ok()); status = impl_->DescribeTableIndex(table_id, index_out); ASSERT_EQ(index_out.metric_type_, index.metric_type_); - ASSERT_NE(index_out.nlist_, index.nlist_); ASSERT_NE(index_out.engine_type_, index.engine_type_); status = impl_->UpdateTableFilesToIndex(table_id); diff --git a/core/unittest/db/test_meta_mysql.cpp b/core/unittest/db/test_meta_mysql.cpp index 0e2210043c..75937e245d 100644 --- a/core/unittest/db/test_meta_mysql.cpp +++ b/core/unittest/db/test_meta_mysql.cpp @@ -700,7 +700,7 @@ TEST_F(MySqlMetaTest, INDEX_TEST) { milvus::engine::TableIndex index; index.metric_type_ = 2; - index.nlist_ = 1234; + index.extra_params_ = {{"nlist", 1234}}; index.engine_type_ = 3; status = impl_->UpdateTableIndex(table_id, index); ASSERT_TRUE(status.ok()); @@ -740,14 +740,13 @@ TEST_F(MySqlMetaTest, INDEX_TEST) { milvus::engine::TableIndex index_out; status = impl_->DescribeTableIndex(table_id, index_out); ASSERT_EQ(index_out.metric_type_, index.metric_type_); - ASSERT_EQ(index_out.nlist_, index.nlist_); + ASSERT_EQ(index_out.extra_params_, index.extra_params_); ASSERT_EQ(index_out.engine_type_, index.engine_type_); status = impl_->DropTableIndex(table_id); ASSERT_TRUE(status.ok()); status = impl_->DescribeTableIndex(table_id, index_out); ASSERT_EQ(index_out.metric_type_, index.metric_type_); - ASSERT_NE(index_out.nlist_, index.nlist_); ASSERT_NE(index_out.engine_type_, index.engine_type_); FIU_ENABLE_FIU("MySQLMetaImpl.DescribeTableIndex.null_connection"); diff --git a/core/unittest/db/test_misc.cpp b/core/unittest/db/test_misc.cpp index 870bd35a9a..4a0ab67315 100644 --- a/core/unittest/db/test_misc.cpp +++ b/core/unittest/db/test_misc.cpp @@ -235,3 +235,17 @@ TEST(DBMiscTest, CHECKER_TEST) { ASSERT_FALSE(checker.IsIgnored(schema)); } } + +TEST(DBMiscTest, IDGENERATOR_TEST) { + milvus::engine::SimpleIDGenerator gen; + size_t n = 1000000; + milvus::engine::IDNumbers ids; + gen.GetNextIDNumbers(n, ids); + + std::set unique_ids; + for (size_t i = 0; i < ids.size(); i++) { + unique_ids.insert(ids[i]); + } + + ASSERT_EQ(ids.size(), unique_ids.size()); +} diff --git a/core/unittest/db/test_search_by_id.cpp b/core/unittest/db/test_search_by_id.cpp index c71a155dd9..a4edb301b0 100644 --- a/core/unittest/db/test_search_by_id.cpp +++ b/core/unittest/db/test_search_by_id.cpp @@ -99,7 +99,8 @@ TEST_F(SearchByIdTest, basic) { stat = db_->Flush(); ASSERT_TRUE(stat.ok()); - int topk = 10, nprobe = 10; + const int topk = 10, nprobe = 10; + milvus::json json_params = {{"nprobe", nprobe}}; for (auto i : ids_to_search) { // std::cout << "xxxxxxxxxxxxxxxxxxxx " << i << std::endl; @@ -107,7 +108,7 @@ TEST_F(SearchByIdTest, basic) { milvus::engine::ResultIds result_ids; milvus::engine::ResultDistances result_distances; - stat = db_->QueryByID(dummy_context_, GetTableName(), tags, topk, nprobe, i, result_ids, result_distances); + stat = db_->QueryByID(dummy_context_, GetTableName(), tags, topk, json_params, i, result_ids, result_distances); ASSERT_EQ(result_ids[0], i); ASSERT_LT(result_distances[0], 1e-4); } @@ -151,10 +152,12 @@ TEST_F(SearchByIdTest, with_index) { milvus::engine::TableIndex index; index.engine_type_ = (int)milvus::engine::EngineType::FAISS_IVFSQ8; + index.extra_params_ = {{"nlist", 10}}; stat = db_->CreateIndex(GetTableName(), index); ASSERT_TRUE(stat.ok()); - int topk = 10, nprobe = 10; + const int topk = 10, nprobe = 10; + milvus::json json_params = {{"nprobe", nprobe}}; for (auto i : ids_to_search) { // std::cout << "xxxxxxxxxxxxxxxxxxxx " << i << std::endl; @@ -162,7 +165,7 @@ TEST_F(SearchByIdTest, with_index) { milvus::engine::ResultIds result_ids; milvus::engine::ResultDistances result_distances; - stat = db_->QueryByID(dummy_context_, GetTableName(), tags, topk, nprobe, i, result_ids, result_distances); + stat = db_->QueryByID(dummy_context_, GetTableName(), tags, topk, json_params, i, result_ids, result_distances); ASSERT_EQ(result_ids[0], i); ASSERT_LT(result_distances[0], 1e-3); } @@ -213,7 +216,8 @@ TEST_F(SearchByIdTest, with_delete) { stat = db_->Flush(); ASSERT_TRUE(stat.ok()); - int topk = 10, nprobe = 10; + const int topk = 10, nprobe = 10; + milvus::json json_params = {{"nprobe", nprobe}}; for (auto i : ids_to_search) { // std::cout << "xxxxxxxxxxxxxxxxxxxx " << i << std::endl; @@ -221,7 +225,7 @@ TEST_F(SearchByIdTest, with_delete) { milvus::engine::ResultIds result_ids; milvus::engine::ResultDistances result_distances; - stat = db_->QueryByID(dummy_context_, GetTableName(), tags, topk, nprobe, i, result_ids, result_distances); + stat = db_->QueryByID(dummy_context_, GetTableName(), tags, topk, json_params, i, result_ids, result_distances); ASSERT_EQ(result_ids[0], -1); ASSERT_EQ(result_distances[0], std::numeric_limits::max()); } @@ -263,7 +267,8 @@ TEST_F(GetVectorByIdTest, basic) { stat = db_->Flush(); ASSERT_TRUE(stat.ok()); - int topk = 10, nprobe = 10; + const int topk = 10, nprobe = 10; + milvus::json json_params = {{"nprobe", nprobe}}; for (auto id : ids_to_search) { // std::cout << "xxxxxxxxxxxxxxxxxxxx " << i << std::endl; @@ -275,7 +280,8 @@ TEST_F(GetVectorByIdTest, basic) { stat = db_->GetVectorByID(GetTableName(), id, vector); ASSERT_TRUE(stat.ok()); - stat = db_->Query(dummy_context_, GetTableName(), tags, topk, nprobe, vector, result_ids, result_distances); + stat = + db_->Query(dummy_context_, GetTableName(), tags, topk, json_params, vector, result_ids, result_distances); ASSERT_TRUE(stat.ok()); ASSERT_EQ(result_ids[0], id); ASSERT_LT(result_distances[0], 1e-4); @@ -319,11 +325,13 @@ TEST_F(GetVectorByIdTest, with_index) { ASSERT_TRUE(stat.ok()); milvus::engine::TableIndex index; + index.extra_params_ = {{"nlist", 10}}; index.engine_type_ = (int)milvus::engine::EngineType::FAISS_IVFSQ8; stat = db_->CreateIndex(GetTableName(), index); ASSERT_TRUE(stat.ok()); - int topk = 10, nprobe = 10; + const int topk = 10, nprobe = 10; + milvus::json json_params = {{"nprobe", nprobe}}; for (auto id : ids_to_search) { // std::cout << "xxxxxxxxxxxxxxxxxxxx " << i << std::endl; @@ -335,7 +343,8 @@ TEST_F(GetVectorByIdTest, with_index) { stat = db_->GetVectorByID(GetTableName(), id, vector); ASSERT_TRUE(stat.ok()); - stat = db_->Query(dummy_context_, GetTableName(), tags, topk, nprobe, vector, result_ids, result_distances); + stat = + db_->Query(dummy_context_, GetTableName(), tags, topk, json_params, vector, result_ids, result_distances); ASSERT_EQ(result_ids[0], id); ASSERT_LT(result_distances[0], 1e-3); } @@ -460,7 +469,8 @@ TEST_F(SearchByIdTest, BINARY) { ASSERT_TRUE(stat.ok()); ASSERT_EQ(row_count, nb * insert_loop); - int topk = 10, nprobe = 10; + const int topk = 10, nprobe = 10; + milvus::json json_params = {{"nprobe", nprobe}}; for (auto id : ids_to_search) { // std::cout << "xxxxxxxxxxxxxxxxxxxx " << i << std::endl; @@ -473,7 +483,8 @@ TEST_F(SearchByIdTest, BINARY) { ASSERT_TRUE(stat.ok()); ASSERT_EQ(vector.vector_count_, 1); - stat = db_->Query(dummy_context_, GetTableName(), tags, topk, nprobe, vector, result_ids, result_distances); + stat = + db_->Query(dummy_context_, GetTableName(), tags, topk, json_params, vector, result_ids, result_distances); ASSERT_TRUE(stat.ok()); ASSERT_EQ(result_ids[0], id); ASSERT_LT(result_distances[0], 1e-4); @@ -482,7 +493,8 @@ TEST_F(SearchByIdTest, BINARY) { result_ids.clear(); result_distances.clear(); - stat = db_->QueryByID(dummy_context_, GetTableName(), tags, topk, nprobe, id, result_ids, result_distances); + stat = + db_->QueryByID(dummy_context_, GetTableName(), tags, topk, json_params, id, result_ids, result_distances); ASSERT_TRUE(stat.ok()); ASSERT_EQ(result_ids[0], id); ASSERT_LT(result_distances[0], 1e-4); diff --git a/core/unittest/scheduler/test_optimizer.cpp b/core/unittest/scheduler/test_optimizer.cpp index 476fec5752..7d04124786 100644 --- a/core/unittest/scheduler/test_optimizer.cpp +++ b/core/unittest/scheduler/test_optimizer.cpp @@ -63,6 +63,8 @@ TEST(OptimizerTest, TEST_OPTIMIZER) { auto file = std::make_shared(); file->engine_type_ = (int)engine::EngineType::FAISS_IVFFLAT; + file->index_params_ = "{ \"nlist\": 100 }"; + file->dimension_ = 64; auto search_task = std::make_shared(nullptr, file, nullptr); ASSERT_FALSE(faiss_ivf_pq_pass.Run(search_task)); diff --git a/core/unittest/scheduler/test_task.cpp b/core/unittest/scheduler/test_task.cpp index 7cea1f523f..6079e537d7 100644 --- a/core/unittest/scheduler/test_task.cpp +++ b/core/unittest/scheduler/test_task.cpp @@ -36,6 +36,8 @@ TEST(TaskTest, INVALID_INDEX) { dummy_context->SetTraceContext(trace_context); TableFileSchemaPtr dummy_file = std::make_shared(); + dummy_file->index_params_ = "{ \"nlist\": 16384 }"; + dummy_file->dimension_ = 64; auto search_task = std::make_shared(dummy_context, dummy_file, nullptr); search_task->Load(LoadType::TEST, 10); @@ -50,6 +52,8 @@ TEST(TaskTest, TEST_TASK) { auto dummy_context = std::make_shared("dummy_request_id"); auto file = std::make_shared(); + file->index_params_ = "{ \"nlist\": 16384 }"; + file->dimension_ = 64; auto label = std::make_shared(); TestTask task(dummy_context, file, label); @@ -90,16 +94,18 @@ TEST(TaskTest, TEST_TASK) { build_index_task.Execute(); // always enable 'create_table_success' fiu_enable("XBuildIndexTask.Execute.create_table_success", 1, NULL, 0); + + milvus::json json = {{"nlist", 16384}}; build_index_task.to_index_engine_ = EngineFactory::Build(file->dimension_, file->location_, (EngineType)file->engine_type_, - (MetricType)file->metric_type_, file->nlist_); + (MetricType)file->metric_type_, json); build_index_task.Execute(); fiu_enable("XBuildIndexTask.Execute.build_index_fail", 1, NULL, 0); build_index_task.to_index_engine_ = EngineFactory::Build(file->dimension_, file->location_, (EngineType)file->engine_type_, - (MetricType)file->metric_type_, file->nlist_); + (MetricType)file->metric_type_, json); build_index_task.Execute(); fiu_disable("XBuildIndexTask.Execute.build_index_fail"); @@ -107,13 +113,13 @@ TEST(TaskTest, TEST_TASK) { fiu_enable("XBuildIndexTask.Execute.has_table", 1, NULL, 0); build_index_task.to_index_engine_ = EngineFactory::Build(file->dimension_, file->location_, (EngineType)file->engine_type_, - (MetricType)file->metric_type_, file->nlist_); + (MetricType)file->metric_type_, json); build_index_task.Execute(); fiu_enable("XBuildIndexTask.Execute.throw_std_exception", 1, NULL, 0); build_index_task.to_index_engine_ = EngineFactory::Build(file->dimension_, file->location_, (EngineType)file->engine_type_, - (MetricType)file->metric_type_, file->nlist_); + (MetricType)file->metric_type_, json); build_index_task.Execute(); fiu_disable("XBuildIndexTask.Execute.throw_std_exception"); @@ -121,13 +127,13 @@ TEST(TaskTest, TEST_TASK) { fiu_enable("XBuildIndexTask.Execute.save_index_file_success", 1, NULL, 0); build_index_task.to_index_engine_ = EngineFactory::Build(file->dimension_, file->location_, (EngineType)file->engine_type_, - (MetricType)file->metric_type_, file->nlist_); + (MetricType)file->metric_type_, json); build_index_task.Execute(); fiu_enable("XBuildIndexTask.Execute.update_table_file_fail", 1, NULL, 0); build_index_task.to_index_engine_ = EngineFactory::Build(file->dimension_, file->location_, (EngineType)file->engine_type_, - (MetricType)file->metric_type_, file->nlist_); + (MetricType)file->metric_type_, json); build_index_task.Execute(); fiu_disable("XBuildIndexTask.Execute.update_table_file_fail"); diff --git a/core/unittest/server/test_rpc.cpp b/core/unittest/server/test_rpc.cpp index 1f53d08114..e1b5bafb3f 100644 --- a/core/unittest/server/test_rpc.cpp +++ b/core/unittest/server/test_rpc.cpp @@ -197,10 +197,12 @@ TEST_F(RpcHandlerTest, INDEX_TEST) { request.set_table_name(TABLE_NAME); handler->CreateIndex(&context, &request, &response); - request.mutable_index()->set_index_type(1); + request.set_index_type(1); handler->CreateIndex(&context, &request, &response); - request.mutable_index()->set_nlist(16384); + ::milvus::grpc::KeyValuePair* kv = request.add_extra_params(); + kv->set_key("param"); + kv->set_value("{ \"nlist\": 16384 }"); grpc_status = handler->CreateIndex(&context, &request, &response); ASSERT_EQ(grpc_status.error_code(), ::grpc::Status::OK.error_code()); int error_code = response.error_code(); @@ -223,7 +225,7 @@ TEST_F(RpcHandlerTest, INDEX_TEST) { fiu_disable("CreateIndexRequest.OnExecute.create_index_fail"); #ifdef MILVUS_GPU_VERSION - request.mutable_index()->set_index_type(static_cast(milvus::engine::EngineType::FAISS_PQ)); + request.set_index_type(static_cast(milvus::engine::EngineType::FAISS_PQ)); fiu_enable("CreateIndexRequest.OnExecute.ip_meteric", 1, NULL, 0); grpc_status = handler->CreateIndex(&context, &request, &response); ASSERT_TRUE(grpc_status.ok()); @@ -361,7 +363,9 @@ TEST_F(RpcHandlerTest, SEARCH_TEST) { handler->Search(&context, &request, &response); // test empty query record array - request.set_nprobe(32); + milvus::grpc::KeyValuePair* kv = request.add_extra_params(); + kv->set_key(milvus::server::grpc::EXTRA_PARAM_KEY); + kv->set_value("{ \"nprobe\": 32 }"); handler->Search(&context, &request, &response); std::vector> record_array; diff --git a/core/unittest/server/test_util.cpp b/core/unittest/server/test_util.cpp index 9071bc0ec7..9fab0815e2 100644 --- a/core/unittest/server/test_util.cpp +++ b/core/unittest/server/test_util.cpp @@ -385,9 +385,6 @@ TEST(ValidationUtilTest, VALIDATE_INDEX_TEST) { milvus::server::ValidationUtil::ValidateTableIndexType((int)milvus::engine::EngineType::MAX_VALUE + 1).code(), milvus::SERVER_INVALID_INDEX_TYPE); - ASSERT_EQ(milvus::server::ValidationUtil::ValidateTableIndexNlist(0).code(), milvus::SERVER_INVALID_INDEX_NLIST); - ASSERT_EQ(milvus::server::ValidationUtil::ValidateTableIndexNlist(100).code(), milvus::SERVER_SUCCESS); - ASSERT_EQ(milvus::server::ValidationUtil::ValidateTableIndexFileSize(0).code(), milvus::SERVER_INVALID_INDEX_FILE_SIZE); ASSERT_EQ(milvus::server::ValidationUtil::ValidateTableIndexFileSize(100).code(), milvus::SERVER_SUCCESS); @@ -398,6 +395,169 @@ TEST(ValidationUtilTest, VALIDATE_INDEX_TEST) { ASSERT_EQ(milvus::server::ValidationUtil::ValidateTableIndexMetricType(2).code(), milvus::SERVER_SUCCESS); } +TEST(ValidationUtilTest, VALIDATE_INDEX_PARAMS_TEST) { + milvus::engine::meta::TableSchema table_schema; + table_schema.dimension_ = 64; + milvus::json json_params = {}; + + auto status = + milvus::server::ValidationUtil::ValidateIndexParams(json_params, + table_schema, + (int32_t)milvus::engine::EngineType::FAISS_IDMAP); + ASSERT_TRUE(status.ok()); + + status = + milvus::server::ValidationUtil::ValidateIndexParams(json_params, + table_schema, + (int32_t)milvus::engine::EngineType::FAISS_IVFFLAT); + ASSERT_FALSE(status.ok()); + + json_params = {{"nlist", "\t"}}; + status = + milvus::server::ValidationUtil::ValidateIndexParams(json_params, + table_schema, + (int32_t)milvus::engine::EngineType::FAISS_IVFSQ8H); + ASSERT_FALSE(status.ok()); + + json_params = {{"nlist", -1}}; + status = + milvus::server::ValidationUtil::ValidateIndexParams(json_params, + table_schema, + (int32_t)milvus::engine::EngineType::FAISS_IVFSQ8); + ASSERT_FALSE(status.ok()); + + json_params = {{"nlist", 32}}; + + status = + milvus::server::ValidationUtil::ValidateIndexParams(json_params, + table_schema, + (int32_t)milvus::engine::EngineType::FAISS_IVFFLAT); + ASSERT_TRUE(status.ok()); + + json_params = {{"nlist", -1}}; + status = + milvus::server::ValidationUtil::ValidateIndexParams(json_params, + table_schema, + (int32_t)milvus::engine::EngineType::FAISS_PQ); + ASSERT_FALSE(status.ok()); + + json_params = {{"nlist", 32}}; + status = + milvus::server::ValidationUtil::ValidateIndexParams(json_params, + table_schema, + (int32_t)milvus::engine::EngineType::FAISS_PQ); + ASSERT_FALSE(status.ok()); + + json_params = {{"nlist", 32}, {"m", 4}}; + status = + milvus::server::ValidationUtil::ValidateIndexParams(json_params, + table_schema, + (int32_t)milvus::engine::EngineType::FAISS_PQ); + ASSERT_TRUE(status.ok()); + + json_params = {{"search_length", -1}}; + status = + milvus::server::ValidationUtil::ValidateIndexParams(json_params, + table_schema, + (int32_t)milvus::engine::EngineType::NSG_MIX); + ASSERT_FALSE(status.ok()); + + json_params = {{"search_length", 50}}; + status = + milvus::server::ValidationUtil::ValidateIndexParams(json_params, + table_schema, + (int32_t)milvus::engine::EngineType::NSG_MIX); + ASSERT_FALSE(status.ok()); + + json_params = {{"search_length", 50}, {"out_degree", -1}}; + status = + milvus::server::ValidationUtil::ValidateIndexParams(json_params, + table_schema, + (int32_t)milvus::engine::EngineType::NSG_MIX); + ASSERT_FALSE(status.ok()); + + json_params = {{"search_length", 50}, {"out_degree", 50}}; + status = + milvus::server::ValidationUtil::ValidateIndexParams(json_params, + table_schema, + (int32_t)milvus::engine::EngineType::NSG_MIX); + ASSERT_FALSE(status.ok()); + + json_params = {{"search_length", 50}, {"out_degree", 50}, {"candidate_pool_size", -1}}; + status = + milvus::server::ValidationUtil::ValidateIndexParams(json_params, + table_schema, + (int32_t)milvus::engine::EngineType::NSG_MIX); + ASSERT_FALSE(status.ok()); + + json_params = {{"search_length", 50}, {"out_degree", 50}, {"candidate_pool_size", 100}}; + status = + milvus::server::ValidationUtil::ValidateIndexParams(json_params, + table_schema, + (int32_t)milvus::engine::EngineType::NSG_MIX); + ASSERT_FALSE(status.ok()); + + json_params = {{"search_length", 50}, {"out_degree", 50}, {"candidate_pool_size", 100}, {"knng", -1}}; + status = + milvus::server::ValidationUtil::ValidateIndexParams(json_params, + table_schema, + (int32_t)milvus::engine::EngineType::NSG_MIX); + ASSERT_FALSE(status.ok()); + + json_params = {{"search_length", 50}, {"out_degree", 50}, {"candidate_pool_size", 100}, {"knng", 100}}; + status = + milvus::server::ValidationUtil::ValidateIndexParams(json_params, + table_schema, + (int32_t)milvus::engine::EngineType::NSG_MIX); + ASSERT_TRUE(status.ok()); +} + +TEST(ValidationUtilTest, VALIDATE_SEARCH_PARAMS_TEST) { + int64_t topk = 10; + milvus::engine::meta::TableSchema table_schema; + table_schema.dimension_ = 64; + + milvus::json json_params = {}; + table_schema.engine_type_ = (int32_t)milvus::engine::EngineType::FAISS_IDMAP; + auto status = milvus::server::ValidationUtil::ValidateSearchParams(json_params, table_schema, topk); + ASSERT_TRUE(status.ok()); + + table_schema.engine_type_ = (int32_t)milvus::engine::EngineType::FAISS_IVFFLAT; + status = milvus::server::ValidationUtil::ValidateSearchParams(json_params, table_schema, topk); + ASSERT_FALSE(status.ok()); + + json_params = {{"nprobe", "\t"}}; + status = milvus::server::ValidationUtil::ValidateSearchParams(json_params, table_schema, topk); + ASSERT_FALSE(status.ok()); + + table_schema.engine_type_ = (int32_t)milvus::engine::EngineType::FAISS_BIN_IDMAP; + json_params = {{"nprobe", 32}}; + status = milvus::server::ValidationUtil::ValidateSearchParams(json_params, table_schema, topk); + ASSERT_TRUE(status.ok()); + + table_schema.engine_type_ = (int32_t)milvus::engine::EngineType::NSG_MIX; + json_params = {}; + status = milvus::server::ValidationUtil::ValidateSearchParams(json_params, table_schema, topk); + ASSERT_FALSE(status.ok()); + + json_params = {{"search_length", 100}}; + status = milvus::server::ValidationUtil::ValidateSearchParams(json_params, table_schema, topk); + ASSERT_TRUE(status.ok()); + + table_schema.engine_type_ = (int32_t)milvus::engine::EngineType::HNSW; + json_params = {}; + status = milvus::server::ValidationUtil::ValidateSearchParams(json_params, table_schema, topk); + ASSERT_FALSE(status.ok()); + + json_params = {{"ef", 5}}; + status = milvus::server::ValidationUtil::ValidateSearchParams(json_params, table_schema, topk); + ASSERT_FALSE(status.ok()); + + json_params = {{"ef", 100}}; + status = milvus::server::ValidationUtil::ValidateSearchParams(json_params, table_schema, topk); + ASSERT_TRUE(status.ok()); +} + TEST(ValidationUtilTest, VALIDATE_TOPK_TEST) { milvus::engine::meta::TableSchema schema; ASSERT_EQ(milvus::server::ValidationUtil::ValidateSearchTopk(10, schema).code(), milvus::SERVER_SUCCESS); @@ -405,14 +565,6 @@ TEST(ValidationUtilTest, VALIDATE_TOPK_TEST) { ASSERT_NE(milvus::server::ValidationUtil::ValidateSearchTopk(0, schema).code(), milvus::SERVER_SUCCESS); } -TEST(ValidationUtilTest, VALIDATE_NPROBE_TEST) { - milvus::engine::meta::TableSchema schema; - schema.nlist_ = 100; - ASSERT_EQ(milvus::server::ValidationUtil::ValidateSearchNprobe(10, schema).code(), milvus::SERVER_SUCCESS); - ASSERT_NE(milvus::server::ValidationUtil::ValidateSearchNprobe(0, schema).code(), milvus::SERVER_SUCCESS); - ASSERT_NE(milvus::server::ValidationUtil::ValidateSearchNprobe(101, schema).code(), milvus::SERVER_SUCCESS); -} - TEST(ValidationUtilTest, VALIDATE_PARTITION_TAGS) { std::vector partition_tags = {"abc"}; ASSERT_EQ(milvus::server::ValidationUtil::ValidatePartitionTags(partition_tags).code(), milvus::SERVER_SUCCESS); diff --git a/core/unittest/server/test_web.cpp b/core/unittest/server/test_web.cpp index 7cf5d8ebfa..073e08ec91 100644 --- a/core/unittest/server/test_web.cpp +++ b/core/unittest/server/test_web.cpp @@ -41,6 +41,7 @@ #include "server/web_impl/handler/WebRequestHandler.h" #include "unittest/server/utils.h" #include "utils/CommonUtil.h" +#include "version.h" #include "wrapper/VecIndex.h" @@ -237,14 +238,14 @@ class WebHandlerTest : public testing::Test { protected: void - GenTable(const std::string& table_name, int64_t dim, int64_t index_size, const std::string& metric) { - auto table_dto = milvus::server::web::TableRequestDto::createShared(); - table_dto->table_name = table_name.c_str(); - table_dto->dimension = dim; - table_dto->index_file_size = index_size; - table_dto->metric_type = metric.c_str(); + GenTable(const std::string& collection_name, int64_t dim, int64_t index_size, const std::string& metric) { + auto collection_dto = milvus::server::web::TableRequestDto::createShared(); + collection_dto->collection_name = collection_name.c_str(); + collection_dto->dimension = dim; + collection_dto->index_file_size = index_size; + collection_dto->metric_type = metric.c_str(); - auto status_dto = handler->CreateTable(table_dto); + handler->CreateTable(collection_dto); } protected: @@ -256,66 +257,66 @@ class WebHandlerTest : public testing::Test { TEST_F(WebHandlerTest, TABLE) { handler->RegisterRequestHandler(milvus::server::RequestHandler()); - auto table_name = milvus::server::web::OString(TABLE_NAME) + RandomName().c_str(); + auto collection_name = milvus::server::web::OString(TABLE_NAME) + RandomName().c_str(); - auto table_dto = milvus::server::web::TableRequestDto::createShared(); - table_dto->table_name = table_name; - table_dto->dimension = TABLE_DIM + 100000; - table_dto->index_file_size = INDEX_FILE_SIZE; - table_dto->metric_type = "L2"; + auto collection_dto = milvus::server::web::TableRequestDto::createShared(); + collection_dto->collection_name = collection_name; + collection_dto->dimension = TABLE_DIM + 100000; + collection_dto->index_file_size = INDEX_FILE_SIZE; + collection_dto->metric_type = "L2"; // invalid dimension - auto status_dto = handler->CreateTable(table_dto); + auto status_dto = handler->CreateTable(collection_dto); ASSERT_EQ(StatusCode::ILLEGAL_DIMENSION, status_dto->code->getValue()); // invalid index file size - table_dto->dimension = TABLE_DIM; - table_dto->index_file_size = -1; - status_dto = handler->CreateTable(table_dto); + collection_dto->dimension = TABLE_DIM; + collection_dto->index_file_size = -1; + status_dto = handler->CreateTable(collection_dto); ASSERT_EQ(StatusCode::ILLEGAL_ARGUMENT, status_dto->code->getValue()); // invalid metric type - table_dto->index_file_size = INDEX_FILE_SIZE; - table_dto->metric_type = "L1"; - status_dto = handler->CreateTable(table_dto); + collection_dto->index_file_size = INDEX_FILE_SIZE; + collection_dto->metric_type = "L1"; + status_dto = handler->CreateTable(collection_dto); ASSERT_EQ(StatusCode::ILLEGAL_METRIC_TYPE, status_dto->code->getValue()); - // create table successfully - table_dto->metric_type = "L2"; - status_dto = handler->CreateTable(table_dto); + // create collection successfully + collection_dto->metric_type = "L2"; + status_dto = handler->CreateTable(collection_dto); ASSERT_EQ(0, status_dto->code->getValue()); sleep(3); - status_dto = handler->DropTable(table_name); + status_dto = handler->DropTable(collection_name); ASSERT_EQ(0, status_dto->code->getValue()); - // drop table which not exists. - status_dto = handler->DropTable(table_name + "57575yfhfdhfhdh436gdsgpppdgsgv3233"); + // drop collection which not exists. + status_dto = handler->DropTable(collection_name + "57575yfhfdhfhdh436gdsgpppdgsgv3233"); ASSERT_EQ(StatusCode::TABLE_NOT_EXISTS, status_dto->code->getValue()); } TEST_F(WebHandlerTest, HAS_TABLE_TEST) { handler->RegisterRequestHandler(milvus::server::RequestHandler()); - auto table_name = milvus::server::web::OString(TABLE_NAME) + RandomName().c_str(); + auto collection_name = milvus::server::web::OString(TABLE_NAME) + RandomName().c_str(); - GenTable(table_name->std_str(), 10, 10, "L2"); + GenTable(collection_name->std_str(), 10, 10, "L2"); milvus::server::web::OQueryParams query_params; OString response; - auto status_dto = handler->GetTable(table_name, query_params, response); + auto status_dto = handler->GetTable(collection_name, query_params, response); ASSERT_EQ(0, status_dto->code->getValue()); } TEST_F(WebHandlerTest, GET_TABLE) { handler->RegisterRequestHandler(milvus::server::RequestHandler()); - auto table_name = milvus::server::web::OString(TABLE_NAME) + RandomName().c_str(); - GenTable(table_name->std_str(), 10, 10, "L2"); + auto collection_name = milvus::server::web::OString(TABLE_NAME) + RandomName().c_str(); + GenTable(collection_name->std_str(), 10, 10, "L2"); milvus::server::web::OQueryParams query_params; OString result; - auto status_dto = handler->GetTable(table_name, query_params, result); + auto status_dto = handler->GetTable(collection_name, query_params, result); ASSERT_EQ(0, status_dto->code->getValue()); auto result_json = nlohmann::json::parse(result->std_str()); @@ -327,13 +328,13 @@ TEST_F(WebHandlerTest, GET_TABLE) { TEST_F(WebHandlerTest, INSERT_COUNT) { handler->RegisterRequestHandler(milvus::server::RequestHandler()); - auto table_name = milvus::server::web::OString(TABLE_NAME) + RandomName().c_str(); - GenTable(table_name->std_str(), 16, 10, "L2"); + auto collection_name = milvus::server::web::OString(TABLE_NAME) + RandomName().c_str(); + GenTable(collection_name->std_str(), 16, 10, "L2"); nlohmann::json body_json; body_json["vectors"] = RandomRecordsJson(16, 1000); auto ids_dto = milvus::server::web::VectorIdsDto::createShared(); - auto status_dto = handler->Insert(table_name, body_json.dump().c_str(), ids_dto); + auto status_dto = handler->Insert(collection_name, body_json.dump().c_str(), ids_dto); ASSERT_EQ(0, status_dto->code->getValue()); ASSERT_EQ(1000, ids_dto->ids->count()); @@ -342,7 +343,7 @@ TEST_F(WebHandlerTest, INSERT_COUNT) { milvus::server::web::OQueryParams query_params; query_params.put("fields", "num"); OString result; - status_dto = handler->GetTable(table_name, query_params, result); + status_dto = handler->GetTable(collection_name, query_params, result); ASSERT_EQ(0, status_dto->code->getValue()); auto result_json = nlohmann::json::parse(result->std_str()); @@ -352,89 +353,92 @@ TEST_F(WebHandlerTest, INSERT_COUNT) { TEST_F(WebHandlerTest, INDEX) { handler->RegisterRequestHandler(milvus::server::RequestHandler()); - auto table_name = milvus::server::web::OString(TABLE_NAME) + RandomName().c_str(); - GenTable(table_name->std_str(), 16, 10, "L2"); + auto collection_name = milvus::server::web::OString(TABLE_NAME) + RandomName().c_str(); + GenTable(collection_name->std_str(), 16, 10, "L2"); - auto index_request_dto = milvus::server::web::IndexRequestDto::createShared(); - index_request_dto->index_type = "FLAT"; - index_request_dto->nlist = 10; + nlohmann::json index_json; - milvus::server::web::StatusDto::createShared(); + index_json["index_type"] = "FLAT"; + index_json["params"] = nlohmann::json::parse("{ \"nlist\": 10 }"); - auto status_dto = handler->CreateIndex(table_name, index_request_dto); + auto status_dto = handler->CreateIndex(collection_name, index_json.dump().c_str()); ASSERT_EQ(0, status_dto->code->getValue()); - status_dto = handler->DropIndex(table_name); + status_dto = handler->DropIndex(collection_name); + ASSERT_EQ(0, status_dto->code->getValue()); + + // drop index + status_dto = handler->DropIndex(collection_name); ASSERT_EQ(0, status_dto->code->getValue()); // invalid index_type - index_request_dto->index_type = "AAA"; - status_dto = handler->CreateIndex(table_name, index_request_dto); + index_json["index_type"] = "AAA"; + status_dto = handler->CreateIndex(collection_name, index_json.dump().c_str()); ASSERT_NE(0, status_dto->code->getValue()); ASSERT_EQ(StatusCode::ILLEGAL_INDEX_TYPE, status_dto->code->getValue()); // invalid nlist - index_request_dto->index_type = "FLAT"; - index_request_dto->nlist = -1; - status_dto = handler->CreateIndex(table_name, index_request_dto); + index_json["index_type"] = "IVFFLAT"; + index_json["params"] = nlohmann::json::parse("{ \"nlist\": -1 }"); + status_dto = handler->CreateIndex(collection_name, index_json.dump().c_str()); ASSERT_NE(0, status_dto->code->getValue()); - ASSERT_EQ(StatusCode::ILLEGAL_NLIST, status_dto->code->getValue()); + ASSERT_EQ(StatusCode::ILLEGAL_ARGUMENT, status_dto->code->getValue()); } TEST_F(WebHandlerTest, PARTITION) { handler->RegisterRequestHandler(milvus::server::RequestHandler()); - auto table_name = milvus::server::web::OString(TABLE_NAME) + RandomName().c_str(); - GenTable(table_name->std_str(), 16, 10, "L2"); + auto collection_name = milvus::server::web::OString(TABLE_NAME) + RandomName().c_str(); + GenTable(collection_name->std_str(), 16, 10, "L2"); auto partition_dto = milvus::server::web::PartitionRequestDto::createShared(); partition_dto->partition_tag = "test"; - auto status_dto = handler->CreatePartition(table_name, partition_dto); + auto status_dto = handler->CreatePartition(collection_name, partition_dto); ASSERT_EQ(0, status_dto->code->getValue()); auto partitions_dto = milvus::server::web::PartitionListDto::createShared(); OQueryParams query_params; query_params.put("offset", "0"); query_params.put("page_size", "10"); - status_dto = handler->ShowPartitions(table_name, query_params, partitions_dto); + status_dto = handler->ShowPartitions(collection_name, query_params, partitions_dto); ASSERT_EQ(milvus::server::web::SUCCESS, status_dto->code->getValue()); ASSERT_EQ(2, partitions_dto->partitions->count()); - status_dto = handler->DropPartition(table_name, "{\"partition_tag\": \"test\"}"); + status_dto = handler->DropPartition(collection_name, "{\"partition_tag\": \"test\"}"); ASSERT_EQ(0, status_dto->code->getValue()); // Show all partitions - status_dto = handler->ShowPartitions(table_name, query_params, partitions_dto); + status_dto = handler->ShowPartitions(collection_name, query_params, partitions_dto); ASSERT_EQ(milvus::server::web::SUCCESS, status_dto->code->getValue()); query_params.put("all_required", "true"); - status_dto = handler->ShowPartitions(table_name, query_params, partitions_dto); + status_dto = handler->ShowPartitions(collection_name, query_params, partitions_dto); ASSERT_EQ(milvus::server::web::SUCCESS, status_dto->code->getValue()); } TEST_F(WebHandlerTest, SEARCH) { handler->RegisterRequestHandler(milvus::server::RequestHandler()); - auto table_name = milvus::server::web::OString(TABLE_NAME) + RandomName().c_str(); - GenTable(table_name->std_str(), TABLE_DIM, 10, "L2"); + auto collection_name = milvus::server::web::OString(TABLE_NAME) + RandomName().c_str(); + GenTable(collection_name->std_str(), TABLE_DIM, 10, "L2"); nlohmann::json insert_json; insert_json["vectors"] = RandomRecordsJson(TABLE_DIM, 1000); auto ids_dto = milvus::server::web::VectorIdsDto::createShared(); - auto status_dto = handler->Insert(table_name, insert_json.dump().c_str(), ids_dto); + auto status_dto = handler->Insert(collection_name, insert_json.dump().c_str(), ids_dto); ASSERT_EQ(milvus::server::web::SUCCESS, status_dto->code->getValue()); nlohmann::json search_pram_json; search_pram_json["vectors"] = RandomRecordsJson(TABLE_DIM, 10); search_pram_json["topk"] = 1; - search_pram_json["nprobe"] = 1; + search_pram_json["params"] = nlohmann::json::parse("{\"nprobe\": 10}"); nlohmann::json search_json; search_json["search"] = search_pram_json; OString result = ""; - status_dto = handler->VectorsOp(table_name, search_json.dump().c_str(), result); + status_dto = handler->VectorsOp(collection_name, search_json.dump().c_str(), result); ASSERT_EQ(0, status_dto->code->getValue()) << status_dto->message->std_str(); } @@ -446,27 +450,28 @@ TEST_F(WebHandlerTest, SYSTEM_INFO) { auto status_dto = handler->SystemInfo("status", query_params, result); ASSERT_EQ(0, status_dto->code->getValue()); -// ASSERT_EQ("OK", cmd_dto->reply->std_str()); status_dto = handler->SystemInfo("version", query_params, result); ASSERT_EQ(0, status_dto->code->getValue()); -// ASSERT_EQ("0.7.0", cmd_dto->reply->std_str()); + auto result_json = nlohmann::json::parse(result->c_str()); + ASSERT_TRUE(result_json.contains("reply")); + ASSERT_EQ(MILVUS_VERSION, result_json["reply"].get()); } TEST_F(WebHandlerTest, FLUSH) { handler->RegisterRequestHandler(milvus::server::RequestHandler()); - auto table_name = milvus::server::web::OString(TABLE_NAME) + RandomName().c_str(); - GenTable(table_name->std_str(), 16, 10, "L2"); + auto collection_name = milvus::server::web::OString(TABLE_NAME) + RandomName().c_str(); + GenTable(collection_name->std_str(), 16, 10, "L2"); nlohmann::json body_json; body_json["vectors"] = RandomRecordsJson(16, 1000); auto ids_dto = milvus::server::web::VectorIdsDto::createShared(); - auto status_dto = handler->Insert(table_name, body_json.dump().c_str(), ids_dto); + auto status_dto = handler->Insert(collection_name, body_json.dump().c_str(), ids_dto); ASSERT_EQ(0, status_dto->code->getValue()) << status_dto->message->std_str(); nlohmann::json flush_json; - flush_json["flush"]["table_names"] = {table_name->std_str()}; + flush_json["flush"]["collection_names"] = {collection_name->std_str()}; OString result; status_dto = handler->SystemOp("task", flush_json.dump().c_str(), result); ASSERT_EQ(milvus::server::web::SUCCESS, status_dto->code->getValue()); @@ -475,17 +480,17 @@ TEST_F(WebHandlerTest, FLUSH) { TEST_F(WebHandlerTest, COMPACT) { handler->RegisterRequestHandler(milvus::server::RequestHandler()); - auto table_name = milvus::server::web::OString(TABLE_NAME) + RandomName().c_str(); - GenTable(table_name->std_str(), 16, 10, "L2"); + auto collection_name = milvus::server::web::OString(TABLE_NAME) + RandomName().c_str(); + GenTable(collection_name->std_str(), 16, 10, "L2"); nlohmann::json body_json; body_json["vectors"] = RandomRecordsJson(16, 1000); auto ids_dto = milvus::server::web::VectorIdsDto::createShared(); - auto status_dto = handler->Insert(table_name, body_json.dump().c_str(), ids_dto); + auto status_dto = handler->Insert(collection_name, body_json.dump().c_str(), ids_dto); ASSERT_EQ(0, status_dto->code->getValue()) << status_dto->message->std_str(); nlohmann::json compact_json; - compact_json["compact"]["table_name"] = table_name->std_str(); + compact_json["compact"]["collection_name"] = collection_name->std_str(); OString result; status_dto = handler->SystemOp("task", compact_json.dump().c_str(), result); ASSERT_EQ(milvus::server::web::SUCCESS, status_dto->code->getValue()); @@ -627,55 +632,55 @@ class TestClient : public oatpp::web::client::ApiClient { BODY_DTO(milvus::server::web::GPUConfigDto::ObjectWrapper, body)) #endif - API_CALL("OPTIONS", "/tables", optionsTables) + API_CALL("OPTIONS", "/collections", optionsTables) - API_CALL("POST", "/tables", createTable, BODY_DTO(milvus::server::web::TableRequestDto::ObjectWrapper, body)) + API_CALL("POST", "/collections", createTable, BODY_DTO(milvus::server::web::TableRequestDto::ObjectWrapper, body)) - API_CALL("GET", "/tables", showTables, QUERY(String, offset), QUERY(String, page_size)) + API_CALL("GET", "/collections", showTables, QUERY(String, offset), QUERY(String, page_size)) - API_CALL("OPTIONS", "/tables/{table_name}", optionsTable, PATH(String, table_name, "table_name")) + API_CALL("OPTIONS", "/collections/{collection_name}", optionsTable, PATH(String, collection_name, "collection_name")) - API_CALL("GET", "/tables/{table_name}", getTable, PATH(String, table_name, "table_name"), QUERY(String, info)) + API_CALL("GET", "/collections/{collection_name}", getTable, PATH(String, collection_name, "collection_name"), QUERY(String, info)) - API_CALL("DELETE", "/tables/{table_name}", dropTable, PATH(String, table_name, "table_name")) + API_CALL("DELETE", "/collections/{collection_name}", dropTable, PATH(String, collection_name, "collection_name")) - API_CALL("OPTIONS", "/tables/{table_name}/indexes", optionsIndexes, PATH(String, table_name, "table_name")) + API_CALL("OPTIONS", "/collections/{collection_name}/indexes", optionsIndexes, PATH(String, collection_name, "collection_name")) API_CALL("POST", "/tables/{table_name}/indexes", createIndex, PATH(String, table_name, "table_name"), - BODY_DTO(milvus::server::web::IndexRequestDto::ObjectWrapper, body)) + BODY_STRING(OString, body)) - API_CALL("GET", "/tables/{table_name}/indexes", getIndex, PATH(String, table_name, "table_name")) + API_CALL("GET", "/collections/{collection_name}/indexes", getIndex, PATH(String, collection_name, "collection_name")) - API_CALL("DELETE", "/tables/{table_name}/indexes", dropIndex, PATH(String, table_name, "table_name")) + API_CALL("DELETE", "/collections/{collection_name}/indexes", dropIndex, PATH(String, collection_name, "collection_name")) - API_CALL("OPTIONS", "/tables/{table_name}/partitions", optionsPartitions, PATH(String, table_name, "table_name")) + API_CALL("OPTIONS", "/collections/{collection_name}/partitions", optionsPartitions, PATH(String, collection_name, "collection_name")) - API_CALL("POST", "/tables/{table_name}/partitions", createPartition, PATH(String, table_name, "table_name"), + API_CALL("POST", "/collections/{collection_name}/partitions", createPartition, PATH(String, collection_name, "collection_name"), BODY_DTO(milvus::server::web::PartitionRequestDto::ObjectWrapper, body)) - API_CALL("GET", "/tables/{table_name}/partitions", showPartitions, PATH(String, table_name, "table_name"), + API_CALL("GET", "/collections/{collection_name}/partitions", showPartitions, PATH(String, collection_name, "collection_name"), QUERY(String, offset), QUERY(String, page_size)) - API_CALL("DELETE", "/tables/{table_name}/partitions", dropPartition, - PATH(String, table_name, "table_name"), BODY_STRING(String, body)) + API_CALL("DELETE", "/collections/{collection_name}/partitions", dropPartition, + PATH(String, collection_name, "collection_name"), BODY_STRING(String, body)) - API_CALL("GET", "/tables/{table_name}/segments", showSegments, PATH(String, table_name, "table_name"), + API_CALL("GET", "/collections/{collection_name}/segments", showSegments, PATH(String, collection_name, "collection_name"), QUERY(String, offset), QUERY(String, page_size), QUERY(String, partition_tag)) - API_CALL("GET", "/tables/{table_name}/segments/{segment_name}/{info}", getSegmentInfo, - PATH(String, table_name, "table_name"), PATH(String, segment_name, "segment_name"), + API_CALL("GET", "/collections/{collection_name}/segments/{segment_name}/{info}", getSegmentInfo, + PATH(String, collection_name, "collection_name"), PATH(String, segment_name, "segment_name"), PATH(String, info, "info"), QUERY(String, offset), QUERY(String, page_size)) - API_CALL("OPTIONS", "/tables/{table_name}/vectors", optionsVectors, PATH(String, table_name, "table_name")) + API_CALL("OPTIONS", "/collections/{collection_name}/vectors", optionsVectors, PATH(String, collection_name, "collection_name")) - API_CALL("GET", "/tables/{table_name}/vectors", getVectors, - PATH(String, table_name, "table_name"), QUERY(String, id)) + API_CALL("GET", "/collections/{collection_name}/vectors", getVectors, + PATH(String, collection_name, "collection_name"), QUERY(String, id)) - API_CALL("POST", "/tables/{table_name}/vectors", insert, - PATH(String, table_name, "table_name"), BODY_STRING(String, body)) + API_CALL("POST", "/collections/{collection_name}/vectors", insert, + PATH(String, collection_name, "collection_name"), BODY_STRING(String, body)) - API_CALL("PUT", "/tables/{table_name}/vectors", vectorsOp, - PATH(String, table_name, "table_name"), BODY_STRING(String, body)) + API_CALL("PUT", "/collections/{collection_name}/vectors", vectorsOp, + PATH(String, collection_name, "collection_name"), BODY_STRING(String, body)) API_CALL("GET", "/system/{msg}", cmd, PATH(String, cmd_str, "msg"), QUERY(String, action), QUERY(String, target)) @@ -740,23 +745,23 @@ class WebControllerTest : public testing::Test { } void - GenTable(const OString& table_name, int64_t dim, int64_t index_size, const OString& metric) { - auto response = client_ptr->getTable(table_name, "", conncetion_ptr); + GenTable(const OString& collection_name, int64_t dim, int64_t index_size, const OString& metric) { + auto response = client_ptr->getTable(collection_name, "", conncetion_ptr); if (OStatus::CODE_200.code == response->getStatusCode()) { return; } - auto table_dto = milvus::server::web::TableRequestDto::createShared(); - table_dto->table_name = table_name; - table_dto->dimension = dim; - table_dto->index_file_size = index_size; - table_dto->metric_type = metric; - client_ptr->createTable(table_dto, conncetion_ptr); + auto collection_dto = milvus::server::web::TableRequestDto::createShared(); + collection_dto->collection_name = collection_name; + collection_dto->dimension = dim; + collection_dto->index_file_size = index_size; + collection_dto->metric_type = metric; + client_ptr->createTable(collection_dto, conncetion_ptr); } milvus::Status - FlushTable(const std::string& table_name) { + FlushTable(const std::string& collection_name) { nlohmann::json flush_json; - flush_json["flush"]["table_names"] = {table_name}; + flush_json["flush"]["collection_names"] = {collection_name}; auto response = client_ptr->op("task", flush_json.dump().c_str(), conncetion_ptr); if (OStatus::CODE_200.code != response->getStatusCode()) { return milvus::Status(milvus::SERVER_UNEXPECTED_ERROR, response->readBodyToString()->std_str()); @@ -766,9 +771,9 @@ class WebControllerTest : public testing::Test { } milvus::Status - FlushTable(const OString& table_name) { + FlushTable(const OString& collection_name) { nlohmann::json flush_json; - flush_json["flush"]["table_names"] = {table_name->std_str()}; + flush_json["flush"]["collection_names"] = {collection_name->std_str()}; auto response = client_ptr->op("task", flush_json.dump().c_str(), conncetion_ptr); if (OStatus::CODE_200.code != response->getStatusCode()) { return milvus::Status(milvus::SERVER_UNEXPECTED_ERROR, response->readBodyToString()->std_str()); @@ -778,7 +783,7 @@ class WebControllerTest : public testing::Test { } milvus::Status - InsertData(const OString& table_name, int64_t dim, int64_t count, std::string tag = "", bool bin = false) { + InsertData(const OString& collection_name, int64_t dim, int64_t count, std::string tag = "", bool bin = false) { nlohmann::json insert_json; if (bin) @@ -790,16 +795,16 @@ class WebControllerTest : public testing::Test { insert_json["partition_tag"] = tag; } - auto response = client_ptr->insert(table_name, insert_json.dump().c_str(), conncetion_ptr); + auto response = client_ptr->insert(collection_name, insert_json.dump().c_str(), conncetion_ptr); if (OStatus::CODE_201.code != response->getStatusCode()) { return milvus::Status(milvus::SERVER_UNEXPECTED_ERROR, response->readBodyToString()->c_str()); } - return FlushTable(table_name); + return FlushTable(collection_name); } milvus::Status - InsertData(const OString& table_name, int64_t dim, int64_t count, + InsertData(const OString& collection_name, int64_t dim, int64_t count, const std::vector& ids, std::string tag = "", bool bin = false) { nlohmann::json insert_json; @@ -816,19 +821,19 @@ class WebControllerTest : public testing::Test { insert_json["partition_tag"] = tag; } - auto response = client_ptr->insert(table_name, insert_json.dump().c_str(), conncetion_ptr); + auto response = client_ptr->insert(collection_name, insert_json.dump().c_str(), conncetion_ptr); if (OStatus::CODE_201.code != response->getStatusCode()) { return milvus::Status(milvus::SERVER_UNEXPECTED_ERROR, response->readBodyToString()->c_str()); } - return FlushTable(table_name); + return FlushTable(collection_name); } milvus::Status - GenPartition(const OString& table_name, const OString& tag) { + GenPartition(const OString& collection_name, const OString& tag) { auto par_param = milvus::server::web::PartitionRequestDto::createShared(); par_param->partition_tag = tag; - auto response = client_ptr->createPartition(table_name, par_param); + auto response = client_ptr->createPartition(collection_name, par_param); if (OStatus::CODE_201.code != response->getStatusCode()) { return milvus::Status(milvus::SERVER_UNEXPECTED_ERROR, response->readBodyToString()->c_str()); } @@ -866,14 +871,14 @@ class WebControllerTest : public testing::Test { protected: void - GenTable(const std::string& table_name, int64_t dim, int64_t index_file_size, int64_t metric_type) { - auto table_dto = milvus::server::web::TableRequestDto::createShared(); - table_dto->table_name = OString(table_name.c_str()); - table_dto->dimension = dim; - table_dto->index_file_size = index_file_size; - table_dto->metric_type = metric_type; + GenTable(const std::string& collection_name, int64_t dim, int64_t index_file_size, int64_t metric_type) { + auto collection_dto = milvus::server::web::TableRequestDto::createShared(); + collection_dto->collection_name = OString(collection_name.c_str()); + collection_dto->dimension = dim; + collection_dto->index_file_size = index_file_size; + collection_dto->metric_type = metric_type; - client_ptr->createTable(table_dto, conncetion_ptr); + client_ptr->createTable(collection_dto, conncetion_ptr); } }; @@ -896,85 +901,85 @@ TEST_F(WebControllerTest, OPTIONS) { response = client_ptr->optionsIndexes("test", conncetion_ptr); ASSERT_EQ(OStatus::CODE_204.code, response->getStatusCode()); - response = client_ptr->optionsPartitions("table_name", conncetion_ptr); + response = client_ptr->optionsPartitions("collection_name", conncetion_ptr); ASSERT_EQ(OStatus::CODE_204.code, response->getStatusCode()); - response = client_ptr->optionsTable("table", conncetion_ptr); + response = client_ptr->optionsTable("collection", conncetion_ptr); ASSERT_EQ(OStatus::CODE_204.code, response->getStatusCode()); response = client_ptr->optionsTables(conncetion_ptr); ASSERT_EQ(OStatus::CODE_204.code, response->getStatusCode()); - response = client_ptr->optionsVectors("table", conncetion_ptr); + response = client_ptr->optionsVectors("collection", conncetion_ptr); ASSERT_EQ(OStatus::CODE_204.code, response->getStatusCode()); } TEST_F(WebControllerTest, CREATE_TABLE) { - auto table_dto = milvus::server::web::TableRequestDto::createShared(); - auto response = client_ptr->createTable(table_dto, conncetion_ptr); + auto collection_dto = milvus::server::web::TableRequestDto::createShared(); + auto response = client_ptr->createTable(collection_dto, conncetion_ptr); ASSERT_EQ(OStatus::CODE_400.code, response->getStatusCode()); auto error_dto = response->readBodyToDto(object_mapper.get()); ASSERT_EQ(milvus::server::web::StatusCode::BODY_FIELD_LOSS, error_dto->code) << error_dto->message->std_str(); - OString table_name = "web_test_create_table" + OString(RandomName().c_str()); + OString collection_name = "web_test_create_collection" + OString(RandomName().c_str()); - table_dto->table_name = table_name; - response = client_ptr->createTable(table_dto, conncetion_ptr); + collection_dto->collection_name = collection_name; + response = client_ptr->createTable(collection_dto, conncetion_ptr); ASSERT_EQ(OStatus::CODE_400.code, response->getStatusCode()); error_dto = response->readBodyToDto(object_mapper.get()); ASSERT_EQ(milvus::server::web::StatusCode::BODY_FIELD_LOSS, error_dto->code) << error_dto->message->std_str(); - table_dto->dimension = 128; - table_dto->index_file_size = 10; - table_dto->metric_type = "L2"; + collection_dto->dimension = 128; + collection_dto->index_file_size = 10; + collection_dto->metric_type = "L2"; - response = client_ptr->createTable(table_dto, conncetion_ptr); + response = client_ptr->createTable(collection_dto, conncetion_ptr); ASSERT_EQ(OStatus::CODE_201.code, response->getStatusCode()); auto result_dto = response->readBodyToDto(object_mapper.get()); ASSERT_EQ(milvus::server::web::StatusCode::SUCCESS, result_dto->code->getValue()) << result_dto->message->std_str(); - // invalid table name - table_dto->table_name = "9090&*&()"; - response = client_ptr->createTable(table_dto, conncetion_ptr); + // invalid collection name + collection_dto->collection_name = "9090&*&()"; + response = client_ptr->createTable(collection_dto, conncetion_ptr); ASSERT_EQ(OStatus::CODE_400.code, response->getStatusCode()); } TEST_F(WebControllerTest, GET_TABLE_META) { - OString table_name = "web_test_create_table" + OString(RandomName().c_str()); - GenTable(table_name, 10, 10, "L2"); + OString collection_name = "web_test_create_collection" + OString(RandomName().c_str()); + GenTable(collection_name, 10, 10, "L2"); OQueryParams params; - auto response = client_ptr->getTable(table_name, "", conncetion_ptr); + auto response = client_ptr->getTable(collection_name, "", conncetion_ptr); ASSERT_EQ(OStatus::CODE_200.code, response->getStatusCode()); auto result_dto = response->readBodyToDto(object_mapper.get()); - ASSERT_EQ(table_name->std_str(), result_dto->table_name->std_str()); + ASSERT_EQ(collection_name->std_str(), result_dto->collection_name->std_str()); ASSERT_EQ(10, result_dto->dimension); ASSERT_EQ("L2", result_dto->metric_type->std_str()); ASSERT_EQ(10, result_dto->index_file_size->getValue()); ASSERT_EQ("FLAT", result_dto->index->std_str()); - // invalid table name - table_name = "57474dgdfhdfhdh dgd"; - response = client_ptr->getTable(table_name, "", conncetion_ptr); + // invalid collection name + collection_name = "57474dgdfhdfhdh dgd"; + response = client_ptr->getTable(collection_name, "", conncetion_ptr); ASSERT_EQ(OStatus::CODE_400.code, response->getStatusCode()); auto status_sto = response->readBodyToDto(object_mapper.get()); ASSERT_EQ(milvus::server::web::StatusCode::ILLEGAL_TABLE_NAME, status_sto->code->getValue()); - table_name = "test_table_not_found_000000000111010101002020203020aaaaa3030435"; - response = client_ptr->getTable(table_name, "", conncetion_ptr); + collection_name = "test_collection_not_found_000000000111010101002020203020aaaaa3030435"; + response = client_ptr->getTable(collection_name, "", conncetion_ptr); ASSERT_EQ(OStatus::CODE_404.code, response->getStatusCode()); } TEST_F(WebControllerTest, GET_TABLE_STAT) { - OString table_name = "web_test_get_table_stat" + OString(RandomName().c_str()); - GenTable(table_name, 128, 5, "L2"); + OString collection_name = "web_test_get_collection_stat" + OString(RandomName().c_str()); + GenTable(collection_name, 128, 5, "L2"); for (size_t i = 0; i < 5; i++) { - InsertData(table_name, 128, 1000); + InsertData(collection_name, 128, 1000); } - auto response = client_ptr->getTable(table_name, "stat", conncetion_ptr); + auto response = client_ptr->getTable(collection_name, "stat", conncetion_ptr); ASSERT_EQ(OStatus::CODE_200.code, response->getStatusCode()); auto result_json = nlohmann::json::parse(response->readBodyToString()->c_str()); ASSERT_TRUE(result_json.contains("count")); @@ -998,13 +1003,13 @@ TEST_F(WebControllerTest, GET_TABLE_STAT) { } TEST_F(WebControllerTest, SHOW_TABLES) { - // test query table limit 1 + // test query collection limit 1 auto response = client_ptr->showTables("1", "1", conncetion_ptr); ASSERT_EQ(OStatus::CODE_200.code, response->getStatusCode()); auto result_dto = response->readBodyToDto(object_mapper.get()); ASSERT_GE(result_dto->count->getValue(), 0); - // test query table empty + // test query collection empty response = client_ptr->showTables("0", "0", conncetion_ptr); ASSERT_EQ(OStatus::CODE_200.code, response->getStatusCode()); @@ -1026,65 +1031,65 @@ TEST_F(WebControllerTest, SHOW_TABLES) { } TEST_F(WebControllerTest, DROP_TABLE) { - auto table_name = "table_drop_test" + OString(RandomName().c_str()); - GenTable(table_name, 128, 100, "L2"); + auto collection_name = "collection_drop_test" + OString(RandomName().c_str()); + GenTable(collection_name, 128, 100, "L2"); sleep(1); - auto response = client_ptr->dropTable(table_name, conncetion_ptr); + auto response = client_ptr->dropTable(collection_name, conncetion_ptr); ASSERT_EQ(OStatus::CODE_204.code, response->getStatusCode()); - table_name = "table_drop_test_not_exists_" + OString(RandomName().c_str()); - response = client_ptr->dropTable(table_name, conncetion_ptr); + collection_name = "collection_drop_test_not_exists_" + OString(RandomName().c_str()); + response = client_ptr->dropTable(collection_name, conncetion_ptr); ASSERT_EQ(OStatus::CODE_404.code, response->getStatusCode()); auto error_dto = response->readBodyToDto(object_mapper.get()); ASSERT_EQ(milvus::server::web::StatusCode::TABLE_NOT_EXISTS, error_dto->code->getValue()); } TEST_F(WebControllerTest, INSERT) { - auto table_name = "test_insert_table_test" + OString(RandomName().c_str()); + auto collection_name = "test_insert_collection_test" + OString(RandomName().c_str()); const int64_t dim = 64; - GenTable(table_name, dim, 100, "L2"); + GenTable(collection_name, dim, 100, "L2"); nlohmann::json insert_json; insert_json["vectors"] = RandomRecordsJson(dim, 20); - auto response = client_ptr->insert(table_name, insert_json.dump().c_str(), conncetion_ptr); + auto response = client_ptr->insert(collection_name, insert_json.dump().c_str(), conncetion_ptr); ASSERT_EQ(OStatus::CODE_201.code, response->getStatusCode()); auto result_dto = response->readBodyToDto(object_mapper.get()); ASSERT_EQ(20, result_dto->ids->count()); - response = client_ptr->insert(table_name + "ooowrweindexsgs", insert_json.dump().c_str(), conncetion_ptr); + response = client_ptr->insert(collection_name + "ooowrweindexsgs", insert_json.dump().c_str(), conncetion_ptr); ASSERT_EQ(OStatus::CODE_404.code, response->getStatusCode()); - response = client_ptr->dropTable(table_name, conncetion_ptr); + response = client_ptr->dropTable(collection_name, conncetion_ptr); ASSERT_EQ(OStatus::CODE_204.code, response->getStatusCode()); } TEST_F(WebControllerTest, INSERT_BIN) { - auto table_name = "test_insert_bin_table_test" + OString(RandomName().c_str()); + auto collection_name = "test_insert_bin_collection_test" + OString(RandomName().c_str()); const int64_t dim = 64; - GenTable(table_name, dim, 100, "HAMMING"); + GenTable(collection_name, dim, 100, "HAMMING"); nlohmann::json insert_json; insert_json["vectors"] = RandomBinRecordsJson(dim, 20); - auto response = client_ptr->insert(table_name, insert_json.dump().c_str(), conncetion_ptr); + auto response = client_ptr->insert(collection_name, insert_json.dump().c_str(), conncetion_ptr); ASSERT_EQ(OStatus::CODE_201.code, response->getStatusCode()) << response->readBodyToString()->std_str(); - auto status = FlushTable(table_name); + auto status = FlushTable(collection_name); ASSERT_TRUE(status.ok()) << status.message(); auto result_dto = response->readBodyToDto(object_mapper.get()); ASSERT_EQ(20, result_dto->ids->count()); - response = client_ptr->dropTable(table_name, conncetion_ptr); + response = client_ptr->dropTable(collection_name, conncetion_ptr); ASSERT_EQ(OStatus::CODE_204.code, response->getStatusCode()); } TEST_F(WebControllerTest, INSERT_IDS) { - auto table_name = "test_insert_table_test" + OString(RandomName().c_str()); + auto collection_name = "test_insert_collection_test" + OString(RandomName().c_str()); const int64_t dim = 64; - GenTable(table_name, dim, 100, "L2"); + GenTable(collection_name, dim, 100, "L2"); std::vector ids; for (size_t i = 0; i < 20; i++) { @@ -1095,157 +1100,153 @@ TEST_F(WebControllerTest, INSERT_IDS) { insert_json["vectors"] = RandomRecordsJson(dim, 20); insert_json["ids"] = ids; - auto response = client_ptr->insert(table_name, insert_json.dump().c_str(), conncetion_ptr); + auto response = client_ptr->insert(collection_name, insert_json.dump().c_str(), conncetion_ptr); ASSERT_EQ(OStatus::CODE_201.code, response->getStatusCode()) << response->readBodyToString()->std_str(); auto result_dto = response->readBodyToDto(object_mapper.get()); ASSERT_EQ(20, result_dto->ids->count()); - response = client_ptr->dropTable(table_name, conncetion_ptr); + response = client_ptr->dropTable(collection_name, conncetion_ptr); ASSERT_EQ(OStatus::CODE_204.code, response->getStatusCode()); } TEST_F(WebControllerTest, INDEX) { - auto table_name = "test_insert_table_test" + OString(RandomName().c_str()); - GenTable(table_name, 64, 100, "L2"); + auto collection_name = "test_insert_collection_test" + OString(RandomName().c_str()); + GenTable(collection_name, 64, 100, "L2"); // test index with imcomplete param - auto index_dto = milvus::server::web::IndexRequestDto::createShared(); - auto response = client_ptr->createIndex(table_name, index_dto, conncetion_ptr); + nlohmann::json index_json; + auto response = client_ptr->createIndex(collection_name, index_json.dump().c_str(), conncetion_ptr); + ASSERT_EQ(OStatus::CODE_400.code, response->getStatusCode()); + + index_json["index_type"] = milvus::server::web::IndexMap.at(milvus::engine::EngineType::FAISS_IDMAP); + + // missing index `params` + response = client_ptr->createIndex(collection_name, index_json.dump().c_str(), conncetion_ptr); + ASSERT_EQ(OStatus::CODE_400.code, response->getStatusCode()); + + index_json["params"] = nlohmann::json::parse("{\"nlist\": 10}"); + response = client_ptr->createIndex(collection_name, index_json.dump().c_str(), conncetion_ptr); ASSERT_EQ(OStatus::CODE_201.code, response->getStatusCode()); - auto create_index_dto = response->readBodyToDto(object_mapper.get()); - ASSERT_EQ(milvus::server::web::StatusCode::SUCCESS, create_index_dto->code); // drop index - response = client_ptr->dropIndex(table_name, conncetion_ptr); - ASSERT_EQ(OStatus::CODE_204.code, response->getStatusCode()); - - index_dto->index_type = milvus::server::web::IndexMap.at(milvus::engine::EngineType::FAISS_IDMAP).c_str(); - - response = client_ptr->createIndex(table_name, index_dto, conncetion_ptr); - ASSERT_EQ(OStatus::CODE_201.code, response->getStatusCode()); - // drop index - response = client_ptr->dropIndex(table_name, conncetion_ptr); + response = client_ptr->dropIndex(collection_name, conncetion_ptr); ASSERT_EQ(OStatus::CODE_204.code, response->getStatusCode()); // create index without existing table - response = client_ptr->createIndex(table_name + "fgafafafafafUUUUUUa124254", index_dto, conncetion_ptr); + response = client_ptr->createIndex(collection_name + "fgafafafafafUUUUUUa124254", index_json.dump().c_str(), conncetion_ptr); ASSERT_EQ(OStatus::CODE_404.code, response->getStatusCode()); - index_dto->index_type = "J46"; - response = client_ptr->createIndex(table_name, index_dto, conncetion_ptr); + // invalid index type + index_json["index_type"] = "J46"; + response = client_ptr->createIndex(collection_name, index_json.dump().c_str(), conncetion_ptr); ASSERT_EQ(OStatus::CODE_400.code, response->getStatusCode()); auto result_dto = response->readBodyToDto(object_mapper.get()); ASSERT_EQ(milvus::server::web::StatusCode::ILLEGAL_INDEX_TYPE, result_dto->code); - index_dto->index_type = milvus::server::web::IndexMap.at(milvus::engine::EngineType::FAISS_IDMAP).c_str(); - index_dto->nlist = 10; - - response = client_ptr->createIndex(table_name, index_dto, conncetion_ptr); - ASSERT_EQ(OStatus::CODE_201.code, response->getStatusCode()); - // drop index - response = client_ptr->dropIndex(table_name, conncetion_ptr); + response = client_ptr->dropIndex(collection_name, conncetion_ptr); ASSERT_EQ(OStatus::CODE_204.code, response->getStatusCode()); - // invalid index type - index_dto->index_type = 100; - response = client_ptr->createIndex(table_name, index_dto, conncetion_ptr); - ASSERT_EQ(OStatus::CODE_400.code, response->getStatusCode()); - // insert data and create index - response = client_ptr->dropIndex(table_name, conncetion_ptr); - ASSERT_EQ(OStatus::CODE_204.code, response->getStatusCode()); - - auto status = InsertData(table_name, 64, 200); + auto status = InsertData(collection_name, 64, 200); ASSERT_TRUE(status.ok()) << status.message(); - index_dto->index_type = milvus::server::web::IndexMap.at(milvus::engine::EngineType::FAISS_IDMAP).c_str(); - response = client_ptr->createIndex(table_name, index_dto, conncetion_ptr); + index_json["index_type"] = milvus::server::web::IndexMap.at(milvus::engine::EngineType::FAISS_IVFFLAT); + response = client_ptr->createIndex(collection_name, index_json.dump().c_str(), conncetion_ptr); ASSERT_EQ(OStatus::CODE_201.code, response->getStatusCode()); // get index - response = client_ptr->getIndex(table_name, conncetion_ptr); + response = client_ptr->getIndex(collection_name, conncetion_ptr); ASSERT_EQ(OStatus::CODE_200.code, response->getStatusCode()); - auto result_index_dto = response->readBodyToDto(object_mapper.get()); - ASSERT_EQ("FLAT", result_index_dto->index_type->std_str()); - ASSERT_EQ(10, result_index_dto->nlist->getValue()); + auto result_index_json = nlohmann::json::parse(response->readBodyToString()->c_str()); + ASSERT_TRUE(result_index_json.contains("index_type")); + ASSERT_EQ("IVFFLAT", result_index_json["index_type"].get()); + ASSERT_TRUE(result_index_json.contains("params")); + + // check index params + auto params_json = result_index_json["params"]; + ASSERT_TRUE(params_json.contains("nlist")); + auto nlist_json = params_json["nlist"]; + ASSERT_TRUE(nlist_json.is_number()); + ASSERT_EQ(10, nlist_json.get()); + // get index of table which not exists - response = client_ptr->getIndex(table_name + "dfaedXXXdfdfet4t343aa4", conncetion_ptr); + response = client_ptr->getIndex(collection_name + "dfaedXXXdfdfet4t343aa4", conncetion_ptr); ASSERT_EQ(OStatus::CODE_404.code, response->getStatusCode()); auto error_dto = response->readBodyToDto(object_mapper.get()); ASSERT_EQ(milvus::server::web::StatusCode::TABLE_NOT_EXISTS, error_dto->code->getValue()); } TEST_F(WebControllerTest, PARTITION) { - const OString table_name = "test_controller_partition_" + OString(RandomName().c_str()); - GenTable(table_name, 64, 100, "L2"); + const OString collection_name = "test_controller_partition_" + OString(RandomName().c_str()); + GenTable(collection_name, 64, 100, "L2"); auto par_param = milvus::server::web::PartitionRequestDto::createShared(); - auto response = client_ptr->createPartition(table_name, par_param); + auto response = client_ptr->createPartition(collection_name, par_param); ASSERT_EQ(OStatus::CODE_400.code, response->getStatusCode()); auto error_dto = response->readBodyToDto(object_mapper.get()); ASSERT_EQ(milvus::server::web::StatusCode::BODY_FIELD_LOSS, error_dto->code); - response = client_ptr->createPartition(table_name, par_param); + response = client_ptr->createPartition(collection_name, par_param); ASSERT_EQ(OStatus::CODE_400.code, response->getStatusCode()); error_dto = response->readBodyToDto(object_mapper.get()); ASSERT_EQ(milvus::server::web::StatusCode::BODY_FIELD_LOSS, error_dto->code); par_param->partition_tag = "tag01"; - response = client_ptr->createPartition(table_name, par_param); + response = client_ptr->createPartition(collection_name, par_param); ASSERT_EQ(OStatus::CODE_201.code, response->getStatusCode()); auto create_result_dto = response->readBodyToDto(object_mapper.get()); ASSERT_EQ(milvus::server::web::StatusCode::SUCCESS, create_result_dto->code); - response = client_ptr->createPartition(table_name + "afafanotgitdiexists", par_param); + response = client_ptr->createPartition(collection_name + "afafanotgitdiexists", par_param); ASSERT_EQ(OStatus::CODE_404.code, response->getStatusCode()); error_dto = response->readBodyToDto(object_mapper.get()); ASSERT_EQ(milvus::server::web::StatusCode::TABLE_NOT_EXISTS, error_dto->code); - // insert 200 vectors into table with tag = 'tag01' - auto status = InsertData(table_name, 64, 200, "tag01"); + // insert 200 vectors into collection with tag = 'tag01' + auto status = InsertData(collection_name, 64, 200, "tag01"); ASSERT_TRUE(status.ok()) << status.message(); // Show all partitins - response = client_ptr->showPartitions(table_name, "0", "10", conncetion_ptr); + response = client_ptr->showPartitions(collection_name, "0", "10", conncetion_ptr); ASSERT_EQ(OStatus::CODE_200.code, response->getStatusCode()); auto result_dto = response->readBodyToDto(object_mapper.get()); ASSERT_EQ(2, result_dto->partitions->count()); ASSERT_EQ("tag01", result_dto->partitions->get(1)->partition_tag->std_str()); - response = client_ptr->showPartitions(table_name, "0", "-1", conncetion_ptr); + response = client_ptr->showPartitions(collection_name, "0", "-1", conncetion_ptr); ASSERT_EQ(OStatus::CODE_400.code, response->getStatusCode()); - response = client_ptr->showPartitions(table_name, "0.1", "7", conncetion_ptr); + response = client_ptr->showPartitions(collection_name, "0.1", "7", conncetion_ptr); ASSERT_EQ(OStatus::CODE_400.code, response->getStatusCode()); - response = client_ptr->showPartitions(table_name, "0", "1.6", conncetion_ptr); + response = client_ptr->showPartitions(collection_name, "0", "1.6", conncetion_ptr); ASSERT_EQ(OStatus::CODE_400.code, response->getStatusCode()); - response = client_ptr->showPartitions(table_name, "567a", "1", conncetion_ptr); + response = client_ptr->showPartitions(collection_name, "567a", "1", conncetion_ptr); ASSERT_EQ(OStatus::CODE_400.code, response->getStatusCode()); - // show without existing tables - response = client_ptr->showPartitions(table_name + "dfafaefaluanqibazao990099", "0", "10", conncetion_ptr); + // show without existing collections + response = client_ptr->showPartitions(collection_name + "dfafaefaluanqibazao990099", "0", "10", conncetion_ptr); ASSERT_EQ(OStatus::CODE_404.code, response->getStatusCode()); error_dto = response->readBodyToDto(object_mapper.get()); ASSERT_EQ(milvus::server::web::StatusCode::TABLE_NOT_EXISTS, error_dto->code->getValue()); - response = client_ptr->dropPartition(table_name, "{\"partition_tag\": \"tag01\"}", conncetion_ptr); + response = client_ptr->dropPartition(collection_name, "{\"partition_tag\": \"tag01\"}", conncetion_ptr); ASSERT_EQ(OStatus::CODE_204.code, response->getStatusCode()); - // drop without existing tables - response = client_ptr->dropPartition(table_name + "565755682353464aaasafdsfagagqq1223", + // drop without existing collections + response = client_ptr->dropPartition(collection_name + "565755682353464aaasafdsfagagqq1223", "{\"partition_tag\": \"tag01\"}", conncetion_ptr); ASSERT_EQ(OStatus::CODE_404.code, response->getStatusCode()); } TEST_F(WebControllerTest, SHOW_SEGMENTS) { - OString table_name = OString("test_milvus_web_segments_test_") + RandomName().c_str(); + OString collection_name = OString("test_milvus_web_segments_test_") + RandomName().c_str(); - GenTable(table_name, 256, 1, "L2"); + GenTable(collection_name, 256, 1, "L2"); - auto status = InsertData(table_name, 256, 2000); + auto status = InsertData(collection_name, 256, 2000); ASSERT_TRUE(status.ok()) << status.message(); - auto response = client_ptr->showSegments(table_name, "0", "10", "", conncetion_ptr); + auto response = client_ptr->showSegments(collection_name, "0", "10", "", conncetion_ptr); ASSERT_EQ(OStatus::CODE_200.code, response->getStatusCode()) << response->readBodyToString()->c_str(); // validate result @@ -1260,14 +1261,14 @@ TEST_F(WebControllerTest, SHOW_SEGMENTS) { } TEST_F(WebControllerTest, GET_SEGMENT_INFO) { - OString table_name = OString("test_milvus_web_get_segment_info_test_") + RandomName().c_str(); + OString collection_name = OString("test_milvus_web_get_segment_info_test_") + RandomName().c_str(); - GenTable(table_name, 16, 1, "L2"); + GenTable(collection_name, 16, 1, "L2"); - auto status = InsertData(table_name, 16, 2000); + auto status = InsertData(collection_name, 16, 2000); ASSERT_TRUE(status.ok()) << status.message(); - auto response = client_ptr->showSegments(table_name, "0", "10", "", conncetion_ptr); + auto response = client_ptr->showSegments(collection_name, "0", "10", "", conncetion_ptr); ASSERT_EQ(OStatus::CODE_200.code, response->getStatusCode()) << response->readBodyToString()->c_str(); // validate result @@ -1278,7 +1279,7 @@ TEST_F(WebControllerTest, GET_SEGMENT_INFO) { // get segment ids - response = client_ptr->getSegmentInfo(table_name, segment_name.c_str(), "ids", "0", "10"); + response = client_ptr->getSegmentInfo(collection_name, segment_name.c_str(), "ids", "0", "10"); ASSERT_EQ(OStatus::CODE_200.code, response->getStatusCode()) << response->readBodyToString()->c_str(); auto ids_result_json = nlohmann::json::parse(response->readBodyToString()->c_str()); @@ -1288,7 +1289,7 @@ TEST_F(WebControllerTest, GET_SEGMENT_INFO) { ASSERT_EQ(10, ids_json.size()); // get segment vectors - response = client_ptr->getSegmentInfo(table_name, segment_name.c_str(), "vectors", "0", "10"); + response = client_ptr->getSegmentInfo(collection_name, segment_name.c_str(), "vectors", "0", "10"); ASSERT_EQ(OStatus::CODE_200.code, response->getStatusCode()) << response->readBodyToString()->c_str(); auto vecs_result_json = nlohmann::json::parse(response->readBodyToString()->c_str()); @@ -1297,32 +1298,32 @@ TEST_F(WebControllerTest, GET_SEGMENT_INFO) { ASSERT_TRUE(vecs_json.is_array()); ASSERT_EQ(10, vecs_json.size()); - // non-existent table - response = client_ptr->getSegmentInfo(table_name + "_non_existent", segment_name.c_str(), "ids", "0", "10"); + // non-existent collection + response = client_ptr->getSegmentInfo(collection_name + "_non_existent", segment_name.c_str(), "ids", "0", "10"); ASSERT_EQ(OStatus::CODE_404.code, response->getStatusCode()) << response->readBodyToString()->c_str(); } TEST_F(WebControllerTest, SEGMENT_FILTER) { - OString table_name = OString("test_milvus_web_segment_filter_test_") + RandomName().c_str(); - GenTable(table_name, 16, 1, "L2"); + OString collection_name = OString("test_milvus_web_segment_filter_test_") + RandomName().c_str(); + GenTable(collection_name, 16, 1, "L2"); - auto status = InsertData(table_name, 16, 1000); + auto status = InsertData(collection_name, 16, 1000); ASSERT_TRUE(status.ok()) << status.message(); - status = GenPartition(table_name, "tag01"); + status = GenPartition(collection_name, "tag01"); ASSERT_TRUE(status.ok()) << status.message(); - status = InsertData(table_name, 16, 1000, "tag01"); + status = InsertData(collection_name, 16, 1000, "tag01"); ASSERT_TRUE(status.ok()) << status.message(); - status = GenPartition(table_name, "tag02"); + status = GenPartition(collection_name, "tag02"); ASSERT_TRUE(status.ok()) << status.message(); - status = InsertData(table_name, 16, 1000, "tag02"); + status = InsertData(collection_name, 16, 1000, "tag02"); ASSERT_TRUE(status.ok()) << status.message(); // show segments filtering tag - auto response = client_ptr->showSegments(table_name, "0", "10", "_default", conncetion_ptr); + auto response = client_ptr->showSegments(collection_name, "0", "10", "_default", conncetion_ptr); ASSERT_EQ(OStatus::CODE_200.code, response->getStatusCode()) << response->readBodyToString()->c_str(); auto result_json = nlohmann::json::parse(response->readBodyToString()->c_str()); @@ -1338,41 +1339,41 @@ TEST_F(WebControllerTest, SEGMENT_FILTER) { } TEST_F(WebControllerTest, SEARCH) { - const OString table_name = "test_search_table_test" + OString(RandomName().c_str()); - GenTable(table_name, 64, 100, "L2"); + const OString collection_name = "test_search_collection_test" + OString(RandomName().c_str()); + GenTable(collection_name, 64, 100, "L2"); - // Insert 200 vectors into table - auto status = InsertData(table_name, 64, 200); + // Insert 200 vectors into collection + auto status = InsertData(collection_name, 64, 200); ASSERT_TRUE(status.ok()) << status.message(); // Create partition and insert 200 vectors into it auto par_param = milvus::server::web::PartitionRequestDto::createShared(); par_param->partition_tag = "tag" + OString(RandomName().c_str()); - auto response = client_ptr->createPartition(table_name, par_param); + auto response = client_ptr->createPartition(collection_name, par_param); ASSERT_EQ(OStatus::CODE_201.code, response->getStatusCode()) << "Error: " << response->getStatusDescription()->std_str(); - status = InsertData(table_name, 64, 200, par_param->partition_tag->std_str()); + status = InsertData(collection_name, 64, 200, par_param->partition_tag->std_str()); ASSERT_TRUE(status.ok()) << status.message(); // Test search nlohmann::json search_json; - response = client_ptr->vectorsOp(table_name, search_json.dump().c_str(), conncetion_ptr); + response = client_ptr->vectorsOp(collection_name, search_json.dump().c_str(), conncetion_ptr); auto error_dto = response->readBodyToDto(object_mapper.get()); ASSERT_NE(milvus::server::web::StatusCode::SUCCESS, error_dto->code); - search_json["search"]["nprobe"] = 1; - response = client_ptr->vectorsOp(table_name, search_json.dump().c_str(), conncetion_ptr); + search_json["search"]["params"]["nprobe"] = 1; + response = client_ptr->vectorsOp(collection_name, search_json.dump().c_str(), conncetion_ptr); error_dto = response->readBodyToDto(object_mapper.get()); ASSERT_EQ(milvus::server::web::StatusCode::BODY_FIELD_LOSS, error_dto->code); search_json["search"]["topk"] = 1; - response = client_ptr->vectorsOp(table_name, search_json.dump().c_str(), conncetion_ptr); + response = client_ptr->vectorsOp(collection_name, search_json.dump().c_str(), conncetion_ptr); error_dto = response->readBodyToDto(object_mapper.get()); ASSERT_NE(milvus::server::web::StatusCode::SUCCESS, error_dto->code); search_json["search"]["vectors"] = RandomRecordsJson(64, 10); - response = client_ptr->vectorsOp(table_name, search_json.dump().c_str(), conncetion_ptr); + response = client_ptr->vectorsOp(collection_name, search_json.dump().c_str(), conncetion_ptr); ASSERT_EQ(OStatus::CODE_200.code, response->getStatusCode()); auto result_json = nlohmann::json::parse(response->readBodyToString()->std_str()); @@ -1392,52 +1393,52 @@ TEST_F(WebControllerTest, SEARCH) { par_json.push_back(par_param->partition_tag->std_str()); search_json["search"]["partition_tags"] = par_json; - response = client_ptr->vectorsOp(table_name, search_json.dump().c_str(), conncetion_ptr); + response = client_ptr->vectorsOp(collection_name, search_json.dump().c_str(), conncetion_ptr); ASSERT_EQ(OStatus::CODE_200.code, response->getStatusCode()); - // Test search without existing table - response = client_ptr->vectorsOp(table_name + "999piyanning", search_json.dump().c_str(), conncetion_ptr); + // Test search without existing collection + response = client_ptr->vectorsOp(collection_name + "999piyanning", search_json.dump().c_str(), conncetion_ptr); ASSERT_EQ(OStatus::CODE_404.code, response->getStatusCode()); error_dto = response->readBodyToDto(object_mapper.get()); ASSERT_EQ(milvus::server::web::StatusCode::TABLE_NOT_EXISTS, error_dto->code->getValue()); } TEST_F(WebControllerTest, SEARCH_BIN) { - const OString table_name = "test_search_bin_table_test" + OString(RandomName().c_str()); - GenTable(table_name, 64, 100, "HAMMING"); + const OString collection_name = "test_search_bin_collection_test" + OString(RandomName().c_str()); + GenTable(collection_name, 64, 100, "HAMMING"); - // Insert 200 vectors into table - auto status = InsertData(table_name, 64, 200, "", true); + // Insert 200 vectors into collection + auto status = InsertData(collection_name, 64, 200, "", true); ASSERT_TRUE(status.ok()) << status.message(); // Create partition and insert 200 vectors into it auto par_param = milvus::server::web::PartitionRequestDto::createShared(); par_param->partition_tag = "tag" + OString(RandomName().c_str()); - auto response = client_ptr->createPartition(table_name, par_param); + auto response = client_ptr->createPartition(collection_name, par_param); ASSERT_EQ(OStatus::CODE_201.code, response->getStatusCode()) << "Error: " << response->readBodyToString()->std_str(); - status = InsertData(table_name, 64, 200, par_param->partition_tag->std_str(), true); + status = InsertData(collection_name, 64, 200, par_param->partition_tag->std_str(), true); ASSERT_TRUE(status.ok()) << status.message(); // Test search nlohmann::json search_json; - response = client_ptr->vectorsOp(table_name, search_json.dump().c_str(), conncetion_ptr); + response = client_ptr->vectorsOp(collection_name, search_json.dump().c_str(), conncetion_ptr); auto result_dto = response->readBodyToDto(object_mapper.get()); ASSERT_NE(milvus::server::web::StatusCode::SUCCESS, result_dto->code); - search_json["search"]["nprobe"] = 1; - response = client_ptr->vectorsOp(table_name, search_json.dump().c_str(), conncetion_ptr); + search_json["search"]["params"]["nprobe"] = 1; + response = client_ptr->vectorsOp(collection_name, search_json.dump().c_str(), conncetion_ptr); result_dto = response->readBodyToDto(object_mapper.get()); ASSERT_NE(milvus::server::web::StatusCode::SUCCESS, result_dto->code); search_json["search"]["topk"] = 1; - response = client_ptr->vectorsOp(table_name, search_json.dump().c_str(), conncetion_ptr); + response = client_ptr->vectorsOp(collection_name, search_json.dump().c_str(), conncetion_ptr); result_dto = response->readBodyToDto(object_mapper.get()); ASSERT_NE(milvus::server::web::StatusCode::SUCCESS, result_dto->code); search_json["search"]["vectors"] = RandomBinRecordsJson(64, 10); - response = client_ptr->vectorsOp(table_name, search_json.dump().c_str(), conncetion_ptr); + response = client_ptr->vectorsOp(collection_name, search_json.dump().c_str(), conncetion_ptr); ASSERT_EQ(OStatus::CODE_200.code, response->getStatusCode()); // validate search result @@ -1453,7 +1454,7 @@ TEST_F(WebControllerTest, SEARCH_BIN) { // Test search with tags search_json["search"]["partition_tags"] = std::vector(); search_json["search"]["partition_tags"].push_back(par_param->partition_tag->std_str()); - response = client_ptr->vectorsOp(table_name, search_json.dump().c_str(), conncetion_ptr); + response = client_ptr->vectorsOp(collection_name, search_json.dump().c_str(), conncetion_ptr); ASSERT_EQ(OStatus::CODE_200.code, response->getStatusCode()); } @@ -1464,16 +1465,16 @@ TEST_F(WebControllerTest, SEARCH_BIN) { // ASSERT_TRUE(config_status.ok()) << config_status.message(); //#endif // -// const OString table_name = "test_search_by_id_table_test_" + OString(RandomName().c_str()); -// GenTable(table_name, 64, 100, "L2"); +// const OString collection_name = "test_search_by_id_collection_test_" + OString(RandomName().c_str()); +// GenTable(collection_name, 64, 100, "L2"); // -// // Insert 100 vectors into table +// // Insert 100 vectors into collection // std::vector ids; // for (size_t i = 0; i < 100; i++) { // ids.emplace_back(i); // } // -// auto status = InsertData(table_name, 64, 100, ids); +// auto status = InsertData(collection_name, 64, 100, ids); // ASSERT_TRUE(status.ok()) << status.message(); // // nlohmann::json search_json; @@ -1481,7 +1482,7 @@ TEST_F(WebControllerTest, SEARCH_BIN) { // search_json["search"]["nprobe"] = 1; // search_json["search"]["vector_id"] = ids.at(0); // -// auto response = client_ptr->vectorsOp(table_name, search_json.dump().c_str(), conncetion_ptr); +// auto response = client_ptr->vectorsOp(collection_name, search_json.dump().c_str(), conncetion_ptr); // ASSERT_EQ(OStatus::CODE_200.code, response->getStatusCode()) << response->readBodyToString()->c_str(); // // // validate search result @@ -1503,21 +1504,21 @@ TEST_F(WebControllerTest, SEARCH_BIN) { //} TEST_F(WebControllerTest, GET_VECTOR_BY_ID) { - const OString table_name = "test_milvus_web_get_vector_by_id_test_" + OString(RandomName().c_str()); - GenTable(table_name, 64, 100, "L2"); + const OString collection_name = "test_milvus_web_get_vector_by_id_test_" + OString(RandomName().c_str()); + GenTable(collection_name, 64, 100, "L2"); - // Insert 100 vectors into table + // Insert 100 vectors into collection std::vector ids; for (size_t i = 0; i < 100; i++) { ids.emplace_back(i); } - auto status = InsertData(table_name, 64, 100, ids); + auto status = InsertData(collection_name, 64, 100, ids); ASSERT_TRUE(status.ok()) << status.message(); /* test task load */ auto id_str = std::to_string(ids.at(0)); - auto response = client_ptr->getVectors(table_name, id_str.c_str(), conncetion_ptr); + auto response = client_ptr->getVectors(collection_name, id_str.c_str(), conncetion_ptr); ASSERT_EQ(OStatus::CODE_200.code, response->getStatusCode()) << response->readBodyToString()->c_str(); // validate result @@ -1541,19 +1542,19 @@ TEST_F(WebControllerTest, GET_VECTOR_BY_ID) { ASSERT_EQ(64, vec.size()); - // non-existent table - response = client_ptr->getVectors(table_name + "_non_existent", id_str.c_str(), conncetion_ptr); + // non-existent collection + response = client_ptr->getVectors(collection_name + "_non_existent", id_str.c_str(), conncetion_ptr); ASSERT_EQ(OStatus::CODE_404.code, response->getStatusCode()) << response->readBodyToString()->c_str(); } TEST_F(WebControllerTest, DELETE_BY_ID) { - const OString table_name = "test_search_bin_table_test" + OString(RandomName().c_str()); - GenTable(table_name, 64, 100, "L2"); + const OString collection_name = "test_search_bin_collection_test" + OString(RandomName().c_str()); + GenTable(collection_name, 64, 100, "L2"); - // Insert 200 vectors into table + // Insert 200 vectors into collection nlohmann::json insert_json; insert_json["vectors"] = RandomRecordsJson(64, 2000); - auto response = client_ptr->insert(table_name, insert_json.dump().c_str(), conncetion_ptr); + auto response = client_ptr->insert(collection_name, insert_json.dump().c_str(), conncetion_ptr); ASSERT_EQ(OStatus::CODE_201.code, response->getStatusCode()) << response->readBodyToString()->c_str(); auto insert_result_json = nlohmann::json::parse(response->readBodyToString()->c_str()); @@ -1571,11 +1572,11 @@ TEST_F(WebControllerTest, DELETE_BY_ID) { nlohmann::json delete_json; delete_json["delete"]["ids"] = delete_ids; - response = client_ptr->vectorsOp(table_name, delete_json.dump().c_str(), conncetion_ptr); + response = client_ptr->vectorsOp(collection_name, delete_json.dump().c_str(), conncetion_ptr); ASSERT_EQ(OStatus::CODE_200.code, response->getStatusCode()) << response->readBodyToString()->c_str(); - // non-existent table - response = client_ptr->vectorsOp(table_name + "_non_existent", delete_json.dump().c_str(), conncetion_ptr); + // non-existent collection + response = client_ptr->vectorsOp(collection_name + "_non_existent", delete_json.dump().c_str(), conncetion_ptr); ASSERT_EQ(OStatus::CODE_404.code, response->getStatusCode()) << response->readBodyToString()->c_str(); } @@ -1590,7 +1591,7 @@ TEST_F(WebControllerTest, CMD) { response = client_ptr->cmd("mode", "", "", conncetion_ptr); ASSERT_EQ(OStatus::CODE_200.code, response->getStatusCode()); - response = client_ptr->cmd("tasktable", "", "", conncetion_ptr); + response = client_ptr->cmd("taskcollection", "", "", conncetion_ptr); ASSERT_EQ(OStatus::CODE_200.code, response->getStatusCode()); response = client_ptr->cmd("info", "", "", conncetion_ptr); @@ -1624,17 +1625,17 @@ TEST_F(WebControllerTest, CONFIG) { auto result_json = nlohmann::json::parse(response->readBodyToString()->c_str()); ASSERT_TRUE(result_json.contains("restart_required")); - OString table_name = "milvus_test_webcontroller_test_preload_table"; - GenTable(table_name, 16, 10, "L2"); + OString collection_name = "milvus_test_webcontroller_test_preload_collection"; + GenTable(collection_name, 16, 10, "L2"); - OString table_name_s = "milvus_test_webcontroller_test_preload_table_s"; - GenTable(table_name_s, 16, 10, "L2"); + OString collection_name_s = "milvus_test_webcontroller_test_preload_collection_s"; + GenTable(collection_name_s, 16, 10, "L2"); - OString body_str = "{\"db_config\": {\"preload_table\": \"" + table_name + "\"}}"; + OString body_str = "{\"db_config\": {\"preload_table\": \"" + collection_name + "\"}}"; response = client_ptr->op("config", body_str, conncetion_ptr); ASSERT_EQ(OStatus::CODE_200.code, response->getStatusCode()) << response->readBodyToString()->c_str(); - body_str = "{\"db_config\": {\"preload_table\": \"" + table_name + "," + table_name_s + "\"}}"; + body_str = "{\"db_config\": {\"preload_table\": \"" + collection_name + "," + collection_name_s + "\"}}"; response = client_ptr->op("config", body_str, conncetion_ptr); ASSERT_EQ(OStatus::CODE_200.code, response->getStatusCode()) << response->readBodyToString()->c_str(); auto set_result_json = nlohmann::json::parse(response->readBodyToString()->c_str()); @@ -1763,53 +1764,53 @@ TEST_F(WebControllerTest, DEVICES_CONFIG) { } TEST_F(WebControllerTest, FLUSH) { - auto table_name = milvus::server::web::OString(TABLE_NAME) + RandomName().c_str(); - GenTable(table_name, 16, 10, "L2"); + auto collection_name = milvus::server::web::OString(TABLE_NAME) + RandomName().c_str(); + GenTable(collection_name, 16, 10, "L2"); - auto status = InsertData(table_name, 16, 1000); + auto status = InsertData(collection_name, 16, 1000); ASSERT_TRUE(status.ok()) << status.message(); nlohmann::json flush_json; - flush_json["flush"]["table_names"] = {table_name->std_str()}; + flush_json["flush"]["collection_names"] = {collection_name->std_str()}; auto response = client_ptr->op("task", flush_json.dump().c_str(), conncetion_ptr); ASSERT_EQ(OStatus::CODE_200.code, response->getStatusCode()); // invalid payload format - flush_json["flush"]["table_names"] = table_name->std_str(); + flush_json["flush"]["collection_names"] = collection_name->std_str(); response = client_ptr->op("task", flush_json.dump().c_str(), conncetion_ptr); ASSERT_EQ(OStatus::CODE_400.code, response->getStatusCode()); // non-existent name - flush_json["flush"]["table_names"] = {"afafaf444353"}; + flush_json["flush"]["collection_names"] = {"afafaf444353"}; response = client_ptr->op("task", flush_json.dump().c_str(), conncetion_ptr); ASSERT_EQ(OStatus::CODE_400.code, response->getStatusCode()); } TEST_F(WebControllerTest, COMPACT) { - auto table_name = milvus::server::web::OString("milvus_web_test_compact_") + RandomName().c_str(); - GenTable(table_name, 16, 10, "L2"); + auto collection_name = milvus::server::web::OString("milvus_web_test_compact_") + RandomName().c_str(); + GenTable(collection_name, 16, 10, "L2"); - auto status = InsertData(table_name, 16, 1000); + auto status = InsertData(collection_name, 16, 1000); ASSERT_TRUE(status.ok()) << status.message(); nlohmann::json compact_json; - compact_json["compact"]["table_name"] = table_name->std_str(); + compact_json["compact"]["collection_name"] = collection_name->std_str(); auto response = client_ptr->op("task", compact_json.dump().c_str(), conncetion_ptr); ASSERT_EQ(OStatus::CODE_200.code, response->getStatusCode()) << response->readBodyToString()->c_str(); } TEST_F(WebControllerTest, LOAD) { - OString table_name = "milvus_web_test_load_" + OString(RandomName().c_str()); - GenTable(table_name, 128, 100, "L2"); + OString collection_name = "milvus_web_test_load_" + OString(RandomName().c_str()); + GenTable(collection_name, 128, 100, "L2"); nlohmann::json load_json; - load_json["load"]["table_name"] = table_name->c_str(); + load_json["load"]["collection_name"] = collection_name->c_str(); auto response = client_ptr->op("task", load_json.dump().c_str(), conncetion_ptr); ASSERT_EQ(OStatus::CODE_200.code, response->getStatusCode()); // load with a non-existent name - load_json["load"]["table_name"] = "sssssssssssssssssssssssfsfsfsrrrttt"; + load_json["load"]["collection_name"] = "sssssssssssssssssssssssfsfsfsrrrttt"; response = client_ptr->op("task", load_json.dump().c_str(), conncetion_ptr); ASSERT_EQ(OStatus::CODE_400.code, response->getStatusCode()); diff --git a/core/unittest/wrapper/test_binindex.cpp b/core/unittest/wrapper/test_binindex.cpp index fac2d2ad62..9c6f626475 100644 --- a/core/unittest/wrapper/test_binindex.cpp +++ b/core/unittest/wrapper/test_binindex.cpp @@ -28,12 +28,12 @@ class BinIndexTest : public BinDataGen, std::tie(index_type, dim, nb, nq, k) = GetParam(); Generate(dim, nb, nq, k); - milvus::engine::TempMetaConf tempconf; - tempconf.metric_type = knowhere::METRICTYPE::TANIMOTO; - tempconf.size = nb; - tempconf.dim = dim; - tempconf.k = k; - tempconf.nprobe = 16; + knowhere::Config tempconf{ + {knowhere::Metric::TYPE, knowhere::Metric::TANIMOTO}, + {knowhere::meta::ROWS, nb}, + {knowhere::meta::DIM, dim}, + {knowhere::meta::TOPK, k} + }; index_ = GetVecIndexFactory(index_type); conf = ParamGenerator::GetInstance().GenBuild(index_type, tempconf); @@ -59,8 +59,8 @@ INSTANTIATE_TEST_CASE_P(WrapperParam, BinIndexTest, TEST_P(BinIndexTest, BASE_TEST) { EXPECT_EQ(index_->GetType(), index_type); - conf->Dump(); - searchconf->Dump(); + // conf->Dump(); + // searchconf->Dump(); auto elems = nq * k; std::vector res_ids(elems); diff --git a/core/unittest/wrapper/test_hybrid_index.cpp b/core/unittest/wrapper/test_hybrid_index.cpp index 2d8ec09b68..0488163107 100644 --- a/core/unittest/wrapper/test_hybrid_index.cpp +++ b/core/unittest/wrapper/test_hybrid_index.cpp @@ -51,19 +51,25 @@ class KnowhereHybrid : public DataGenBase, public ::testing::Test { TEST_F(KnowhereHybrid, test_interface) { assert(!xb.empty()); + knowhere::Config tempconf{ + {knowhere::Metric::TYPE, knowhere::Metric::L2}, + {knowhere::meta::ROWS, nb}, + {knowhere::meta::DIM, dim}, + {knowhere::meta::TOPK, k}, + {knowhere::meta::DEVICEID, DEVICEID} + }; + index_type = milvus::engine::IndexType::FAISS_IVFSQ8_HYBRID; index_ = GetVecIndexFactory(index_type); - conf = ParamGenerator::GetInstance().Gen(index_type); + conf = ParamGenerator::GetInstance().GenBuild(index_type, tempconf); + auto search_cfg = ParamGenerator::GetInstance().GenSearchConf(index_type, tempconf); auto elems = nq * k; std::vector res_ids(elems); std::vector res_dis(elems); - conf->gpu_id = DEVICEID; - conf->d = dim; - conf->k = k; index_->BuildAll(nb, xb.data(), ids.data(), conf); - index_->Search(nq, xq.data(), res_dis.data(), res_ids.data(), conf); + index_->Search(nq, xq.data(), res_dis.data(), res_ids.data(), search_cfg); AssertResult(res_ids, res_dis); EXPECT_EQ(index_->Count(), nb); EXPECT_EQ(index_->Dimension(), dim); @@ -76,7 +82,7 @@ TEST_F(KnowhereHybrid, test_interface) { { for (int i = 0; i < 2; ++i) { auto gpu_idx = cpu_idx->CopyToGpu(DEVICEID, conf); - gpu_idx->Search(nq, xq.data(), res_dis.data(), res_ids.data(), conf); + gpu_idx->Search(nq, xq.data(), res_dis.data(), res_ids.data(), search_cfg); AssertResult(res_ids, res_dis); } } @@ -91,18 +97,19 @@ TEST_F(KnowhereHybrid, test_interface) { auto gpu_idx = pair.first; auto quantization = pair.second; - gpu_idx->Search(nq, xq.data(), res_dis.data(), res_ids.data(), conf); + gpu_idx->Search(nq, xq.data(), res_dis.data(), res_ids.data(), search_cfg); AssertResult(res_ids, res_dis); - auto quantizer_conf = std::make_shared(); - quantizer_conf->mode = 2; - quantizer_conf->gpu_id = DEVICEID; + milvus::json quantizer_conf { + {"mode", 2}, + {knowhere::meta::DEVICEID, DEVICEID} + }; for (int i = 0; i < 2; ++i) { auto hybrid_idx = GetVecIndexFactory(index_type); hybrid_idx->Load(binaryset); hybrid_idx->LoadData(quantization, quantizer_conf); - hybrid_idx->Search(nq, xq.data(), res_dis.data(), res_ids.data(), conf); + hybrid_idx->Search(nq, xq.data(), res_dis.data(), res_ids.data(), search_cfg); AssertResult(res_ids, res_dis); } } @@ -120,82 +127,82 @@ TEST_F(KnowhereHybrid, test_interface) { hybrid_idx->Load(binaryset); hybrid_idx->SetQuantizer(quantization); - hybrid_idx->Search(nq, xq.data(), res_dis.data(), res_ids.data(), conf); + hybrid_idx->Search(nq, xq.data(), res_dis.data(), res_ids.data(), search_cfg); AssertResult(res_ids, res_dis); hybrid_idx->UnsetQuantizer(); } } } -TEST_F(KnowhereHybrid, test_invalid_index) { - assert(!xb.empty()); - fiu_init(0); +// TEST_F(KnowhereHybrid, test_invalid_index) { +// assert(!xb.empty()); +// fiu_init(0); - index_type = milvus::engine::IndexType::FAISS_IVFSQ8_HYBRID; - fiu_enable("GetVecIndexFactory.IVFSQHybrid.mock", 1, nullptr, 0); - index_ = GetVecIndexFactory(index_type); - fiu_disable("GetVecIndexFactory.IVFSQHybrid.mock"); - conf = ParamGenerator::GetInstance().Gen(index_type); +// index_type = milvus::engine::IndexType::FAISS_IVFSQ8_HYBRID; +// fiu_enable("GetVecIndexFactory.IVFSQHybrid.mock", 1, nullptr, 0); +// index_ = GetVecIndexFactory(index_type); +// fiu_disable("GetVecIndexFactory.IVFSQHybrid.mock"); +// conf = ParamGenerator::GetInstance().Gen(index_type); - auto elems = nq * k; - std::vector res_ids(elems); - std::vector res_dis(elems); +// auto elems = nq * k; +// std::vector res_ids(elems); +// std::vector res_dis(elems); - conf->gpu_id = DEVICEID; - conf->d = dim; - conf->k = k; +// conf->gpu_id = DEVICEID; +// conf->d = dim; +// conf->k = k; - auto status = index_->BuildAll(nb, xb.data(), ids.data(), conf); - ASSERT_FALSE(status.ok()); +// auto status = index_->BuildAll(nb, xb.data(), ids.data(), conf); +// ASSERT_FALSE(status.ok()); - auto quantizer = std::make_shared(); - auto quantizer_cfg = std::make_shared(); - index_->LoadQuantizer(quantizer_cfg); - status = index_->SetQuantizer(quantizer); - ASSERT_FALSE(status.ok()); +// auto quantizer = std::make_shared(); +// auto quantizer_cfg = std::make_shared(); +// index_->LoadQuantizer(quantizer_cfg); +// status = index_->SetQuantizer(quantizer); +// ASSERT_FALSE(status.ok()); - fiu_enable("IVFHybridIndex.SetQuantizer.throw_knowhere_exception", 1, nullptr, 0); - status = index_->SetQuantizer(quantizer); - ASSERT_FALSE(status.ok()); - fiu_disable("IVFHybridIndex.SetQuantizer.throw_knowhere_exception"); +// fiu_enable("IVFHybridIndex.SetQuantizer.throw_knowhere_exception", 1, nullptr, 0); +// status = index_->SetQuantizer(quantizer); +// ASSERT_FALSE(status.ok()); +// fiu_disable("IVFHybridIndex.SetQuantizer.throw_knowhere_exception"); - fiu_enable("IVFHybridIndex.SetQuantizer.throw_std_exception", 1, nullptr, 0); - status = index_->SetQuantizer(quantizer); - ASSERT_FALSE(status.ok()); - fiu_disable("IVFHybridIndex.SetQuantizer.throw_std_exception"); +// fiu_enable("IVFHybridIndex.SetQuantizer.throw_std_exception", 1, nullptr, 0); +// status = index_->SetQuantizer(quantizer); +// ASSERT_FALSE(status.ok()); +// fiu_disable("IVFHybridIndex.SetQuantizer.throw_std_exception"); - status = index_->UnsetQuantizer(); - ASSERT_FALSE(status.ok()); - fiu_enable("IVFHybridIndex.UnsetQuantizer.throw_knowhere_exception", 1, nullptr, 0); - status = index_->UnsetQuantizer(); - ASSERT_FALSE(status.ok()); - fiu_disable("IVFHybridIndex.UnsetQuantizer.throw_knowhere_exception"); +// status = index_->UnsetQuantizer(); +// ASSERT_FALSE(status.ok()); +// fiu_enable("IVFHybridIndex.UnsetQuantizer.throw_knowhere_exception", 1, nullptr, 0); +// status = index_->UnsetQuantizer(); +// ASSERT_FALSE(status.ok()); +// fiu_disable("IVFHybridIndex.UnsetQuantizer.throw_knowhere_exception"); - fiu_enable("IVFHybridIndex.UnsetQuantizer.throw_std_exception", 1, nullptr, 0); - status = index_->UnsetQuantizer(); - ASSERT_FALSE(status.ok()); - fiu_disable("IVFHybridIndex.UnsetQuantizer.throw_std_exception"); +// fiu_enable("IVFHybridIndex.UnsetQuantizer.throw_std_exception", 1, nullptr, 0); +// status = index_->UnsetQuantizer(); +// ASSERT_FALSE(status.ok()); +// fiu_disable("IVFHybridIndex.UnsetQuantizer.throw_std_exception"); - auto vecindex = index_->LoadData(quantizer, quantizer_cfg); - ASSERT_EQ(vecindex, nullptr); +// auto vecindex = index_->LoadData(quantizer, quantizer_cfg); +// ASSERT_EQ(vecindex, nullptr); - fiu_enable("IVFHybridIndex.LoadData.throw_std_exception", 1, nullptr, 0); - vecindex = index_->LoadData(quantizer, quantizer_cfg); - ASSERT_EQ(vecindex, nullptr); - fiu_disable("IVFHybridIndex.LoadData.throw_std_exception"); +// fiu_enable("IVFHybridIndex.LoadData.throw_std_exception", 1, nullptr, 0); +// vecindex = index_->LoadData(quantizer, quantizer_cfg); +// ASSERT_EQ(vecindex, nullptr); +// fiu_disable("IVFHybridIndex.LoadData.throw_std_exception"); - fiu_enable("IVFHybridIndex.LoadData.throw_knowhere_exception", 1, nullptr, 0); - vecindex = index_->LoadData(quantizer, quantizer_cfg); - ASSERT_EQ(vecindex, nullptr); - fiu_disable("IVFHybridIndex.LoadData.throw_knowhere_exception"); +// fiu_enable("IVFHybridIndex.LoadData.throw_knowhere_exception", 1, nullptr, 0); +// vecindex = index_->LoadData(quantizer, quantizer_cfg); +// ASSERT_EQ(vecindex, nullptr); +// fiu_disable("IVFHybridIndex.LoadData.throw_knowhere_exception"); - index_->CopyToGpuWithQuantizer(DEVICEID, conf); - fiu_enable("IVFHybridIndex.LoadData.throw_knowhere_exception", 1, nullptr, 0); - index_->CopyToGpuWithQuantizer(DEVICEID, conf); - fiu_disable("IVFHybridIndex.LoadData.throw_knowhere_exception"); - fiu_enable("IVFHybridIndex.LoadData.throw_std_exception", 1, nullptr, 0); - index_->CopyToGpuWithQuantizer(DEVICEID, conf); - fiu_disable("IVFHybridIndex.LoadData.throw_std_exception"); -} +// index_->CopyToGpuWithQuantizer(DEVICEID, conf); +// fiu_enable("IVFHybridIndex.LoadData.throw_knowhere_exception", 1, nullptr, 0); +// index_->CopyToGpuWithQuantizer(DEVICEID, conf); +// fiu_disable("IVFHybridIndex.LoadData.throw_knowhere_exception"); +// fiu_enable("IVFHybridIndex.LoadData.throw_std_exception", 1, nullptr, 0); +// index_->CopyToGpuWithQuantizer(DEVICEID, conf); +// fiu_disable("IVFHybridIndex.LoadData.throw_std_exception"); +// } #endif diff --git a/core/unittest/wrapper/test_wrapper.cpp b/core/unittest/wrapper/test_wrapper.cpp index 0320aa3dcd..ba3ed0f1a1 100644 --- a/core/unittest/wrapper/test_wrapper.cpp +++ b/core/unittest/wrapper/test_wrapper.cpp @@ -45,13 +45,13 @@ class KnowhereWrapperTest std::tie(index_type, generator_type, dim, nb, nq, k) = GetParam(); GenData(dim, nb, nq, xb, xq, ids, k, gt_ids, gt_dis); - milvus::engine::TempMetaConf tempconf; - tempconf.metric_type = knowhere::METRICTYPE::L2; - tempconf.gpu_id = DEVICEID; - tempconf.size = nb; - tempconf.dim = dim; - tempconf.k = k; - tempconf.nprobe = 16; + knowhere::Config tempconf{ + {knowhere::Metric::TYPE, knowhere::Metric::L2}, + {knowhere::meta::ROWS, nb}, + {knowhere::meta::DIM, dim}, + {knowhere::meta::TOPK, k}, + {knowhere::meta::DEVICEID, DEVICEID} + }; index_ = GetVecIndexFactory(index_type); conf = ParamGenerator::GetInstance().GenBuild(index_type, tempconf); @@ -104,7 +104,6 @@ TEST_P(KnowhereWrapperTest, WRAPPER_EXCEPTION_TEST) { TEST_P(KnowhereWrapperTest, BASE_TEST) { EXPECT_EQ(index_->GetType(), index_type); - // conf->Dump(); auto elems = nq * k; std::vector res_ids(elems); @@ -249,69 +248,69 @@ TEST_P(KnowhereWrapperTest, SERIALIZE_TEST) { } } -#include "wrapper/ConfAdapter.h" +// #include "wrapper/ConfAdapter.h" -TEST(whatever, test_config) { - milvus::engine::TempMetaConf conf; - conf.nprobe = 16; - conf.dim = 128; - auto nsg_conf = std::make_shared(); - nsg_conf->Match(conf); - nsg_conf->MatchSearch(conf, milvus::engine::IndexType::NSG_MIX); +// TEST(whatever, test_config) { +// milvus::engine::TempMetaConf conf; +// conf.nprobe = 16; +// conf.dim = 128; +// auto nsg_conf = std::make_shared(); +// nsg_conf->Match(conf); +// nsg_conf->MatchSearch(conf, milvus::engine::IndexType::NSG_MIX); - auto pq_conf = std::make_shared(); - pq_conf->Match(conf); - pq_conf->MatchSearch(conf, milvus::engine::IndexType::FAISS_IVFPQ_MIX); +// auto pq_conf = std::make_shared(); +// pq_conf->Match(conf); +// pq_conf->MatchSearch(conf, milvus::engine::IndexType::FAISS_IVFPQ_MIX); - auto kdt_conf = std::make_shared(); - kdt_conf->Match(conf); - kdt_conf->MatchSearch(conf, milvus::engine::IndexType::SPTAG_KDT_RNT_CPU); +// auto kdt_conf = std::make_shared(); +// kdt_conf->Match(conf); +// kdt_conf->MatchSearch(conf, milvus::engine::IndexType::SPTAG_KDT_RNT_CPU); - auto bkt_conf = std::make_shared(); - bkt_conf->Match(conf); - bkt_conf->MatchSearch(conf, milvus::engine::IndexType::SPTAG_BKT_RNT_CPU); +// auto bkt_conf = std::make_shared(); +// bkt_conf->Match(conf); +// bkt_conf->MatchSearch(conf, milvus::engine::IndexType::SPTAG_BKT_RNT_CPU); - auto config_mgr = milvus::engine::AdapterMgr::GetInstance(); - try { - config_mgr.GetAdapter(milvus::engine::IndexType::INVALID); - } catch (std::exception& e) { - std::cout << "catch an expected exception" << std::endl; - } +// auto config_mgr = milvus::engine::AdapterMgr::GetInstance(); +// try { +// config_mgr.GetAdapter(milvus::engine::IndexType::INVALID); +// } catch (std::exception& e) { +// std::cout << "catch an expected exception" << std::endl; +// } - conf.size = 1000000.0; - conf.nlist = 10; - auto ivf_conf = std::make_shared(); - ivf_conf->Match(conf); - conf.nprobe = -1; - ivf_conf->MatchSearch(conf, milvus::engine::IndexType::FAISS_IVFFLAT_GPU); - conf.nprobe = 4096; - ivf_conf->MatchSearch(conf, milvus::engine::IndexType::FAISS_IVFPQ_GPU); +// conf.size = 1000000.0; +// conf.nlist = 10; +// auto ivf_conf = std::make_shared(); +// ivf_conf->Match(conf); +// conf.nprobe = -1; +// ivf_conf->MatchSearch(conf, milvus::engine::IndexType::FAISS_IVFFLAT_GPU); +// conf.nprobe = 4096; +// ivf_conf->MatchSearch(conf, milvus::engine::IndexType::FAISS_IVFPQ_GPU); - auto ivf_pq_conf = std::make_shared(); - conf.metric_type = knowhere::METRICTYPE::IP; - try { - ivf_pq_conf->Match(conf); - } catch (std::exception& e) { - std::cout << "catch an expected exception" << std::endl; - } +// auto ivf_pq_conf = std::make_shared(); +// conf.metric_type = knowhere::METRICTYPE::IP; +// try { +// ivf_pq_conf->Match(conf); +// } catch (std::exception& e) { +// std::cout << "catch an expected exception" << std::endl; +// } - conf.metric_type = knowhere::METRICTYPE::L2; - fiu_init(0); - fiu_enable("IVFPQConfAdapter.Match.empty_resset", 1, NULL, 0); - try { - ivf_pq_conf->Match(conf); - } catch (std::exception& e) { - std::cout << "catch an expected exception" << std::endl; - } - fiu_disable("IVFPQConfAdapter.Match.empty_resset"); +// conf.metric_type = knowhere::METRICTYPE::L2; +// fiu_init(0); +// fiu_enable("IVFPQConfAdapter.Match.empty_resset", 1, NULL, 0); +// try { +// ivf_pq_conf->Match(conf); +// } catch (std::exception& e) { +// std::cout << "catch an expected exception" << std::endl; +// } +// fiu_disable("IVFPQConfAdapter.Match.empty_resset"); - conf.nprobe = -1; - try { - ivf_pq_conf->MatchSearch(conf, milvus::engine::IndexType::FAISS_IVFPQ_GPU); - } catch (std::exception& e) { - std::cout << "catch an expected exception" << std::endl; - } -} +// conf.nprobe = -1; +// try { +// ivf_pq_conf->MatchSearch(conf, milvus::engine::IndexType::FAISS_IVFPQ_GPU); +// } catch (std::exception& e) { +// std::cout << "catch an expected exception" << std::endl; +// } +// } #include "wrapper/VecImpl.h" diff --git a/core/unittest/wrapper/utils.h b/core/unittest/wrapper/utils.h index a50e155e22..527d2c306f 100644 --- a/core/unittest/wrapper/utils.h +++ b/core/unittest/wrapper/utils.h @@ -9,21 +9,21 @@ // is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express // or implied. See the License for the specific language governing permissions and limitations under the License. - #pragma once #include + +#include +#include +#include #include #include -#include -#include -#include +#include "knowhere/index/vector_index/helpers/IndexParameter.h" +#include "wrapper/ConfAdapter.h" +#include "wrapper/ConfAdapterMgr.h" #include "wrapper/VecIndex.h" #include "wrapper/utils.h" -#include "knowhere/index/vector_index/helpers/IndexParameter.h" -#include "wrapper/ConfAdapterMgr.h" -#include "wrapper/ConfAdapter.h" class DataGenBase; @@ -37,31 +37,29 @@ constexpr int64_t PINMEM = 1024 * 1024 * 200; constexpr int64_t TEMPMEM = 1024 * 1024 * 300; constexpr int64_t RESNUM = 2; -static const char *CONFIG_PATH = "/tmp/milvus_test"; -static const char *CONFIG_FILE = "/server_config.yaml"; +static const char* CONFIG_PATH = "/tmp/milvus_test"; +static const char* CONFIG_FILE = "/server_config.yaml"; class KnowhereTest : public ::testing::Test { protected: - void SetUp() override; - void TearDown() override; + void + SetUp() override; + void + TearDown() override; }; class DataGenBase { public: - virtual void GenData(const int& dim, const int& nb, const int& nq, float* xb, float* xq, int64_t* ids, - const int& k, int64_t* gt_ids, float* gt_dis); + virtual void + GenData(const int& dim, const int& nb, const int& nq, float* xb, float* xq, int64_t* ids, const int& k, + int64_t* gt_ids, float* gt_dis); - virtual void GenData(const int& dim, - const int& nb, - const int& nq, - std::vector& xb, - std::vector& xq, - std::vector& ids, - const int& k, - std::vector& gt_ids, - std::vector& gt_dis); + virtual void + GenData(const int& dim, const int& nb, const int& nq, std::vector& xb, std::vector& xq, + std::vector& ids, const int& k, std::vector& gt_ids, std::vector& gt_dis); - void AssertResult(const std::vector& ids, const std::vector& dis); + void + AssertResult(const std::vector& ids, const std::vector& dis); int dim = DIM; int nb = NB; @@ -78,22 +76,19 @@ class DataGenBase { class BinDataGen { public: - virtual void GenData(const int& dim, const int& nb, const int& nq, uint8_t* xb, uint8_t* xq, int64_t* ids, - const int& k, int64_t* gt_ids, float* gt_dis); + virtual void + GenData(const int& dim, const int& nb, const int& nq, uint8_t* xb, uint8_t* xq, int64_t* ids, const int& k, + int64_t* gt_ids, float* gt_dis); - virtual void GenData(const int& dim, - const int& nb, - const int& nq, - std::vector& xb, - std::vector& xq, - std::vector& ids, - const int& k, - std::vector& gt_ids, - std::vector& gt_dis); + virtual void + GenData(const int& dim, const int& nb, const int& nq, std::vector& xb, std::vector& xq, + std::vector& ids, const int& k, std::vector& gt_ids, std::vector& gt_dis); - void AssertResult(const std::vector& ids, const std::vector& dis); + void + AssertResult(const std::vector& ids, const std::vector& dis); - void Generate(const int& dim, const int& nb, const int& nq, const int& k); + void + Generate(const int& dim, const int& nb, const int& nq, const int& k); int dim = DIM; int nb = NB; @@ -110,82 +105,95 @@ class BinDataGen { class ParamGenerator { public: - static ParamGenerator& GetInstance() { + static ParamGenerator& + GetInstance() { static ParamGenerator instance; return instance; } knowhere::Config - GenSearchConf(const milvus::engine::IndexType& type, const milvus::engine::TempMetaConf& conf) { - auto adapter = milvus::engine::AdapterMgr::GetInstance().GetAdapter(type); - return adapter->MatchSearch(conf, type); - } - - knowhere::Config - GenBuild(const milvus::engine::IndexType& type, const milvus::engine::TempMetaConf& conf) { - auto adapter = milvus::engine::AdapterMgr::GetInstance().GetAdapter(type); - return adapter->Match(conf); - } - - knowhere::Config - Gen(const milvus::engine::IndexType& type) { + GenSearchConf(const milvus::engine::IndexType& type, const milvus::json& conf) { + auto search_cfg = conf; switch (type) { case milvus::engine::IndexType::FAISS_IDMAP: { - auto tempconf = std::make_shared(); - tempconf->metric_type = knowhere::METRICTYPE::L2; - return tempconf; + break; } + case milvus::engine::IndexType::FAISS_BIN_IVFLAT_CPU: case milvus::engine::IndexType::FAISS_IVFFLAT_CPU: case milvus::engine::IndexType::FAISS_IVFFLAT_GPU: - case milvus::engine::IndexType::FAISS_IVFFLAT_MIX: { - auto tempconf = std::make_shared(); - tempconf->nlist = 100; - tempconf->nprobe = 16; - tempconf->metric_type = knowhere::METRICTYPE::L2; - return tempconf; - } + case milvus::engine::IndexType::FAISS_IVFFLAT_MIX: + case milvus::engine::IndexType::FAISS_IVFPQ_CPU: + case milvus::engine::IndexType::FAISS_IVFPQ_GPU: + case milvus::engine::IndexType::FAISS_IVFPQ_MIX: case milvus::engine::IndexType::FAISS_IVFSQ8_HYBRID: case milvus::engine::IndexType::FAISS_IVFSQ8_CPU: case milvus::engine::IndexType::FAISS_IVFSQ8_GPU: case milvus::engine::IndexType::FAISS_IVFSQ8_MIX: { - auto tempconf = std::make_shared(); - tempconf->nlist = 100; - tempconf->nprobe = 16; - tempconf->nbits = 8; - tempconf->metric_type = knowhere::METRICTYPE::L2; - return tempconf; - } - case milvus::engine::IndexType::FAISS_IVFPQ_CPU: - case milvus::engine::IndexType::FAISS_IVFPQ_GPU: - case milvus::engine::IndexType::FAISS_IVFPQ_MIX: { - auto tempconf = std::make_shared(); - tempconf->nlist = 100; - tempconf->nprobe = 16; - tempconf->nbits = 8; - tempconf->m = 8; - tempconf->metric_type = knowhere::METRICTYPE::L2; - return tempconf; + search_cfg[knowhere::IndexParams::nprobe] = 16; + break; } case milvus::engine::IndexType::NSG_MIX: { - auto tempconf = std::make_shared(); - tempconf->nlist = 100; - tempconf->nprobe = 16; - tempconf->search_length = 8; - tempconf->knng = 200; - tempconf->search_length = 40; // TODO(linxj): be 20 when search - tempconf->out_degree = 60; - tempconf->candidate_pool_size = 200; - tempconf->metric_type = knowhere::METRICTYPE::L2; - return tempconf; + search_cfg[knowhere::IndexParams::search_length] = 20; + break; + } + case milvus::engine::IndexType::HNSW: { + search_cfg[knowhere::IndexParams::ef] = conf[knowhere::meta::TOPK].get() + 10; + break; } } + auto adapter = milvus::engine::AdapterMgr::GetInstance().GetAdapter(type); + adapter->CheckSearch(search_cfg, type); + return search_cfg; + } + + knowhere::Config + GenBuild(const milvus::engine::IndexType& type, const milvus::json& conf) { + auto build_cfg = conf; + switch (type) { + case milvus::engine::IndexType::FAISS_IDMAP: { + case milvus::engine::IndexType::FAISS_BIN_IDMAP: { + break; + } + case milvus::engine::IndexType::FAISS_BIN_IVFLAT_CPU: + case milvus::engine::IndexType::FAISS_IVFFLAT_CPU: + case milvus::engine::IndexType::FAISS_IVFFLAT_GPU: + case milvus::engine::IndexType::FAISS_IVFFLAT_MIX: + case milvus::engine::IndexType::FAISS_IVFSQ8_HYBRID: + case milvus::engine::IndexType::FAISS_IVFSQ8_CPU: + case milvus::engine::IndexType::FAISS_IVFSQ8_GPU: + case milvus::engine::IndexType::FAISS_IVFSQ8_MIX: { + build_cfg[knowhere::IndexParams::nlist] = 16; + break; + } + case milvus::engine::IndexType::FAISS_IVFPQ_CPU: + case milvus::engine::IndexType::FAISS_IVFPQ_GPU: + case milvus::engine::IndexType::FAISS_IVFPQ_MIX: { + build_cfg[knowhere::IndexParams::nlist] = 16; + build_cfg[knowhere::IndexParams::m] = 8; + break; + } + case milvus::engine::IndexType::NSG_MIX: { + build_cfg[knowhere::IndexParams::knng] = 10; + build_cfg[knowhere::IndexParams::search_length] = 20; + build_cfg[knowhere::IndexParams::out_degree] = 10; + build_cfg[knowhere::IndexParams::candidate] = 50; + break; + } + case milvus::engine::IndexType::HNSW: { + build_cfg[knowhere::IndexParams::efConstruction] = 100; + build_cfg[knowhere::IndexParams::M] = 12; + break; + } + } + } + auto adapter = milvus::engine::AdapterMgr::GetInstance().GetAdapter(type); + adapter->CheckTrain(build_cfg); + return build_cfg; } }; - -//class SanityCheck : public DataGenBase { +// class SanityCheck : public DataGenBase { // public: // void GenData(const int &dim, const int &nb, const int &nq, float *xb, float *xq, long *ids, // const int &k, long *gt_ids, float *gt_dis) override; //}; - diff --git a/sdk/examples/binary_vector/src/ClientTest.cpp b/sdk/examples/binary_vector/src/ClientTest.cpp index a03b0634f0..f6dbd4b8d2 100644 --- a/sdk/examples/binary_vector/src/ClientTest.cpp +++ b/sdk/examples/binary_vector/src/ClientTest.cpp @@ -9,8 +9,8 @@ // is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express // or implied. See the License for the specific language governing permissions and limitations under the License. -#include "examples/binary_vector/src/ClientTest.h" #include "include/MilvusApi.h" +#include "examples/binary_vector/src/ClientTest.h" #include "examples/utils/TimeRecorder.h" #include "examples/utils/Utils.h" @@ -34,7 +34,6 @@ constexpr int64_t NPROBE = 32; constexpr int64_t SEARCH_TARGET = 5000; // change this value, result is different, ensure less than BATCH_ROW_COUNT constexpr int64_t ADD_VECTOR_LOOP = 20; constexpr milvus::IndexType INDEX_TYPE = milvus::IndexType::IVFFLAT; -constexpr int32_t N_LIST = 1024; milvus::TableSchema BuildTableSchema() { @@ -44,7 +43,8 @@ BuildTableSchema() { milvus::IndexParam BuildIndexParam() { - milvus::IndexParam index_param = {TABLE_NAME, INDEX_TYPE, N_LIST}; + JSON json_params = {{"nlist", 1024}}; + milvus::IndexParam index_param = {TABLE_NAME, INDEX_TYPE, json_params.dump()}; return index_param; } @@ -123,7 +123,11 @@ ClientTest::Test(const std::string& address, const std::string& port) { } } - milvus_sdk::Utils::Sleep(3); + { // flush buffer + stat = conn->FlushTable(TABLE_NAME); + std::cout << "FlushTable function call status: " << stat.message() << std::endl; + } + { // search vectors std::vector partition_tags; milvus::TopKQueryResult topk_query_result; diff --git a/sdk/examples/partition/src/ClientTest.cpp b/sdk/examples/partition/src/ClientTest.cpp index 31b0207363..aca815c6ed 100644 --- a/sdk/examples/partition/src/ClientTest.cpp +++ b/sdk/examples/partition/src/ClientTest.cpp @@ -9,8 +9,8 @@ // is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express // or implied. See the License for the specific language governing permissions and limitations under the License. -#include "examples/partition/src/ClientTest.h" #include "include/MilvusApi.h" +#include "examples/partition/src/ClientTest.h" #include "examples/utils/TimeRecorder.h" #include "examples/utils/Utils.h" @@ -33,7 +33,6 @@ constexpr int64_t TOP_K = 10; constexpr int64_t NPROBE = 32; constexpr int64_t SEARCH_TARGET = 5000; // change this value, result is different constexpr milvus::IndexType INDEX_TYPE = milvus::IndexType::IVFSQ8; -constexpr int32_t N_LIST = 16384; constexpr int32_t PARTITION_COUNT = 5; constexpr int32_t TARGET_PARTITION = 3; @@ -53,7 +52,8 @@ BuildPartitionParam(int32_t index) { milvus::IndexParam BuildIndexParam() { - milvus::IndexParam index_param = {TABLE_NAME, INDEX_TYPE, N_LIST}; + JSON json_params = {{"nlist", 16384}}; + milvus::IndexParam index_param = {TABLE_NAME, INDEX_TYPE, json_params.dump()}; return index_param; } diff --git a/sdk/examples/simple/src/ClientTest.cpp b/sdk/examples/simple/src/ClientTest.cpp index b7427cdd41..635dabdc16 100644 --- a/sdk/examples/simple/src/ClientTest.cpp +++ b/sdk/examples/simple/src/ClientTest.cpp @@ -9,6 +9,9 @@ // is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express // or implied. See the License for the specific language governing permissions and limitations under the License. +#include "include/MilvusApi.h" +#include "examples/utils/TimeRecorder.h" +#include "examples/utils/Utils.h" #include "examples/simple/src/ClientTest.h" #include @@ -16,9 +19,6 @@ #include #include -#include "examples/utils/TimeRecorder.h" -#include "examples/utils/Utils.h" -#include "include/MilvusApi.h" namespace { @@ -34,7 +34,7 @@ constexpr int64_t NPROBE = 32; constexpr int64_t SEARCH_TARGET = 5000; // change this value, result is different constexpr int64_t ADD_VECTOR_LOOP = 5; constexpr milvus::IndexType INDEX_TYPE = milvus::IndexType::IVFSQ8; -constexpr int32_t N_LIST = 16384; +constexpr int32_t NLIST = 16384; } // namespace @@ -159,18 +159,12 @@ ClientTest::SearchVectors(const std::string& table_name, int64_t topk, int64_t n topk_query_result); } -void -ClientTest::SearchVectorsByIds(const std::string& table_name, int64_t topk, int64_t nprobe) { - std::vector partition_tags; - milvus::TopKQueryResult topk_query_result; - milvus_sdk::Utils::DoSearch(conn_, table_name, partition_tags, topk, nprobe, search_id_array_, topk_query_result); -} - void ClientTest::CreateIndex(const std::string& table_name, milvus::IndexType type, int64_t nlist) { milvus_sdk::TimeRecorder rc("Create index"); std::cout << "Wait until create all index done" << std::endl; - milvus::IndexParam index1 = {table_name, type, nlist}; + JSON json_params = {{"nlist", nlist}}; + milvus::IndexParam index1 = {table_name, type, json_params.dump()}; milvus_sdk::Utils::PrintIndexParam(index1); milvus::Status stat = conn_->CreateIndex(index1); std::cout << "CreateIndex function call status: " << stat.message() << std::endl; @@ -244,9 +238,8 @@ ClientTest::Test() { GetVectorById(table_name, search_id_array_[0]); SearchVectors(table_name, TOP_K, NPROBE); - SearchVectorsByIds(table_name, TOP_K, NPROBE); - CreateIndex(table_name, INDEX_TYPE, N_LIST); + CreateIndex(table_name, INDEX_TYPE, NLIST); ShowTableInfo(table_name); PreloadTable(table_name); @@ -257,4 +250,4 @@ ClientTest::Test() { DropIndex(table_name); DropTable(table_name); -} \ No newline at end of file +} diff --git a/sdk/examples/simple/src/ClientTest.h b/sdk/examples/simple/src/ClientTest.h index 52fdbe7b72..0c62b29729 100644 --- a/sdk/examples/simple/src/ClientTest.h +++ b/sdk/examples/simple/src/ClientTest.h @@ -12,6 +12,9 @@ #pragma once #include +#include +#include +#include #include @@ -26,36 +29,49 @@ class ClientTest { private: void ShowServerVersion(); + void ShowSdkVersion(); + void ShowTables(std::vector&); + void CreateTable(const std::string&, int64_t, milvus::MetricType); + void DescribeTable(const std::string&); + void InsertVectors(const std::string&, int64_t); + void BuildSearchVectors(int64_t, int64_t); + void Flush(const std::string&); + void ShowTableInfo(const std::string&); + void GetVectorById(const std::string&, int64_t); + void SearchVectors(const std::string&, int64_t, int64_t); - void - SearchVectorsByIds(const std::string&, int64_t, int64_t); + void CreateIndex(const std::string&, milvus::IndexType, int64_t); + void PreloadTable(const std::string&); + void DeleteByIds(const std::string&, const std::vector&); + void DropIndex(const std::string&); + void DropTable(const std::string&); diff --git a/sdk/examples/utils/Utils.cpp b/sdk/examples/utils/Utils.cpp index 8bc80c1dc2..c9294cb3aa 100644 --- a/sdk/examples/utils/Utils.cpp +++ b/sdk/examples/utils/Utils.cpp @@ -72,42 +72,27 @@ Utils::GenTableName() { std::string Utils::MetricTypeName(const milvus::MetricType& metric_type) { switch (metric_type) { - case milvus::MetricType::L2: - return "L2 distance"; - case milvus::MetricType::IP: - return "Inner product"; - case milvus::MetricType::HAMMING: - return "Hamming distance"; - case milvus::MetricType::JACCARD: - return "Jaccard distance"; - case milvus::MetricType::TANIMOTO: - return "Tanimoto distance"; - default: - return "Unknown metric type"; + case milvus::MetricType::L2:return "L2 distance"; + case milvus::MetricType::IP:return "Inner product"; + case milvus::MetricType::HAMMING:return "Hamming distance"; + case milvus::MetricType::JACCARD:return "Jaccard distance"; + case milvus::MetricType::TANIMOTO:return "Tanimoto distance"; + default:return "Unknown metric type"; } } std::string Utils::IndexTypeName(const milvus::IndexType& index_type) { switch (index_type) { - case milvus::IndexType::FLAT: - return "FLAT"; - case milvus::IndexType::IVFFLAT: - return "IVFFLAT"; - case milvus::IndexType::IVFSQ8: - return "IVFSQ8"; - case milvus::IndexType::RNSG: - return "NSG"; - case milvus::IndexType::IVFSQ8H: - return "IVFSQ8H"; - case milvus::IndexType::IVFPQ: - return "IVFPQ"; - case milvus::IndexType::SPTAGKDT: - return "SPTAGKDT"; - case milvus::IndexType::SPTAGBKT: - return "SPTAGBKT"; - default: - return "Unknown index type"; + case milvus::IndexType::FLAT:return "FLAT"; + case milvus::IndexType::IVFFLAT:return "IVFFLAT"; + case milvus::IndexType::IVFSQ8:return "IVFSQ8"; + case milvus::IndexType::RNSG:return "NSG"; + case milvus::IndexType::IVFSQ8H:return "IVFSQ8H"; + case milvus::IndexType::IVFPQ:return "IVFPQ"; + case milvus::IndexType::SPTAGKDT:return "SPTAGKDT"; + case milvus::IndexType::SPTAGBKT:return "SPTAGBKT"; + default:return "Unknown index type"; } } @@ -134,7 +119,7 @@ Utils::PrintIndexParam(const milvus::IndexParam& index_param) { BLOCK_SPLITER std::cout << "Index table name: " << index_param.table_name << std::endl; std::cout << "Index type: " << IndexTypeName(index_param.index_type) << std::endl; - std::cout << "Index nlist: " << index_param.nlist << std::endl; + std::cout << "Index extra_params: " << index_param.extra_params << std::endl; BLOCK_SPLITER } @@ -166,7 +151,7 @@ Utils::PrintSearchResult(const std::vector std::cout << "Returned result count: " << topk_query_result.size() << std::endl; if (topk_query_result.size() != search_record_array.size()) { - std::cout << "ERROR: Returned result count dones equal nq" << std::endl; + std::cout << "ERROR: Returned result count not equal nq" << std::endl; return; } @@ -223,9 +208,10 @@ Utils::DoSearch(std::shared_ptr conn, const std::string& tab { BLOCK_SPLITER + JSON json_params = {{"nprobe", nprobe}}; milvus_sdk::TimeRecorder rc("search"); milvus::Status stat = - conn->Search(table_name, partition_tags, record_array, top_k, nprobe, topk_query_result); + conn->Search(table_name, partition_tags, record_array, top_k, json_params.dump(), topk_query_result); std::cout << "SearchVector function call status: " << stat.message() << std::endl; BLOCK_SPLITER } @@ -234,66 +220,6 @@ Utils::DoSearch(std::shared_ptr conn, const std::string& tab CheckSearchResult(search_record_array, topk_query_result); } -void -Utils::DoSearch(std::shared_ptr conn, const std::string& table_name, - const std::vector& partition_tags, int64_t top_k, int64_t nprobe, - const std::vector& search_id_array, milvus::TopKQueryResult& topk_query_result) { - topk_query_result.clear(); - - { - BLOCK_SPLITER - for (auto& search_id : search_id_array) { - milvus_sdk::TimeRecorder rc("search by id " + std::to_string(search_id)); - milvus::TopKQueryResult result; - milvus::Status stat = conn->SearchByID(table_name, partition_tags, search_id, top_k, nprobe, result); - topk_query_result.insert(topk_query_result.end(), std::make_move_iterator(result.begin()), - std::make_move_iterator(result.end())); - std::cout << "SearchByID function call status: " << stat.message() << std::endl; - } - BLOCK_SPLITER - } - - if (topk_query_result.size() != search_id_array.size()) { - std::cout << "ERROR: Returned result count does not equal nq" << std::endl; - return; - } - - BLOCK_SPLITER - for (size_t i = 0; i < topk_query_result.size(); i++) { - const milvus::QueryResult& one_result = topk_query_result[i]; - size_t topk = one_result.ids.size(); - auto search_id = search_id_array[i]; - std::cout << "No." << i << " vector " << search_id << " top " << topk << " search result:" << std::endl; - for (size_t j = 0; j < topk; j++) { - std::cout << "\t" << one_result.ids[j] << "\t" << one_result.distances[j] << std::endl; - } - } - BLOCK_SPLITER - - BLOCK_SPLITER - size_t nq = topk_query_result.size(); - for (size_t i = 0; i < nq; i++) { - const milvus::QueryResult& one_result = topk_query_result[i]; - auto search_id = search_id_array[i]; - - uint64_t match_index = one_result.ids.size(); - for (uint64_t index = 0; index < one_result.ids.size(); index++) { - if (search_id == one_result.ids[index]) { - match_index = index; - break; - } - } - - if (match_index >= one_result.ids.size()) { - std::cout << "The topk result is wrong: not return search target in result set" << std::endl; - } else { - std::cout << "No." << i << " Check result successfully for target: " << search_id << " at top " - << match_index << std::endl; - } - } - BLOCK_SPLITER -} - void PrintPartitionStat(const milvus::PartitionStat& partition_stat) { std::cout << "\tPartition " << partition_stat.tag << " row count: " << partition_stat.row_count << std::endl; diff --git a/sdk/examples/utils/Utils.h b/sdk/examples/utils/Utils.h index c9ce5983cf..4f33dafc33 100644 --- a/sdk/examples/utils/Utils.h +++ b/sdk/examples/utils/Utils.h @@ -12,12 +12,15 @@ #pragma once #include "MilvusApi.h" +#include "thirdparty/nlohmann/json.hpp" #include #include #include #include +using JSON = nlohmann::json; + namespace milvus_sdk { class Utils { @@ -67,12 +70,6 @@ class Utils { const std::vector>& search_record_array, milvus::TopKQueryResult& topk_query_result); - static void - DoSearch(std::shared_ptr conn, const std::string& table_name, - const std::vector& partition_tags, int64_t top_k, int64_t nprobe, - const std::vector& search_id_array, - milvus::TopKQueryResult& topk_query_result); - static void PrintTableInfo(const milvus::TableInfo& info); }; diff --git a/sdk/grpc-gen/gen-milvus/milvus.pb.cc b/sdk/grpc-gen/gen-milvus/milvus.pb.cc index 13ebbffb8a..3886e57da1 100644 --- a/sdk/grpc-gen/gen-milvus/milvus.pb.cc +++ b/sdk/grpc-gen/gen-milvus/milvus.pb.cc @@ -15,14 +15,18 @@ #include // @@protoc_insertion_point(includes) #include -extern PROTOBUF_INTERNAL_EXPORT_milvus_2eproto ::PROTOBUF_NAMESPACE_ID::internal::SCCInfo<0> scc_info_Index_milvus_2eproto; +extern PROTOBUF_INTERNAL_EXPORT_milvus_2eproto ::PROTOBUF_NAMESPACE_ID::internal::SCCInfo<0> scc_info_KeyValuePair_milvus_2eproto; extern PROTOBUF_INTERNAL_EXPORT_milvus_2eproto ::PROTOBUF_NAMESPACE_ID::internal::SCCInfo<1> scc_info_PartitionStat_milvus_2eproto; extern PROTOBUF_INTERNAL_EXPORT_milvus_2eproto ::PROTOBUF_NAMESPACE_ID::internal::SCCInfo<0> scc_info_RowRecord_milvus_2eproto; -extern PROTOBUF_INTERNAL_EXPORT_milvus_2eproto ::PROTOBUF_NAMESPACE_ID::internal::SCCInfo<1> scc_info_SearchParam_milvus_2eproto; +extern PROTOBUF_INTERNAL_EXPORT_milvus_2eproto ::PROTOBUF_NAMESPACE_ID::internal::SCCInfo<2> scc_info_SearchParam_milvus_2eproto; extern PROTOBUF_INTERNAL_EXPORT_milvus_2eproto ::PROTOBUF_NAMESPACE_ID::internal::SCCInfo<0> scc_info_SegmentStat_milvus_2eproto; extern PROTOBUF_INTERNAL_EXPORT_status_2eproto ::PROTOBUF_NAMESPACE_ID::internal::SCCInfo<0> scc_info_Status_status_2eproto; namespace milvus { namespace grpc { +class KeyValuePairDefaultTypeInternal { + public: + ::PROTOBUF_NAMESPACE_ID::internal::ExplicitlyConstructed _instance; +} _KeyValuePair_default_instance_; class TableNameDefaultTypeInternal { public: ::PROTOBUF_NAMESPACE_ID::internal::ExplicitlyConstructed _instance; @@ -87,10 +91,6 @@ class CommandDefaultTypeInternal { public: ::PROTOBUF_NAMESPACE_ID::internal::ExplicitlyConstructed _instance; } _Command_default_instance_; -class IndexDefaultTypeInternal { - public: - ::PROTOBUF_NAMESPACE_ID::internal::ExplicitlyConstructed _instance; -} _Index_default_instance_; class IndexParamDefaultTypeInternal { public: ::PROTOBUF_NAMESPACE_ID::internal::ExplicitlyConstructed _instance; @@ -200,20 +200,6 @@ static void InitDefaultsscc_info_GetVectorIDsParam_milvus_2eproto() { ::PROTOBUF_NAMESPACE_ID::internal::SCCInfo<0> scc_info_GetVectorIDsParam_milvus_2eproto = {{ATOMIC_VAR_INIT(::PROTOBUF_NAMESPACE_ID::internal::SCCInfoBase::kUninitialized), 0, InitDefaultsscc_info_GetVectorIDsParam_milvus_2eproto}, {}}; -static void InitDefaultsscc_info_Index_milvus_2eproto() { - GOOGLE_PROTOBUF_VERIFY_VERSION; - - { - void* ptr = &::milvus::grpc::_Index_default_instance_; - new (ptr) ::milvus::grpc::Index(); - ::PROTOBUF_NAMESPACE_ID::internal::OnShutdownDestroyMessage(ptr); - } - ::milvus::grpc::Index::InitAsDefaultInstance(); -} - -::PROTOBUF_NAMESPACE_ID::internal::SCCInfo<0> scc_info_Index_milvus_2eproto = - {{ATOMIC_VAR_INIT(::PROTOBUF_NAMESPACE_ID::internal::SCCInfoBase::kUninitialized), 0, InitDefaultsscc_info_Index_milvus_2eproto}, {}}; - static void InitDefaultsscc_info_IndexParam_milvus_2eproto() { GOOGLE_PROTOBUF_VERIFY_VERSION; @@ -228,7 +214,7 @@ static void InitDefaultsscc_info_IndexParam_milvus_2eproto() { ::PROTOBUF_NAMESPACE_ID::internal::SCCInfo<2> scc_info_IndexParam_milvus_2eproto = {{ATOMIC_VAR_INIT(::PROTOBUF_NAMESPACE_ID::internal::SCCInfoBase::kUninitialized), 2, InitDefaultsscc_info_IndexParam_milvus_2eproto}, { &scc_info_Status_status_2eproto.base, - &scc_info_Index_milvus_2eproto.base,}}; + &scc_info_KeyValuePair_milvus_2eproto.base,}}; static void InitDefaultsscc_info_InsertParam_milvus_2eproto() { GOOGLE_PROTOBUF_VERIFY_VERSION; @@ -241,9 +227,24 @@ static void InitDefaultsscc_info_InsertParam_milvus_2eproto() { ::milvus::grpc::InsertParam::InitAsDefaultInstance(); } -::PROTOBUF_NAMESPACE_ID::internal::SCCInfo<1> scc_info_InsertParam_milvus_2eproto = - {{ATOMIC_VAR_INIT(::PROTOBUF_NAMESPACE_ID::internal::SCCInfoBase::kUninitialized), 1, InitDefaultsscc_info_InsertParam_milvus_2eproto}, { - &scc_info_RowRecord_milvus_2eproto.base,}}; +::PROTOBUF_NAMESPACE_ID::internal::SCCInfo<2> scc_info_InsertParam_milvus_2eproto = + {{ATOMIC_VAR_INIT(::PROTOBUF_NAMESPACE_ID::internal::SCCInfoBase::kUninitialized), 2, InitDefaultsscc_info_InsertParam_milvus_2eproto}, { + &scc_info_RowRecord_milvus_2eproto.base, + &scc_info_KeyValuePair_milvus_2eproto.base,}}; + +static void InitDefaultsscc_info_KeyValuePair_milvus_2eproto() { + GOOGLE_PROTOBUF_VERIFY_VERSION; + + { + void* ptr = &::milvus::grpc::_KeyValuePair_default_instance_; + new (ptr) ::milvus::grpc::KeyValuePair(); + ::PROTOBUF_NAMESPACE_ID::internal::OnShutdownDestroyMessage(ptr); + } + ::milvus::grpc::KeyValuePair::InitAsDefaultInstance(); +} + +::PROTOBUF_NAMESPACE_ID::internal::SCCInfo<0> scc_info_KeyValuePair_milvus_2eproto = + {{ATOMIC_VAR_INIT(::PROTOBUF_NAMESPACE_ID::internal::SCCInfoBase::kUninitialized), 0, InitDefaultsscc_info_KeyValuePair_milvus_2eproto}, {}}; static void InitDefaultsscc_info_PartitionList_milvus_2eproto() { GOOGLE_PROTOBUF_VERIFY_VERSION; @@ -314,8 +315,9 @@ static void InitDefaultsscc_info_SearchByIDParam_milvus_2eproto() { ::milvus::grpc::SearchByIDParam::InitAsDefaultInstance(); } -::PROTOBUF_NAMESPACE_ID::internal::SCCInfo<0> scc_info_SearchByIDParam_milvus_2eproto = - {{ATOMIC_VAR_INIT(::PROTOBUF_NAMESPACE_ID::internal::SCCInfoBase::kUninitialized), 0, InitDefaultsscc_info_SearchByIDParam_milvus_2eproto}, {}}; +::PROTOBUF_NAMESPACE_ID::internal::SCCInfo<1> scc_info_SearchByIDParam_milvus_2eproto = + {{ATOMIC_VAR_INIT(::PROTOBUF_NAMESPACE_ID::internal::SCCInfoBase::kUninitialized), 1, InitDefaultsscc_info_SearchByIDParam_milvus_2eproto}, { + &scc_info_KeyValuePair_milvus_2eproto.base,}}; static void InitDefaultsscc_info_SearchInFilesParam_milvus_2eproto() { GOOGLE_PROTOBUF_VERIFY_VERSION; @@ -343,9 +345,10 @@ static void InitDefaultsscc_info_SearchParam_milvus_2eproto() { ::milvus::grpc::SearchParam::InitAsDefaultInstance(); } -::PROTOBUF_NAMESPACE_ID::internal::SCCInfo<1> scc_info_SearchParam_milvus_2eproto = - {{ATOMIC_VAR_INIT(::PROTOBUF_NAMESPACE_ID::internal::SCCInfoBase::kUninitialized), 1, InitDefaultsscc_info_SearchParam_milvus_2eproto}, { - &scc_info_RowRecord_milvus_2eproto.base,}}; +::PROTOBUF_NAMESPACE_ID::internal::SCCInfo<2> scc_info_SearchParam_milvus_2eproto = + {{ATOMIC_VAR_INIT(::PROTOBUF_NAMESPACE_ID::internal::SCCInfoBase::kUninitialized), 2, InitDefaultsscc_info_SearchParam_milvus_2eproto}, { + &scc_info_RowRecord_milvus_2eproto.base, + &scc_info_KeyValuePair_milvus_2eproto.base,}}; static void InitDefaultsscc_info_SegmentStat_milvus_2eproto() { GOOGLE_PROTOBUF_VERIFY_VERSION; @@ -447,9 +450,10 @@ static void InitDefaultsscc_info_TableSchema_milvus_2eproto() { ::milvus::grpc::TableSchema::InitAsDefaultInstance(); } -::PROTOBUF_NAMESPACE_ID::internal::SCCInfo<1> scc_info_TableSchema_milvus_2eproto = - {{ATOMIC_VAR_INIT(::PROTOBUF_NAMESPACE_ID::internal::SCCInfoBase::kUninitialized), 1, InitDefaultsscc_info_TableSchema_milvus_2eproto}, { - &scc_info_Status_status_2eproto.base,}}; +::PROTOBUF_NAMESPACE_ID::internal::SCCInfo<2> scc_info_TableSchema_milvus_2eproto = + {{ATOMIC_VAR_INIT(::PROTOBUF_NAMESPACE_ID::internal::SCCInfoBase::kUninitialized), 2, InitDefaultsscc_info_TableSchema_milvus_2eproto}, { + &scc_info_Status_status_2eproto.base, + &scc_info_KeyValuePair_milvus_2eproto.base,}}; static void InitDefaultsscc_info_TopKQueryResult_milvus_2eproto() { GOOGLE_PROTOBUF_VERIFY_VERSION; @@ -516,6 +520,13 @@ static constexpr ::PROTOBUF_NAMESPACE_ID::EnumDescriptor const** file_level_enum static constexpr ::PROTOBUF_NAMESPACE_ID::ServiceDescriptor const** file_level_service_descriptors_milvus_2eproto = nullptr; const ::PROTOBUF_NAMESPACE_ID::uint32 TableStruct_milvus_2eproto::offsets[] PROTOBUF_SECTION_VARIABLE(protodesc_cold) = { + ~0u, // no _has_bits_ + PROTOBUF_FIELD_OFFSET(::milvus::grpc::KeyValuePair, _internal_metadata_), + ~0u, // no _extensions_ + ~0u, // no _oneof_case_ + ~0u, // no _weak_field_map_ + PROTOBUF_FIELD_OFFSET(::milvus::grpc::KeyValuePair, key_), + PROTOBUF_FIELD_OFFSET(::milvus::grpc::KeyValuePair, value_), ~0u, // no _has_bits_ PROTOBUF_FIELD_OFFSET(::milvus::grpc::TableName, _internal_metadata_), ~0u, // no _extensions_ @@ -539,6 +550,7 @@ const ::PROTOBUF_NAMESPACE_ID::uint32 TableStruct_milvus_2eproto::offsets[] PROT PROTOBUF_FIELD_OFFSET(::milvus::grpc::TableSchema, dimension_), PROTOBUF_FIELD_OFFSET(::milvus::grpc::TableSchema, index_file_size_), PROTOBUF_FIELD_OFFSET(::milvus::grpc::TableSchema, metric_type_), + PROTOBUF_FIELD_OFFSET(::milvus::grpc::TableSchema, extra_params_), ~0u, // no _has_bits_ PROTOBUF_FIELD_OFFSET(::milvus::grpc::PartitionParam, _internal_metadata_), ~0u, // no _extensions_ @@ -569,6 +581,7 @@ const ::PROTOBUF_NAMESPACE_ID::uint32 TableStruct_milvus_2eproto::offsets[] PROT PROTOBUF_FIELD_OFFSET(::milvus::grpc::InsertParam, row_record_array_), PROTOBUF_FIELD_OFFSET(::milvus::grpc::InsertParam, row_id_array_), PROTOBUF_FIELD_OFFSET(::milvus::grpc::InsertParam, partition_tag_), + PROTOBUF_FIELD_OFFSET(::milvus::grpc::InsertParam, extra_params_), ~0u, // no _has_bits_ PROTOBUF_FIELD_OFFSET(::milvus::grpc::VectorIds, _internal_metadata_), ~0u, // no _extensions_ @@ -582,10 +595,10 @@ const ::PROTOBUF_NAMESPACE_ID::uint32 TableStruct_milvus_2eproto::offsets[] PROT ~0u, // no _oneof_case_ ~0u, // no _weak_field_map_ PROTOBUF_FIELD_OFFSET(::milvus::grpc::SearchParam, table_name_), + PROTOBUF_FIELD_OFFSET(::milvus::grpc::SearchParam, partition_tag_array_), PROTOBUF_FIELD_OFFSET(::milvus::grpc::SearchParam, query_record_array_), PROTOBUF_FIELD_OFFSET(::milvus::grpc::SearchParam, topk_), - PROTOBUF_FIELD_OFFSET(::milvus::grpc::SearchParam, nprobe_), - PROTOBUF_FIELD_OFFSET(::milvus::grpc::SearchParam, partition_tag_array_), + PROTOBUF_FIELD_OFFSET(::milvus::grpc::SearchParam, extra_params_), ~0u, // no _has_bits_ PROTOBUF_FIELD_OFFSET(::milvus::grpc::SearchInFilesParam, _internal_metadata_), ~0u, // no _extensions_ @@ -599,10 +612,10 @@ const ::PROTOBUF_NAMESPACE_ID::uint32 TableStruct_milvus_2eproto::offsets[] PROT ~0u, // no _oneof_case_ ~0u, // no _weak_field_map_ PROTOBUF_FIELD_OFFSET(::milvus::grpc::SearchByIDParam, table_name_), + PROTOBUF_FIELD_OFFSET(::milvus::grpc::SearchByIDParam, partition_tag_array_), PROTOBUF_FIELD_OFFSET(::milvus::grpc::SearchByIDParam, id_), PROTOBUF_FIELD_OFFSET(::milvus::grpc::SearchByIDParam, topk_), - PROTOBUF_FIELD_OFFSET(::milvus::grpc::SearchByIDParam, nprobe_), - PROTOBUF_FIELD_OFFSET(::milvus::grpc::SearchByIDParam, partition_tag_array_), + PROTOBUF_FIELD_OFFSET(::milvus::grpc::SearchByIDParam, extra_params_), ~0u, // no _has_bits_ PROTOBUF_FIELD_OFFSET(::milvus::grpc::TopKQueryResult, _internal_metadata_), ~0u, // no _extensions_ @@ -640,20 +653,14 @@ const ::PROTOBUF_NAMESPACE_ID::uint32 TableStruct_milvus_2eproto::offsets[] PROT ~0u, // no _weak_field_map_ PROTOBUF_FIELD_OFFSET(::milvus::grpc::Command, cmd_), ~0u, // no _has_bits_ - PROTOBUF_FIELD_OFFSET(::milvus::grpc::Index, _internal_metadata_), - ~0u, // no _extensions_ - ~0u, // no _oneof_case_ - ~0u, // no _weak_field_map_ - PROTOBUF_FIELD_OFFSET(::milvus::grpc::Index, index_type_), - PROTOBUF_FIELD_OFFSET(::milvus::grpc::Index, nlist_), - ~0u, // no _has_bits_ PROTOBUF_FIELD_OFFSET(::milvus::grpc::IndexParam, _internal_metadata_), ~0u, // no _extensions_ ~0u, // no _oneof_case_ ~0u, // no _weak_field_map_ PROTOBUF_FIELD_OFFSET(::milvus::grpc::IndexParam, status_), PROTOBUF_FIELD_OFFSET(::milvus::grpc::IndexParam, table_name_), - PROTOBUF_FIELD_OFFSET(::milvus::grpc::IndexParam, index_), + PROTOBUF_FIELD_OFFSET(::milvus::grpc::IndexParam, index_type_), + PROTOBUF_FIELD_OFFSET(::milvus::grpc::IndexParam, extra_params_), ~0u, // no _has_bits_ PROTOBUF_FIELD_OFFSET(::milvus::grpc::FlushParam, _internal_metadata_), ~0u, // no _extensions_ @@ -715,35 +722,36 @@ const ::PROTOBUF_NAMESPACE_ID::uint32 TableStruct_milvus_2eproto::offsets[] PROT PROTOBUF_FIELD_OFFSET(::milvus::grpc::GetVectorIDsParam, segment_name_), }; static const ::PROTOBUF_NAMESPACE_ID::internal::MigrationSchema schemas[] PROTOBUF_SECTION_VARIABLE(protodesc_cold) = { - { 0, -1, sizeof(::milvus::grpc::TableName)}, - { 6, -1, sizeof(::milvus::grpc::TableNameList)}, - { 13, -1, sizeof(::milvus::grpc::TableSchema)}, - { 23, -1, sizeof(::milvus::grpc::PartitionParam)}, - { 30, -1, sizeof(::milvus::grpc::PartitionList)}, - { 37, -1, sizeof(::milvus::grpc::RowRecord)}, - { 44, -1, sizeof(::milvus::grpc::InsertParam)}, - { 53, -1, sizeof(::milvus::grpc::VectorIds)}, - { 60, -1, sizeof(::milvus::grpc::SearchParam)}, - { 70, -1, sizeof(::milvus::grpc::SearchInFilesParam)}, - { 77, -1, sizeof(::milvus::grpc::SearchByIDParam)}, - { 87, -1, sizeof(::milvus::grpc::TopKQueryResult)}, - { 96, -1, sizeof(::milvus::grpc::StringReply)}, - { 103, -1, sizeof(::milvus::grpc::BoolReply)}, - { 110, -1, sizeof(::milvus::grpc::TableRowCount)}, - { 117, -1, sizeof(::milvus::grpc::Command)}, - { 123, -1, sizeof(::milvus::grpc::Index)}, - { 130, -1, sizeof(::milvus::grpc::IndexParam)}, - { 138, -1, sizeof(::milvus::grpc::FlushParam)}, - { 144, -1, sizeof(::milvus::grpc::DeleteByIDParam)}, - { 151, -1, sizeof(::milvus::grpc::SegmentStat)}, - { 160, -1, sizeof(::milvus::grpc::PartitionStat)}, - { 168, -1, sizeof(::milvus::grpc::TableInfo)}, - { 176, -1, sizeof(::milvus::grpc::VectorIdentity)}, - { 183, -1, sizeof(::milvus::grpc::VectorData)}, - { 190, -1, sizeof(::milvus::grpc::GetVectorIDsParam)}, + { 0, -1, sizeof(::milvus::grpc::KeyValuePair)}, + { 7, -1, sizeof(::milvus::grpc::TableName)}, + { 13, -1, sizeof(::milvus::grpc::TableNameList)}, + { 20, -1, sizeof(::milvus::grpc::TableSchema)}, + { 31, -1, sizeof(::milvus::grpc::PartitionParam)}, + { 38, -1, sizeof(::milvus::grpc::PartitionList)}, + { 45, -1, sizeof(::milvus::grpc::RowRecord)}, + { 52, -1, sizeof(::milvus::grpc::InsertParam)}, + { 62, -1, sizeof(::milvus::grpc::VectorIds)}, + { 69, -1, sizeof(::milvus::grpc::SearchParam)}, + { 79, -1, sizeof(::milvus::grpc::SearchInFilesParam)}, + { 86, -1, sizeof(::milvus::grpc::SearchByIDParam)}, + { 96, -1, sizeof(::milvus::grpc::TopKQueryResult)}, + { 105, -1, sizeof(::milvus::grpc::StringReply)}, + { 112, -1, sizeof(::milvus::grpc::BoolReply)}, + { 119, -1, sizeof(::milvus::grpc::TableRowCount)}, + { 126, -1, sizeof(::milvus::grpc::Command)}, + { 132, -1, sizeof(::milvus::grpc::IndexParam)}, + { 141, -1, sizeof(::milvus::grpc::FlushParam)}, + { 147, -1, sizeof(::milvus::grpc::DeleteByIDParam)}, + { 154, -1, sizeof(::milvus::grpc::SegmentStat)}, + { 163, -1, sizeof(::milvus::grpc::PartitionStat)}, + { 171, -1, sizeof(::milvus::grpc::TableInfo)}, + { 179, -1, sizeof(::milvus::grpc::VectorIdentity)}, + { 186, -1, sizeof(::milvus::grpc::VectorData)}, + { 193, -1, sizeof(::milvus::grpc::GetVectorIDsParam)}, }; static ::PROTOBUF_NAMESPACE_ID::Message const * const file_default_instances[] = { + reinterpret_cast(&::milvus::grpc::_KeyValuePair_default_instance_), reinterpret_cast(&::milvus::grpc::_TableName_default_instance_), reinterpret_cast(&::milvus::grpc::_TableNameList_default_instance_), reinterpret_cast(&::milvus::grpc::_TableSchema_default_instance_), @@ -760,7 +768,6 @@ static ::PROTOBUF_NAMESPACE_ID::Message const * const file_default_instances[] = reinterpret_cast(&::milvus::grpc::_BoolReply_default_instance_), reinterpret_cast(&::milvus::grpc::_TableRowCount_default_instance_), reinterpret_cast(&::milvus::grpc::_Command_default_instance_), - reinterpret_cast(&::milvus::grpc::_Index_default_instance_), reinterpret_cast(&::milvus::grpc::_IndexParam_default_instance_), reinterpret_cast(&::milvus::grpc::_FlushParam_default_instance_), reinterpret_cast(&::milvus::grpc::_DeleteByIDParam_default_instance_), @@ -774,44 +781,49 @@ static ::PROTOBUF_NAMESPACE_ID::Message const * const file_default_instances[] = const char descriptor_table_protodef_milvus_2eproto[] PROTOBUF_SECTION_VARIABLE(protodesc_cold) = "\n\014milvus.proto\022\013milvus.grpc\032\014status.prot" - "o\"\037\n\tTableName\022\022\n\ntable_name\030\001 \001(\t\"I\n\rTa" - "bleNameList\022#\n\006status\030\001 \001(\0132\023.milvus.grp" - "c.Status\022\023\n\013table_names\030\002 \003(\t\"\207\001\n\013TableS" - "chema\022#\n\006status\030\001 \001(\0132\023.milvus.grpc.Stat" - "us\022\022\n\ntable_name\030\002 \001(\t\022\021\n\tdimension\030\003 \001(" - "\003\022\027\n\017index_file_size\030\004 \001(\003\022\023\n\013metric_typ" - "e\030\005 \001(\005\"1\n\016PartitionParam\022\022\n\ntable_name\030" - "\001 \001(\t\022\013\n\003tag\030\002 \001(\t\"Q\n\rPartitionList\022#\n\006s" - "tatus\030\001 \001(\0132\023.milvus.grpc.Status\022\033\n\023part" - "ition_tag_array\030\002 \003(\t\"4\n\tRowRecord\022\022\n\nfl" - "oat_data\030\001 \003(\002\022\023\n\013binary_data\030\002 \001(\014\"\200\001\n\013" - "InsertParam\022\022\n\ntable_name\030\001 \001(\t\0220\n\020row_r" - "ecord_array\030\002 \003(\0132\026.milvus.grpc.RowRecor" - "d\022\024\n\014row_id_array\030\003 \003(\003\022\025\n\rpartition_tag" - "\030\004 \001(\t\"I\n\tVectorIds\022#\n\006status\030\001 \001(\0132\023.mi" - "lvus.grpc.Status\022\027\n\017vector_id_array\030\002 \003(" - "\003\"\220\001\n\013SearchParam\022\022\n\ntable_name\030\001 \001(\t\0222\n" - "\022query_record_array\030\002 \003(\0132\026.milvus.grpc." - "RowRecord\022\014\n\004topk\030\003 \001(\003\022\016\n\006nprobe\030\004 \001(\003\022" - "\033\n\023partition_tag_array\030\005 \003(\t\"[\n\022SearchIn" - "FilesParam\022\025\n\rfile_id_array\030\001 \003(\t\022.\n\014sea" - "rch_param\030\002 \001(\0132\030.milvus.grpc.SearchPara" - "m\"l\n\017SearchByIDParam\022\022\n\ntable_name\030\001 \001(\t" - "\022\n\n\002id\030\002 \001(\003\022\014\n\004topk\030\003 \001(\003\022\016\n\006nprobe\030\004 \001" - "(\003\022\033\n\023partition_tag_array\030\005 \003(\t\"g\n\017TopKQ" - "ueryResult\022#\n\006status\030\001 \001(\0132\023.milvus.grpc" - ".Status\022\017\n\007row_num\030\002 \001(\003\022\013\n\003ids\030\003 \003(\003\022\021\n" - "\tdistances\030\004 \003(\002\"H\n\013StringReply\022#\n\006statu" - "s\030\001 \001(\0132\023.milvus.grpc.Status\022\024\n\014string_r" - "eply\030\002 \001(\t\"D\n\tBoolReply\022#\n\006status\030\001 \001(\0132" - "\023.milvus.grpc.Status\022\022\n\nbool_reply\030\002 \001(\010" - "\"M\n\rTableRowCount\022#\n\006status\030\001 \001(\0132\023.milv" - "us.grpc.Status\022\027\n\017table_row_count\030\002 \001(\003\"" - "\026\n\007Command\022\013\n\003cmd\030\001 \001(\t\"*\n\005Index\022\022\n\ninde" - "x_type\030\001 \001(\005\022\r\n\005nlist\030\002 \001(\005\"h\n\nIndexPara" - "m\022#\n\006status\030\001 \001(\0132\023.milvus.grpc.Status\022\022" - "\n\ntable_name\030\002 \001(\t\022!\n\005index\030\003 \001(\0132\022.milv" - "us.grpc.Index\"&\n\nFlushParam\022\030\n\020table_nam" + "o\"*\n\014KeyValuePair\022\013\n\003key\030\001 \001(\t\022\r\n\005value\030" + "\002 \001(\t\"\037\n\tTableName\022\022\n\ntable_name\030\001 \001(\t\"I" + "\n\rTableNameList\022#\n\006status\030\001 \001(\0132\023.milvus" + ".grpc.Status\022\023\n\013table_names\030\002 \003(\t\"\270\001\n\013Ta" + "bleSchema\022#\n\006status\030\001 \001(\0132\023.milvus.grpc." + "Status\022\022\n\ntable_name\030\002 \001(\t\022\021\n\tdimension\030" + "\003 \001(\003\022\027\n\017index_file_size\030\004 \001(\003\022\023\n\013metric" + "_type\030\005 \001(\005\022/\n\014extra_params\030\006 \003(\0132\031.milv" + "us.grpc.KeyValuePair\"1\n\016PartitionParam\022\022" + "\n\ntable_name\030\001 \001(\t\022\013\n\003tag\030\002 \001(\t\"Q\n\rParti" + "tionList\022#\n\006status\030\001 \001(\0132\023.milvus.grpc.S" + "tatus\022\033\n\023partition_tag_array\030\002 \003(\t\"4\n\tRo" + "wRecord\022\022\n\nfloat_data\030\001 \003(\002\022\023\n\013binary_da" + "ta\030\002 \001(\014\"\261\001\n\013InsertParam\022\022\n\ntable_name\030\001" + " \001(\t\0220\n\020row_record_array\030\002 \003(\0132\026.milvus." + "grpc.RowRecord\022\024\n\014row_id_array\030\003 \003(\003\022\025\n\r" + "partition_tag\030\004 \001(\t\022/\n\014extra_params\030\005 \003(" + "\0132\031.milvus.grpc.KeyValuePair\"I\n\tVectorId" + "s\022#\n\006status\030\001 \001(\0132\023.milvus.grpc.Status\022\027" + "\n\017vector_id_array\030\002 \003(\003\"\261\001\n\013SearchParam\022" + "\022\n\ntable_name\030\001 \001(\t\022\033\n\023partition_tag_arr" + "ay\030\002 \003(\t\0222\n\022query_record_array\030\003 \003(\0132\026.m" + "ilvus.grpc.RowRecord\022\014\n\004topk\030\004 \001(\003\022/\n\014ex" + "tra_params\030\005 \003(\0132\031.milvus.grpc.KeyValueP" + "air\"[\n\022SearchInFilesParam\022\025\n\rfile_id_arr" + "ay\030\001 \003(\t\022.\n\014search_param\030\002 \001(\0132\030.milvus." + "grpc.SearchParam\"\215\001\n\017SearchByIDParam\022\022\n\n" + "table_name\030\001 \001(\t\022\033\n\023partition_tag_array\030" + "\002 \003(\t\022\n\n\002id\030\003 \001(\003\022\014\n\004topk\030\004 \001(\003\022/\n\014extra" + "_params\030\005 \003(\0132\031.milvus.grpc.KeyValuePair" + "\"g\n\017TopKQueryResult\022#\n\006status\030\001 \001(\0132\023.mi" + "lvus.grpc.Status\022\017\n\007row_num\030\002 \001(\003\022\013\n\003ids" + "\030\003 \003(\003\022\021\n\tdistances\030\004 \003(\002\"H\n\013StringReply" + "\022#\n\006status\030\001 \001(\0132\023.milvus.grpc.Status\022\024\n" + "\014string_reply\030\002 \001(\t\"D\n\tBoolReply\022#\n\006stat" + "us\030\001 \001(\0132\023.milvus.grpc.Status\022\022\n\nbool_re" + "ply\030\002 \001(\010\"M\n\rTableRowCount\022#\n\006status\030\001 \001" + "(\0132\023.milvus.grpc.Status\022\027\n\017table_row_cou" + "nt\030\002 \001(\003\"\026\n\007Command\022\013\n\003cmd\030\001 \001(\t\"\212\001\n\nInd" + "exParam\022#\n\006status\030\001 \001(\0132\023.milvus.grpc.St" + "atus\022\022\n\ntable_name\030\002 \001(\t\022\022\n\nindex_type\030\003" + " \001(\005\022/\n\014extra_params\030\004 \003(\0132\031.milvus.grpc" + ".KeyValuePair\"&\n\nFlushParam\022\030\n\020table_nam" "e_array\030\001 \003(\t\"7\n\017DeleteByIDParam\022\022\n\ntabl" "e_name\030\001 \001(\t\022\020\n\010id_array\030\002 \003(\003\"]\n\013Segmen" "tStat\022\024\n\014segment_name\030\001 \001(\t\022\021\n\trow_count" @@ -878,9 +890,9 @@ static ::PROTOBUF_NAMESPACE_ID::internal::SCCInfoBase*const descriptor_table_mil &scc_info_DeleteByIDParam_milvus_2eproto.base, &scc_info_FlushParam_milvus_2eproto.base, &scc_info_GetVectorIDsParam_milvus_2eproto.base, - &scc_info_Index_milvus_2eproto.base, &scc_info_IndexParam_milvus_2eproto.base, &scc_info_InsertParam_milvus_2eproto.base, + &scc_info_KeyValuePair_milvus_2eproto.base, &scc_info_PartitionList_milvus_2eproto.base, &scc_info_PartitionParam_milvus_2eproto.base, &scc_info_PartitionStat_milvus_2eproto.base, @@ -903,7 +915,7 @@ static ::PROTOBUF_NAMESPACE_ID::internal::SCCInfoBase*const descriptor_table_mil static ::PROTOBUF_NAMESPACE_ID::internal::once_flag descriptor_table_milvus_2eproto_once; static bool descriptor_table_milvus_2eproto_initialized = false; const ::PROTOBUF_NAMESPACE_ID::internal::DescriptorTable descriptor_table_milvus_2eproto = { - &descriptor_table_milvus_2eproto_initialized, descriptor_table_protodef_milvus_2eproto, "milvus.proto", 3788, + &descriptor_table_milvus_2eproto_initialized, descriptor_table_protodef_milvus_2eproto, "milvus.proto", 3988, &descriptor_table_milvus_2eproto_once, descriptor_table_milvus_2eproto_sccs, descriptor_table_milvus_2eproto_deps, 26, 1, schemas, file_default_instances, TableStruct_milvus_2eproto::offsets, file_level_metadata_milvus_2eproto, 26, file_level_enum_descriptors_milvus_2eproto, file_level_service_descriptors_milvus_2eproto, @@ -914,6 +926,338 @@ static bool dynamic_init_dummy_milvus_2eproto = ( ::PROTOBUF_NAMESPACE_ID::inte namespace milvus { namespace grpc { +// =================================================================== + +void KeyValuePair::InitAsDefaultInstance() { +} +class KeyValuePair::_Internal { + public: +}; + +KeyValuePair::KeyValuePair() + : ::PROTOBUF_NAMESPACE_ID::Message(), _internal_metadata_(nullptr) { + SharedCtor(); + // @@protoc_insertion_point(constructor:milvus.grpc.KeyValuePair) +} +KeyValuePair::KeyValuePair(const KeyValuePair& from) + : ::PROTOBUF_NAMESPACE_ID::Message(), + _internal_metadata_(nullptr) { + _internal_metadata_.MergeFrom(from._internal_metadata_); + key_.UnsafeSetDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); + if (!from.key().empty()) { + key_.AssignWithDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), from.key_); + } + value_.UnsafeSetDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); + if (!from.value().empty()) { + value_.AssignWithDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), from.value_); + } + // @@protoc_insertion_point(copy_constructor:milvus.grpc.KeyValuePair) +} + +void KeyValuePair::SharedCtor() { + ::PROTOBUF_NAMESPACE_ID::internal::InitSCC(&scc_info_KeyValuePair_milvus_2eproto.base); + key_.UnsafeSetDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); + value_.UnsafeSetDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); +} + +KeyValuePair::~KeyValuePair() { + // @@protoc_insertion_point(destructor:milvus.grpc.KeyValuePair) + SharedDtor(); +} + +void KeyValuePair::SharedDtor() { + key_.DestroyNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); + value_.DestroyNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); +} + +void KeyValuePair::SetCachedSize(int size) const { + _cached_size_.Set(size); +} +const KeyValuePair& KeyValuePair::default_instance() { + ::PROTOBUF_NAMESPACE_ID::internal::InitSCC(&::scc_info_KeyValuePair_milvus_2eproto.base); + return *internal_default_instance(); +} + + +void KeyValuePair::Clear() { +// @@protoc_insertion_point(message_clear_start:milvus.grpc.KeyValuePair) + ::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0; + // Prevent compiler warnings about cached_has_bits being unused + (void) cached_has_bits; + + key_.ClearToEmptyNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); + value_.ClearToEmptyNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); + _internal_metadata_.Clear(); +} + +#if GOOGLE_PROTOBUF_ENABLE_EXPERIMENTAL_PARSER +const char* KeyValuePair::_InternalParse(const char* ptr, ::PROTOBUF_NAMESPACE_ID::internal::ParseContext* ctx) { +#define CHK_(x) if (PROTOBUF_PREDICT_FALSE(!(x))) goto failure + while (!ctx->Done(&ptr)) { + ::PROTOBUF_NAMESPACE_ID::uint32 tag; + ptr = ::PROTOBUF_NAMESPACE_ID::internal::ReadTag(ptr, &tag); + CHK_(ptr); + switch (tag >> 3) { + // string key = 1; + case 1: + if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 10)) { + ptr = ::PROTOBUF_NAMESPACE_ID::internal::InlineGreedyStringParserUTF8(mutable_key(), ptr, ctx, "milvus.grpc.KeyValuePair.key"); + CHK_(ptr); + } else goto handle_unusual; + continue; + // string value = 2; + case 2: + if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 18)) { + ptr = ::PROTOBUF_NAMESPACE_ID::internal::InlineGreedyStringParserUTF8(mutable_value(), ptr, ctx, "milvus.grpc.KeyValuePair.value"); + CHK_(ptr); + } else goto handle_unusual; + continue; + default: { + handle_unusual: + if ((tag & 7) == 4 || tag == 0) { + ctx->SetLastTag(tag); + goto success; + } + ptr = UnknownFieldParse(tag, &_internal_metadata_, ptr, ctx); + CHK_(ptr != nullptr); + continue; + } + } // switch + } // while +success: + return ptr; +failure: + ptr = nullptr; + goto success; +#undef CHK_ +} +#else // GOOGLE_PROTOBUF_ENABLE_EXPERIMENTAL_PARSER +bool KeyValuePair::MergePartialFromCodedStream( + ::PROTOBUF_NAMESPACE_ID::io::CodedInputStream* input) { +#define DO_(EXPRESSION) if (!PROTOBUF_PREDICT_TRUE(EXPRESSION)) goto failure + ::PROTOBUF_NAMESPACE_ID::uint32 tag; + // @@protoc_insertion_point(parse_start:milvus.grpc.KeyValuePair) + for (;;) { + ::std::pair<::PROTOBUF_NAMESPACE_ID::uint32, bool> p = input->ReadTagWithCutoffNoLastTag(127u); + tag = p.first; + if (!p.second) goto handle_unusual; + switch (::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::GetTagFieldNumber(tag)) { + // string key = 1; + case 1: { + if (static_cast< ::PROTOBUF_NAMESPACE_ID::uint8>(tag) == (10 & 0xFF)) { + DO_(::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::ReadString( + input, this->mutable_key())); + DO_(::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::VerifyUtf8String( + this->key().data(), static_cast(this->key().length()), + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::PARSE, + "milvus.grpc.KeyValuePair.key")); + } else { + goto handle_unusual; + } + break; + } + + // string value = 2; + case 2: { + if (static_cast< ::PROTOBUF_NAMESPACE_ID::uint8>(tag) == (18 & 0xFF)) { + DO_(::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::ReadString( + input, this->mutable_value())); + DO_(::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::VerifyUtf8String( + this->value().data(), static_cast(this->value().length()), + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::PARSE, + "milvus.grpc.KeyValuePair.value")); + } else { + goto handle_unusual; + } + break; + } + + default: { + handle_unusual: + if (tag == 0) { + goto success; + } + DO_(::PROTOBUF_NAMESPACE_ID::internal::WireFormat::SkipField( + input, tag, _internal_metadata_.mutable_unknown_fields())); + break; + } + } + } +success: + // @@protoc_insertion_point(parse_success:milvus.grpc.KeyValuePair) + return true; +failure: + // @@protoc_insertion_point(parse_failure:milvus.grpc.KeyValuePair) + return false; +#undef DO_ +} +#endif // GOOGLE_PROTOBUF_ENABLE_EXPERIMENTAL_PARSER + +void KeyValuePair::SerializeWithCachedSizes( + ::PROTOBUF_NAMESPACE_ID::io::CodedOutputStream* output) const { + // @@protoc_insertion_point(serialize_start:milvus.grpc.KeyValuePair) + ::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0; + (void) cached_has_bits; + + // string key = 1; + if (this->key().size() > 0) { + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::VerifyUtf8String( + this->key().data(), static_cast(this->key().length()), + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::SERIALIZE, + "milvus.grpc.KeyValuePair.key"); + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteStringMaybeAliased( + 1, this->key(), output); + } + + // string value = 2; + if (this->value().size() > 0) { + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::VerifyUtf8String( + this->value().data(), static_cast(this->value().length()), + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::SERIALIZE, + "milvus.grpc.KeyValuePair.value"); + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteStringMaybeAliased( + 2, this->value(), output); + } + + if (_internal_metadata_.have_unknown_fields()) { + ::PROTOBUF_NAMESPACE_ID::internal::WireFormat::SerializeUnknownFields( + _internal_metadata_.unknown_fields(), output); + } + // @@protoc_insertion_point(serialize_end:milvus.grpc.KeyValuePair) +} + +::PROTOBUF_NAMESPACE_ID::uint8* KeyValuePair::InternalSerializeWithCachedSizesToArray( + ::PROTOBUF_NAMESPACE_ID::uint8* target) const { + // @@protoc_insertion_point(serialize_to_array_start:milvus.grpc.KeyValuePair) + ::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0; + (void) cached_has_bits; + + // string key = 1; + if (this->key().size() > 0) { + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::VerifyUtf8String( + this->key().data(), static_cast(this->key().length()), + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::SERIALIZE, + "milvus.grpc.KeyValuePair.key"); + target = + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteStringToArray( + 1, this->key(), target); + } + + // string value = 2; + if (this->value().size() > 0) { + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::VerifyUtf8String( + this->value().data(), static_cast(this->value().length()), + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::SERIALIZE, + "milvus.grpc.KeyValuePair.value"); + target = + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteStringToArray( + 2, this->value(), target); + } + + if (_internal_metadata_.have_unknown_fields()) { + target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormat::SerializeUnknownFieldsToArray( + _internal_metadata_.unknown_fields(), target); + } + // @@protoc_insertion_point(serialize_to_array_end:milvus.grpc.KeyValuePair) + return target; +} + +size_t KeyValuePair::ByteSizeLong() const { +// @@protoc_insertion_point(message_byte_size_start:milvus.grpc.KeyValuePair) + size_t total_size = 0; + + if (_internal_metadata_.have_unknown_fields()) { + total_size += + ::PROTOBUF_NAMESPACE_ID::internal::WireFormat::ComputeUnknownFieldsSize( + _internal_metadata_.unknown_fields()); + } + ::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0; + // Prevent compiler warnings about cached_has_bits being unused + (void) cached_has_bits; + + // string key = 1; + if (this->key().size() > 0) { + total_size += 1 + + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::StringSize( + this->key()); + } + + // string value = 2; + if (this->value().size() > 0) { + total_size += 1 + + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::StringSize( + this->value()); + } + + int cached_size = ::PROTOBUF_NAMESPACE_ID::internal::ToCachedSize(total_size); + SetCachedSize(cached_size); + return total_size; +} + +void KeyValuePair::MergeFrom(const ::PROTOBUF_NAMESPACE_ID::Message& from) { +// @@protoc_insertion_point(generalized_merge_from_start:milvus.grpc.KeyValuePair) + GOOGLE_DCHECK_NE(&from, this); + const KeyValuePair* source = + ::PROTOBUF_NAMESPACE_ID::DynamicCastToGenerated( + &from); + if (source == nullptr) { + // @@protoc_insertion_point(generalized_merge_from_cast_fail:milvus.grpc.KeyValuePair) + ::PROTOBUF_NAMESPACE_ID::internal::ReflectionOps::Merge(from, this); + } else { + // @@protoc_insertion_point(generalized_merge_from_cast_success:milvus.grpc.KeyValuePair) + MergeFrom(*source); + } +} + +void KeyValuePair::MergeFrom(const KeyValuePair& from) { +// @@protoc_insertion_point(class_specific_merge_from_start:milvus.grpc.KeyValuePair) + GOOGLE_DCHECK_NE(&from, this); + _internal_metadata_.MergeFrom(from._internal_metadata_); + ::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0; + (void) cached_has_bits; + + if (from.key().size() > 0) { + + key_.AssignWithDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), from.key_); + } + if (from.value().size() > 0) { + + value_.AssignWithDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), from.value_); + } +} + +void KeyValuePair::CopyFrom(const ::PROTOBUF_NAMESPACE_ID::Message& from) { +// @@protoc_insertion_point(generalized_copy_from_start:milvus.grpc.KeyValuePair) + if (&from == this) return; + Clear(); + MergeFrom(from); +} + +void KeyValuePair::CopyFrom(const KeyValuePair& from) { +// @@protoc_insertion_point(class_specific_copy_from_start:milvus.grpc.KeyValuePair) + if (&from == this) return; + Clear(); + MergeFrom(from); +} + +bool KeyValuePair::IsInitialized() const { + return true; +} + +void KeyValuePair::InternalSwap(KeyValuePair* other) { + using std::swap; + _internal_metadata_.Swap(&other->_internal_metadata_); + key_.Swap(&other->key_, &::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), + GetArenaNoVirtual()); + value_.Swap(&other->value_, &::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), + GetArenaNoVirtual()); +} + +::PROTOBUF_NAMESPACE_ID::Metadata KeyValuePair::GetMetadata() const { + return GetMetadataStatic(); +} + + // =================================================================== void TableName::InitAsDefaultInstance() { @@ -1543,7 +1887,8 @@ TableSchema::TableSchema() } TableSchema::TableSchema(const TableSchema& from) : ::PROTOBUF_NAMESPACE_ID::Message(), - _internal_metadata_(nullptr) { + _internal_metadata_(nullptr), + extra_params_(from.extra_params_) { _internal_metadata_.MergeFrom(from._internal_metadata_); table_name_.UnsafeSetDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); if (!from.table_name().empty()) { @@ -1593,6 +1938,7 @@ void TableSchema::Clear() { // Prevent compiler warnings about cached_has_bits being unused (void) cached_has_bits; + extra_params_.Clear(); table_name_.ClearToEmptyNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); if (GetArenaNoVirtual() == nullptr && status_ != nullptr) { delete status_; @@ -1647,6 +1993,18 @@ const char* TableSchema::_InternalParse(const char* ptr, ::PROTOBUF_NAMESPACE_ID CHK_(ptr); } else goto handle_unusual; continue; + // repeated .milvus.grpc.KeyValuePair extra_params = 6; + case 6: + if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 50)) { + ptr -= 1; + do { + ptr += 1; + ptr = ctx->ParseMessage(add_extra_params(), ptr); + CHK_(ptr); + if (!ctx->DataAvailable(ptr)) break; + } while (::PROTOBUF_NAMESPACE_ID::internal::UnalignedLoad<::PROTOBUF_NAMESPACE_ID::uint8>(ptr) == 50); + } else goto handle_unusual; + continue; default: { handle_unusual: if ((tag & 7) == 4 || tag == 0) { @@ -1742,6 +2100,17 @@ bool TableSchema::MergePartialFromCodedStream( break; } + // repeated .milvus.grpc.KeyValuePair extra_params = 6; + case 6: { + if (static_cast< ::PROTOBUF_NAMESPACE_ID::uint8>(tag) == (50 & 0xFF)) { + DO_(::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::ReadMessage( + input, add_extra_params())); + } else { + goto handle_unusual; + } + break; + } + default: { handle_unusual: if (tag == 0) { @@ -1800,6 +2169,15 @@ void TableSchema::SerializeWithCachedSizes( ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt32(5, this->metric_type(), output); } + // repeated .milvus.grpc.KeyValuePair extra_params = 6; + for (unsigned int i = 0, + n = static_cast(this->extra_params_size()); i < n; i++) { + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteMessageMaybeToArray( + 6, + this->extra_params(static_cast(i)), + output); + } + if (_internal_metadata_.have_unknown_fields()) { ::PROTOBUF_NAMESPACE_ID::internal::WireFormat::SerializeUnknownFields( _internal_metadata_.unknown_fields(), output); @@ -1846,6 +2224,14 @@ void TableSchema::SerializeWithCachedSizes( target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt32ToArray(5, this->metric_type(), target); } + // repeated .milvus.grpc.KeyValuePair extra_params = 6; + for (unsigned int i = 0, + n = static_cast(this->extra_params_size()); i < n; i++) { + target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite:: + InternalWriteMessageToArray( + 6, this->extra_params(static_cast(i)), target); + } + if (_internal_metadata_.have_unknown_fields()) { target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormat::SerializeUnknownFieldsToArray( _internal_metadata_.unknown_fields(), target); @@ -1867,6 +2253,17 @@ size_t TableSchema::ByteSizeLong() const { // Prevent compiler warnings about cached_has_bits being unused (void) cached_has_bits; + // repeated .milvus.grpc.KeyValuePair extra_params = 6; + { + unsigned int count = static_cast(this->extra_params_size()); + total_size += 1UL * count; + for (unsigned int i = 0; i < count; i++) { + total_size += + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::MessageSize( + this->extra_params(static_cast(i))); + } + } + // string table_name = 2; if (this->table_name().size() > 0) { total_size += 1 + @@ -1929,6 +2326,7 @@ void TableSchema::MergeFrom(const TableSchema& from) { ::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0; (void) cached_has_bits; + extra_params_.MergeFrom(from.extra_params_); if (from.table_name().size() > 0) { table_name_.AssignWithDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), from.table_name_); @@ -1968,6 +2366,7 @@ bool TableSchema::IsInitialized() const { void TableSchema::InternalSwap(TableSchema* other) { using std::swap; _internal_metadata_.Swap(&other->_internal_metadata_); + CastToBase(&extra_params_)->InternalSwap(CastToBase(&other->extra_params_)); table_name_.Swap(&other->table_name_, &::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), GetArenaNoVirtual()); swap(status_, other->status_); @@ -2986,7 +3385,8 @@ InsertParam::InsertParam(const InsertParam& from) : ::PROTOBUF_NAMESPACE_ID::Message(), _internal_metadata_(nullptr), row_record_array_(from.row_record_array_), - row_id_array_(from.row_id_array_) { + row_id_array_(from.row_id_array_), + extra_params_(from.extra_params_) { _internal_metadata_.MergeFrom(from._internal_metadata_); table_name_.UnsafeSetDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); if (!from.table_name().empty()) { @@ -3032,6 +3432,7 @@ void InsertParam::Clear() { row_record_array_.Clear(); row_id_array_.Clear(); + extra_params_.Clear(); table_name_.ClearToEmptyNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); partition_tag_.ClearToEmptyNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); _internal_metadata_.Clear(); @@ -3081,6 +3482,18 @@ const char* InsertParam::_InternalParse(const char* ptr, ::PROTOBUF_NAMESPACE_ID CHK_(ptr); } else goto handle_unusual; continue; + // repeated .milvus.grpc.KeyValuePair extra_params = 5; + case 5: + if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 42)) { + ptr -= 1; + do { + ptr += 1; + ptr = ctx->ParseMessage(add_extra_params(), ptr); + CHK_(ptr); + if (!ctx->DataAvailable(ptr)) break; + } while (::PROTOBUF_NAMESPACE_ID::internal::UnalignedLoad<::PROTOBUF_NAMESPACE_ID::uint8>(ptr) == 42); + } else goto handle_unusual; + continue; default: { handle_unusual: if ((tag & 7) == 4 || tag == 0) { @@ -3168,6 +3581,17 @@ bool InsertParam::MergePartialFromCodedStream( break; } + // repeated .milvus.grpc.KeyValuePair extra_params = 5; + case 5: { + if (static_cast< ::PROTOBUF_NAMESPACE_ID::uint8>(tag) == (42 & 0xFF)) { + DO_(::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::ReadMessage( + input, add_extra_params())); + } else { + goto handle_unusual; + } + break; + } + default: { handle_unusual: if (tag == 0) { @@ -3235,6 +3659,15 @@ void InsertParam::SerializeWithCachedSizes( 4, this->partition_tag(), output); } + // repeated .milvus.grpc.KeyValuePair extra_params = 5; + for (unsigned int i = 0, + n = static_cast(this->extra_params_size()); i < n; i++) { + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteMessageMaybeToArray( + 5, + this->extra_params(static_cast(i)), + output); + } + if (_internal_metadata_.have_unknown_fields()) { ::PROTOBUF_NAMESPACE_ID::internal::WireFormat::SerializeUnknownFields( _internal_metadata_.unknown_fields(), output); @@ -3291,6 +3724,14 @@ void InsertParam::SerializeWithCachedSizes( 4, this->partition_tag(), target); } + // repeated .milvus.grpc.KeyValuePair extra_params = 5; + for (unsigned int i = 0, + n = static_cast(this->extra_params_size()); i < n; i++) { + target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite:: + InternalWriteMessageToArray( + 5, this->extra_params(static_cast(i)), target); + } + if (_internal_metadata_.have_unknown_fields()) { target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormat::SerializeUnknownFieldsToArray( _internal_metadata_.unknown_fields(), target); @@ -3338,6 +3779,17 @@ size_t InsertParam::ByteSizeLong() const { total_size += data_size; } + // repeated .milvus.grpc.KeyValuePair extra_params = 5; + { + unsigned int count = static_cast(this->extra_params_size()); + total_size += 1UL * count; + for (unsigned int i = 0; i < count; i++) { + total_size += + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::MessageSize( + this->extra_params(static_cast(i))); + } + } + // string table_name = 1; if (this->table_name().size() > 0) { total_size += 1 + @@ -3381,6 +3833,7 @@ void InsertParam::MergeFrom(const InsertParam& from) { row_record_array_.MergeFrom(from.row_record_array_); row_id_array_.MergeFrom(from.row_id_array_); + extra_params_.MergeFrom(from.extra_params_); if (from.table_name().size() > 0) { table_name_.AssignWithDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), from.table_name_); @@ -3414,6 +3867,7 @@ void InsertParam::InternalSwap(InsertParam* other) { _internal_metadata_.Swap(&other->_internal_metadata_); CastToBase(&row_record_array_)->InternalSwap(CastToBase(&other->row_record_array_)); row_id_array_.InternalSwap(&other->row_id_array_); + CastToBase(&extra_params_)->InternalSwap(CastToBase(&other->extra_params_)); table_name_.Swap(&other->table_name_, &::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), GetArenaNoVirtual()); partition_tag_.Swap(&other->partition_tag_, &::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), @@ -3782,25 +4236,22 @@ SearchParam::SearchParam() SearchParam::SearchParam(const SearchParam& from) : ::PROTOBUF_NAMESPACE_ID::Message(), _internal_metadata_(nullptr), + partition_tag_array_(from.partition_tag_array_), query_record_array_(from.query_record_array_), - partition_tag_array_(from.partition_tag_array_) { + extra_params_(from.extra_params_) { _internal_metadata_.MergeFrom(from._internal_metadata_); table_name_.UnsafeSetDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); if (!from.table_name().empty()) { table_name_.AssignWithDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), from.table_name_); } - ::memcpy(&topk_, &from.topk_, - static_cast(reinterpret_cast(&nprobe_) - - reinterpret_cast(&topk_)) + sizeof(nprobe_)); + topk_ = from.topk_; // @@protoc_insertion_point(copy_constructor:milvus.grpc.SearchParam) } void SearchParam::SharedCtor() { ::PROTOBUF_NAMESPACE_ID::internal::InitSCC(&scc_info_SearchParam_milvus_2eproto.base); table_name_.UnsafeSetDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); - ::memset(&topk_, 0, static_cast( - reinterpret_cast(&nprobe_) - - reinterpret_cast(&topk_)) + sizeof(nprobe_)); + topk_ = PROTOBUF_LONGLONG(0); } SearchParam::~SearchParam() { @@ -3827,12 +4278,11 @@ void SearchParam::Clear() { // Prevent compiler warnings about cached_has_bits being unused (void) cached_has_bits; - query_record_array_.Clear(); partition_tag_array_.Clear(); + query_record_array_.Clear(); + extra_params_.Clear(); table_name_.ClearToEmptyNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); - ::memset(&topk_, 0, static_cast( - reinterpret_cast(&nprobe_) - - reinterpret_cast(&topk_)) + sizeof(nprobe_)); + topk_ = PROTOBUF_LONGLONG(0); _internal_metadata_.Clear(); } @@ -3851,39 +4301,44 @@ const char* SearchParam::_InternalParse(const char* ptr, ::PROTOBUF_NAMESPACE_ID CHK_(ptr); } else goto handle_unusual; continue; - // repeated .milvus.grpc.RowRecord query_record_array = 2; + // repeated string partition_tag_array = 2; case 2: if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 18)) { + ptr -= 1; + do { + ptr += 1; + ptr = ::PROTOBUF_NAMESPACE_ID::internal::InlineGreedyStringParserUTF8(add_partition_tag_array(), ptr, ctx, "milvus.grpc.SearchParam.partition_tag_array"); + CHK_(ptr); + if (!ctx->DataAvailable(ptr)) break; + } while (::PROTOBUF_NAMESPACE_ID::internal::UnalignedLoad<::PROTOBUF_NAMESPACE_ID::uint8>(ptr) == 18); + } else goto handle_unusual; + continue; + // repeated .milvus.grpc.RowRecord query_record_array = 3; + case 3: + if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 26)) { ptr -= 1; do { ptr += 1; ptr = ctx->ParseMessage(add_query_record_array(), ptr); CHK_(ptr); if (!ctx->DataAvailable(ptr)) break; - } while (::PROTOBUF_NAMESPACE_ID::internal::UnalignedLoad<::PROTOBUF_NAMESPACE_ID::uint8>(ptr) == 18); + } while (::PROTOBUF_NAMESPACE_ID::internal::UnalignedLoad<::PROTOBUF_NAMESPACE_ID::uint8>(ptr) == 26); } else goto handle_unusual; continue; - // int64 topk = 3; - case 3: - if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 24)) { + // int64 topk = 4; + case 4: + if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 32)) { topk_ = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint(&ptr); CHK_(ptr); } else goto handle_unusual; continue; - // int64 nprobe = 4; - case 4: - if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 32)) { - nprobe_ = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint(&ptr); - CHK_(ptr); - } else goto handle_unusual; - continue; - // repeated string partition_tag_array = 5; + // repeated .milvus.grpc.KeyValuePair extra_params = 5; case 5: if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 42)) { ptr -= 1; do { ptr += 1; - ptr = ::PROTOBUF_NAMESPACE_ID::internal::InlineGreedyStringParserUTF8(add_partition_tag_array(), ptr, ctx, "milvus.grpc.SearchParam.partition_tag_array"); + ptr = ctx->ParseMessage(add_extra_params(), ptr); CHK_(ptr); if (!ctx->DataAvailable(ptr)) break; } while (::PROTOBUF_NAMESPACE_ID::internal::UnalignedLoad<::PROTOBUF_NAMESPACE_ID::uint8>(ptr) == 42); @@ -3934,9 +4389,25 @@ bool SearchParam::MergePartialFromCodedStream( break; } - // repeated .milvus.grpc.RowRecord query_record_array = 2; + // repeated string partition_tag_array = 2; case 2: { if (static_cast< ::PROTOBUF_NAMESPACE_ID::uint8>(tag) == (18 & 0xFF)) { + DO_(::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::ReadString( + input, this->add_partition_tag_array())); + DO_(::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::VerifyUtf8String( + this->partition_tag_array(this->partition_tag_array_size() - 1).data(), + static_cast(this->partition_tag_array(this->partition_tag_array_size() - 1).length()), + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::PARSE, + "milvus.grpc.SearchParam.partition_tag_array")); + } else { + goto handle_unusual; + } + break; + } + + // repeated .milvus.grpc.RowRecord query_record_array = 3; + case 3: { + if (static_cast< ::PROTOBUF_NAMESPACE_ID::uint8>(tag) == (26 & 0xFF)) { DO_(::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::ReadMessage( input, add_query_record_array())); } else { @@ -3945,9 +4416,9 @@ bool SearchParam::MergePartialFromCodedStream( break; } - // int64 topk = 3; - case 3: { - if (static_cast< ::PROTOBUF_NAMESPACE_ID::uint8>(tag) == (24 & 0xFF)) { + // int64 topk = 4; + case 4: { + if (static_cast< ::PROTOBUF_NAMESPACE_ID::uint8>(tag) == (32 & 0xFF)) { DO_((::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::ReadPrimitive< ::PROTOBUF_NAMESPACE_ID::int64, ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::TYPE_INT64>( @@ -3958,29 +4429,11 @@ bool SearchParam::MergePartialFromCodedStream( break; } - // int64 nprobe = 4; - case 4: { - if (static_cast< ::PROTOBUF_NAMESPACE_ID::uint8>(tag) == (32 & 0xFF)) { - - DO_((::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::ReadPrimitive< - ::PROTOBUF_NAMESPACE_ID::int64, ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::TYPE_INT64>( - input, &nprobe_))); - } else { - goto handle_unusual; - } - break; - } - - // repeated string partition_tag_array = 5; + // repeated .milvus.grpc.KeyValuePair extra_params = 5; case 5: { if (static_cast< ::PROTOBUF_NAMESPACE_ID::uint8>(tag) == (42 & 0xFF)) { - DO_(::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::ReadString( - input, this->add_partition_tag_array())); - DO_(::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::VerifyUtf8String( - this->partition_tag_array(this->partition_tag_array_size() - 1).data(), - static_cast(this->partition_tag_array(this->partition_tag_array_size() - 1).length()), - ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::PARSE, - "milvus.grpc.SearchParam.partition_tag_array")); + DO_(::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::ReadMessage( + input, add_extra_params())); } else { goto handle_unusual; } @@ -4024,33 +4477,37 @@ void SearchParam::SerializeWithCachedSizes( 1, this->table_name(), output); } - // repeated .milvus.grpc.RowRecord query_record_array = 2; - for (unsigned int i = 0, - n = static_cast(this->query_record_array_size()); i < n; i++) { - ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteMessageMaybeToArray( - 2, - this->query_record_array(static_cast(i)), - output); - } - - // int64 topk = 3; - if (this->topk() != 0) { - ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt64(3, this->topk(), output); - } - - // int64 nprobe = 4; - if (this->nprobe() != 0) { - ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt64(4, this->nprobe(), output); - } - - // repeated string partition_tag_array = 5; + // repeated string partition_tag_array = 2; for (int i = 0, n = this->partition_tag_array_size(); i < n; i++) { ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::VerifyUtf8String( this->partition_tag_array(i).data(), static_cast(this->partition_tag_array(i).length()), ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::SERIALIZE, "milvus.grpc.SearchParam.partition_tag_array"); ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteString( - 5, this->partition_tag_array(i), output); + 2, this->partition_tag_array(i), output); + } + + // repeated .milvus.grpc.RowRecord query_record_array = 3; + for (unsigned int i = 0, + n = static_cast(this->query_record_array_size()); i < n; i++) { + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteMessageMaybeToArray( + 3, + this->query_record_array(static_cast(i)), + output); + } + + // int64 topk = 4; + if (this->topk() != 0) { + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt64(4, this->topk(), output); + } + + // repeated .milvus.grpc.KeyValuePair extra_params = 5; + for (unsigned int i = 0, + n = static_cast(this->extra_params_size()); i < n; i++) { + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteMessageMaybeToArray( + 5, + this->extra_params(static_cast(i)), + output); } if (_internal_metadata_.have_unknown_fields()) { @@ -4077,32 +4534,35 @@ void SearchParam::SerializeWithCachedSizes( 1, this->table_name(), target); } - // repeated .milvus.grpc.RowRecord query_record_array = 2; - for (unsigned int i = 0, - n = static_cast(this->query_record_array_size()); i < n; i++) { - target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite:: - InternalWriteMessageToArray( - 2, this->query_record_array(static_cast(i)), target); - } - - // int64 topk = 3; - if (this->topk() != 0) { - target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt64ToArray(3, this->topk(), target); - } - - // int64 nprobe = 4; - if (this->nprobe() != 0) { - target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt64ToArray(4, this->nprobe(), target); - } - - // repeated string partition_tag_array = 5; + // repeated string partition_tag_array = 2; for (int i = 0, n = this->partition_tag_array_size(); i < n; i++) { ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::VerifyUtf8String( this->partition_tag_array(i).data(), static_cast(this->partition_tag_array(i).length()), ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::SERIALIZE, "milvus.grpc.SearchParam.partition_tag_array"); target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite:: - WriteStringToArray(5, this->partition_tag_array(i), target); + WriteStringToArray(2, this->partition_tag_array(i), target); + } + + // repeated .milvus.grpc.RowRecord query_record_array = 3; + for (unsigned int i = 0, + n = static_cast(this->query_record_array_size()); i < n; i++) { + target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite:: + InternalWriteMessageToArray( + 3, this->query_record_array(static_cast(i)), target); + } + + // int64 topk = 4; + if (this->topk() != 0) { + target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt64ToArray(4, this->topk(), target); + } + + // repeated .milvus.grpc.KeyValuePair extra_params = 5; + for (unsigned int i = 0, + n = static_cast(this->extra_params_size()); i < n; i++) { + target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite:: + InternalWriteMessageToArray( + 5, this->extra_params(static_cast(i)), target); } if (_internal_metadata_.have_unknown_fields()) { @@ -4126,7 +4586,15 @@ size_t SearchParam::ByteSizeLong() const { // Prevent compiler warnings about cached_has_bits being unused (void) cached_has_bits; - // repeated .milvus.grpc.RowRecord query_record_array = 2; + // repeated string partition_tag_array = 2; + total_size += 1 * + ::PROTOBUF_NAMESPACE_ID::internal::FromIntSize(this->partition_tag_array_size()); + for (int i = 0, n = this->partition_tag_array_size(); i < n; i++) { + total_size += ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::StringSize( + this->partition_tag_array(i)); + } + + // repeated .milvus.grpc.RowRecord query_record_array = 3; { unsigned int count = static_cast(this->query_record_array_size()); total_size += 1UL * count; @@ -4137,12 +4605,15 @@ size_t SearchParam::ByteSizeLong() const { } } - // repeated string partition_tag_array = 5; - total_size += 1 * - ::PROTOBUF_NAMESPACE_ID::internal::FromIntSize(this->partition_tag_array_size()); - for (int i = 0, n = this->partition_tag_array_size(); i < n; i++) { - total_size += ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::StringSize( - this->partition_tag_array(i)); + // repeated .milvus.grpc.KeyValuePair extra_params = 5; + { + unsigned int count = static_cast(this->extra_params_size()); + total_size += 1UL * count; + for (unsigned int i = 0; i < count; i++) { + total_size += + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::MessageSize( + this->extra_params(static_cast(i))); + } } // string table_name = 1; @@ -4152,20 +4623,13 @@ size_t SearchParam::ByteSizeLong() const { this->table_name()); } - // int64 topk = 3; + // int64 topk = 4; if (this->topk() != 0) { total_size += 1 + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::Int64Size( this->topk()); } - // int64 nprobe = 4; - if (this->nprobe() != 0) { - total_size += 1 + - ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::Int64Size( - this->nprobe()); - } - int cached_size = ::PROTOBUF_NAMESPACE_ID::internal::ToCachedSize(total_size); SetCachedSize(cached_size); return total_size; @@ -4193,8 +4657,9 @@ void SearchParam::MergeFrom(const SearchParam& from) { ::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0; (void) cached_has_bits; - query_record_array_.MergeFrom(from.query_record_array_); partition_tag_array_.MergeFrom(from.partition_tag_array_); + query_record_array_.MergeFrom(from.query_record_array_); + extra_params_.MergeFrom(from.extra_params_); if (from.table_name().size() > 0) { table_name_.AssignWithDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), from.table_name_); @@ -4202,9 +4667,6 @@ void SearchParam::MergeFrom(const SearchParam& from) { if (from.topk() != 0) { set_topk(from.topk()); } - if (from.nprobe() != 0) { - set_nprobe(from.nprobe()); - } } void SearchParam::CopyFrom(const ::PROTOBUF_NAMESPACE_ID::Message& from) { @@ -4228,12 +4690,12 @@ bool SearchParam::IsInitialized() const { void SearchParam::InternalSwap(SearchParam* other) { using std::swap; _internal_metadata_.Swap(&other->_internal_metadata_); - CastToBase(&query_record_array_)->InternalSwap(CastToBase(&other->query_record_array_)); partition_tag_array_.InternalSwap(CastToBase(&other->partition_tag_array_)); + CastToBase(&query_record_array_)->InternalSwap(CastToBase(&other->query_record_array_)); + CastToBase(&extra_params_)->InternalSwap(CastToBase(&other->extra_params_)); table_name_.Swap(&other->table_name_, &::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), GetArenaNoVirtual()); swap(topk_, other->topk_); - swap(nprobe_, other->nprobe_); } ::PROTOBUF_NAMESPACE_ID::Metadata SearchParam::GetMetadata() const { @@ -4583,15 +5045,16 @@ SearchByIDParam::SearchByIDParam() SearchByIDParam::SearchByIDParam(const SearchByIDParam& from) : ::PROTOBUF_NAMESPACE_ID::Message(), _internal_metadata_(nullptr), - partition_tag_array_(from.partition_tag_array_) { + partition_tag_array_(from.partition_tag_array_), + extra_params_(from.extra_params_) { _internal_metadata_.MergeFrom(from._internal_metadata_); table_name_.UnsafeSetDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); if (!from.table_name().empty()) { table_name_.AssignWithDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), from.table_name_); } ::memcpy(&id_, &from.id_, - static_cast(reinterpret_cast(&nprobe_) - - reinterpret_cast(&id_)) + sizeof(nprobe_)); + static_cast(reinterpret_cast(&topk_) - + reinterpret_cast(&id_)) + sizeof(topk_)); // @@protoc_insertion_point(copy_constructor:milvus.grpc.SearchByIDParam) } @@ -4599,8 +5062,8 @@ void SearchByIDParam::SharedCtor() { ::PROTOBUF_NAMESPACE_ID::internal::InitSCC(&scc_info_SearchByIDParam_milvus_2eproto.base); table_name_.UnsafeSetDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); ::memset(&id_, 0, static_cast( - reinterpret_cast(&nprobe_) - - reinterpret_cast(&id_)) + sizeof(nprobe_)); + reinterpret_cast(&topk_) - + reinterpret_cast(&id_)) + sizeof(topk_)); } SearchByIDParam::~SearchByIDParam() { @@ -4628,10 +5091,11 @@ void SearchByIDParam::Clear() { (void) cached_has_bits; partition_tag_array_.Clear(); + extra_params_.Clear(); table_name_.ClearToEmptyNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); ::memset(&id_, 0, static_cast( - reinterpret_cast(&nprobe_) - - reinterpret_cast(&id_)) + sizeof(nprobe_)); + reinterpret_cast(&topk_) - + reinterpret_cast(&id_)) + sizeof(topk_)); _internal_metadata_.Clear(); } @@ -4650,34 +5114,39 @@ const char* SearchByIDParam::_InternalParse(const char* ptr, ::PROTOBUF_NAMESPAC CHK_(ptr); } else goto handle_unusual; continue; - // int64 id = 2; + // repeated string partition_tag_array = 2; case 2: - if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 16)) { + if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 18)) { + ptr -= 1; + do { + ptr += 1; + ptr = ::PROTOBUF_NAMESPACE_ID::internal::InlineGreedyStringParserUTF8(add_partition_tag_array(), ptr, ctx, "milvus.grpc.SearchByIDParam.partition_tag_array"); + CHK_(ptr); + if (!ctx->DataAvailable(ptr)) break; + } while (::PROTOBUF_NAMESPACE_ID::internal::UnalignedLoad<::PROTOBUF_NAMESPACE_ID::uint8>(ptr) == 18); + } else goto handle_unusual; + continue; + // int64 id = 3; + case 3: + if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 24)) { id_ = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint(&ptr); CHK_(ptr); } else goto handle_unusual; continue; - // int64 topk = 3; - case 3: - if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 24)) { + // int64 topk = 4; + case 4: + if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 32)) { topk_ = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint(&ptr); CHK_(ptr); } else goto handle_unusual; continue; - // int64 nprobe = 4; - case 4: - if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 32)) { - nprobe_ = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint(&ptr); - CHK_(ptr); - } else goto handle_unusual; - continue; - // repeated string partition_tag_array = 5; + // repeated .milvus.grpc.KeyValuePair extra_params = 5; case 5: if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 42)) { ptr -= 1; do { ptr += 1; - ptr = ::PROTOBUF_NAMESPACE_ID::internal::InlineGreedyStringParserUTF8(add_partition_tag_array(), ptr, ctx, "milvus.grpc.SearchByIDParam.partition_tag_array"); + ptr = ctx->ParseMessage(add_extra_params(), ptr); CHK_(ptr); if (!ctx->DataAvailable(ptr)) break; } while (::PROTOBUF_NAMESPACE_ID::internal::UnalignedLoad<::PROTOBUF_NAMESPACE_ID::uint8>(ptr) == 42); @@ -4728,9 +5197,25 @@ bool SearchByIDParam::MergePartialFromCodedStream( break; } - // int64 id = 2; + // repeated string partition_tag_array = 2; case 2: { - if (static_cast< ::PROTOBUF_NAMESPACE_ID::uint8>(tag) == (16 & 0xFF)) { + if (static_cast< ::PROTOBUF_NAMESPACE_ID::uint8>(tag) == (18 & 0xFF)) { + DO_(::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::ReadString( + input, this->add_partition_tag_array())); + DO_(::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::VerifyUtf8String( + this->partition_tag_array(this->partition_tag_array_size() - 1).data(), + static_cast(this->partition_tag_array(this->partition_tag_array_size() - 1).length()), + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::PARSE, + "milvus.grpc.SearchByIDParam.partition_tag_array")); + } else { + goto handle_unusual; + } + break; + } + + // int64 id = 3; + case 3: { + if (static_cast< ::PROTOBUF_NAMESPACE_ID::uint8>(tag) == (24 & 0xFF)) { DO_((::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::ReadPrimitive< ::PROTOBUF_NAMESPACE_ID::int64, ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::TYPE_INT64>( @@ -4741,9 +5226,9 @@ bool SearchByIDParam::MergePartialFromCodedStream( break; } - // int64 topk = 3; - case 3: { - if (static_cast< ::PROTOBUF_NAMESPACE_ID::uint8>(tag) == (24 & 0xFF)) { + // int64 topk = 4; + case 4: { + if (static_cast< ::PROTOBUF_NAMESPACE_ID::uint8>(tag) == (32 & 0xFF)) { DO_((::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::ReadPrimitive< ::PROTOBUF_NAMESPACE_ID::int64, ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::TYPE_INT64>( @@ -4754,29 +5239,11 @@ bool SearchByIDParam::MergePartialFromCodedStream( break; } - // int64 nprobe = 4; - case 4: { - if (static_cast< ::PROTOBUF_NAMESPACE_ID::uint8>(tag) == (32 & 0xFF)) { - - DO_((::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::ReadPrimitive< - ::PROTOBUF_NAMESPACE_ID::int64, ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::TYPE_INT64>( - input, &nprobe_))); - } else { - goto handle_unusual; - } - break; - } - - // repeated string partition_tag_array = 5; + // repeated .milvus.grpc.KeyValuePair extra_params = 5; case 5: { if (static_cast< ::PROTOBUF_NAMESPACE_ID::uint8>(tag) == (42 & 0xFF)) { - DO_(::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::ReadString( - input, this->add_partition_tag_array())); - DO_(::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::VerifyUtf8String( - this->partition_tag_array(this->partition_tag_array_size() - 1).data(), - static_cast(this->partition_tag_array(this->partition_tag_array_size() - 1).length()), - ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::PARSE, - "milvus.grpc.SearchByIDParam.partition_tag_array")); + DO_(::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::ReadMessage( + input, add_extra_params())); } else { goto handle_unusual; } @@ -4820,29 +5287,33 @@ void SearchByIDParam::SerializeWithCachedSizes( 1, this->table_name(), output); } - // int64 id = 2; - if (this->id() != 0) { - ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt64(2, this->id(), output); - } - - // int64 topk = 3; - if (this->topk() != 0) { - ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt64(3, this->topk(), output); - } - - // int64 nprobe = 4; - if (this->nprobe() != 0) { - ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt64(4, this->nprobe(), output); - } - - // repeated string partition_tag_array = 5; + // repeated string partition_tag_array = 2; for (int i = 0, n = this->partition_tag_array_size(); i < n; i++) { ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::VerifyUtf8String( this->partition_tag_array(i).data(), static_cast(this->partition_tag_array(i).length()), ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::SERIALIZE, "milvus.grpc.SearchByIDParam.partition_tag_array"); ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteString( - 5, this->partition_tag_array(i), output); + 2, this->partition_tag_array(i), output); + } + + // int64 id = 3; + if (this->id() != 0) { + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt64(3, this->id(), output); + } + + // int64 topk = 4; + if (this->topk() != 0) { + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt64(4, this->topk(), output); + } + + // repeated .milvus.grpc.KeyValuePair extra_params = 5; + for (unsigned int i = 0, + n = static_cast(this->extra_params_size()); i < n; i++) { + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteMessageMaybeToArray( + 5, + this->extra_params(static_cast(i)), + output); } if (_internal_metadata_.have_unknown_fields()) { @@ -4869,29 +5340,32 @@ void SearchByIDParam::SerializeWithCachedSizes( 1, this->table_name(), target); } - // int64 id = 2; - if (this->id() != 0) { - target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt64ToArray(2, this->id(), target); - } - - // int64 topk = 3; - if (this->topk() != 0) { - target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt64ToArray(3, this->topk(), target); - } - - // int64 nprobe = 4; - if (this->nprobe() != 0) { - target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt64ToArray(4, this->nprobe(), target); - } - - // repeated string partition_tag_array = 5; + // repeated string partition_tag_array = 2; for (int i = 0, n = this->partition_tag_array_size(); i < n; i++) { ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::VerifyUtf8String( this->partition_tag_array(i).data(), static_cast(this->partition_tag_array(i).length()), ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::SERIALIZE, "milvus.grpc.SearchByIDParam.partition_tag_array"); target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite:: - WriteStringToArray(5, this->partition_tag_array(i), target); + WriteStringToArray(2, this->partition_tag_array(i), target); + } + + // int64 id = 3; + if (this->id() != 0) { + target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt64ToArray(3, this->id(), target); + } + + // int64 topk = 4; + if (this->topk() != 0) { + target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt64ToArray(4, this->topk(), target); + } + + // repeated .milvus.grpc.KeyValuePair extra_params = 5; + for (unsigned int i = 0, + n = static_cast(this->extra_params_size()); i < n; i++) { + target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite:: + InternalWriteMessageToArray( + 5, this->extra_params(static_cast(i)), target); } if (_internal_metadata_.have_unknown_fields()) { @@ -4915,7 +5389,7 @@ size_t SearchByIDParam::ByteSizeLong() const { // Prevent compiler warnings about cached_has_bits being unused (void) cached_has_bits; - // repeated string partition_tag_array = 5; + // repeated string partition_tag_array = 2; total_size += 1 * ::PROTOBUF_NAMESPACE_ID::internal::FromIntSize(this->partition_tag_array_size()); for (int i = 0, n = this->partition_tag_array_size(); i < n; i++) { @@ -4923,6 +5397,17 @@ size_t SearchByIDParam::ByteSizeLong() const { this->partition_tag_array(i)); } + // repeated .milvus.grpc.KeyValuePair extra_params = 5; + { + unsigned int count = static_cast(this->extra_params_size()); + total_size += 1UL * count; + for (unsigned int i = 0; i < count; i++) { + total_size += + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::MessageSize( + this->extra_params(static_cast(i))); + } + } + // string table_name = 1; if (this->table_name().size() > 0) { total_size += 1 + @@ -4930,27 +5415,20 @@ size_t SearchByIDParam::ByteSizeLong() const { this->table_name()); } - // int64 id = 2; + // int64 id = 3; if (this->id() != 0) { total_size += 1 + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::Int64Size( this->id()); } - // int64 topk = 3; + // int64 topk = 4; if (this->topk() != 0) { total_size += 1 + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::Int64Size( this->topk()); } - // int64 nprobe = 4; - if (this->nprobe() != 0) { - total_size += 1 + - ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::Int64Size( - this->nprobe()); - } - int cached_size = ::PROTOBUF_NAMESPACE_ID::internal::ToCachedSize(total_size); SetCachedSize(cached_size); return total_size; @@ -4979,6 +5457,7 @@ void SearchByIDParam::MergeFrom(const SearchByIDParam& from) { (void) cached_has_bits; partition_tag_array_.MergeFrom(from.partition_tag_array_); + extra_params_.MergeFrom(from.extra_params_); if (from.table_name().size() > 0) { table_name_.AssignWithDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), from.table_name_); @@ -4989,9 +5468,6 @@ void SearchByIDParam::MergeFrom(const SearchByIDParam& from) { if (from.topk() != 0) { set_topk(from.topk()); } - if (from.nprobe() != 0) { - set_nprobe(from.nprobe()); - } } void SearchByIDParam::CopyFrom(const ::PROTOBUF_NAMESPACE_ID::Message& from) { @@ -5016,11 +5492,11 @@ void SearchByIDParam::InternalSwap(SearchByIDParam* other) { using std::swap; _internal_metadata_.Swap(&other->_internal_metadata_); partition_tag_array_.InternalSwap(CastToBase(&other->partition_tag_array_)); + CastToBase(&extra_params_)->InternalSwap(CastToBase(&other->extra_params_)); table_name_.Swap(&other->table_name_, &::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), GetArenaNoVirtual()); swap(id_, other->id_); swap(topk_, other->topk_); - swap(nprobe_, other->nprobe_); } ::PROTOBUF_NAMESPACE_ID::Metadata SearchByIDParam::GetMetadata() const { @@ -6717,324 +7193,21 @@ void Command::InternalSwap(Command* other) { } -// =================================================================== - -void Index::InitAsDefaultInstance() { -} -class Index::_Internal { - public: -}; - -Index::Index() - : ::PROTOBUF_NAMESPACE_ID::Message(), _internal_metadata_(nullptr) { - SharedCtor(); - // @@protoc_insertion_point(constructor:milvus.grpc.Index) -} -Index::Index(const Index& from) - : ::PROTOBUF_NAMESPACE_ID::Message(), - _internal_metadata_(nullptr) { - _internal_metadata_.MergeFrom(from._internal_metadata_); - ::memcpy(&index_type_, &from.index_type_, - static_cast(reinterpret_cast(&nlist_) - - reinterpret_cast(&index_type_)) + sizeof(nlist_)); - // @@protoc_insertion_point(copy_constructor:milvus.grpc.Index) -} - -void Index::SharedCtor() { - ::memset(&index_type_, 0, static_cast( - reinterpret_cast(&nlist_) - - reinterpret_cast(&index_type_)) + sizeof(nlist_)); -} - -Index::~Index() { - // @@protoc_insertion_point(destructor:milvus.grpc.Index) - SharedDtor(); -} - -void Index::SharedDtor() { -} - -void Index::SetCachedSize(int size) const { - _cached_size_.Set(size); -} -const Index& Index::default_instance() { - ::PROTOBUF_NAMESPACE_ID::internal::InitSCC(&::scc_info_Index_milvus_2eproto.base); - return *internal_default_instance(); -} - - -void Index::Clear() { -// @@protoc_insertion_point(message_clear_start:milvus.grpc.Index) - ::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0; - // Prevent compiler warnings about cached_has_bits being unused - (void) cached_has_bits; - - ::memset(&index_type_, 0, static_cast( - reinterpret_cast(&nlist_) - - reinterpret_cast(&index_type_)) + sizeof(nlist_)); - _internal_metadata_.Clear(); -} - -#if GOOGLE_PROTOBUF_ENABLE_EXPERIMENTAL_PARSER -const char* Index::_InternalParse(const char* ptr, ::PROTOBUF_NAMESPACE_ID::internal::ParseContext* ctx) { -#define CHK_(x) if (PROTOBUF_PREDICT_FALSE(!(x))) goto failure - while (!ctx->Done(&ptr)) { - ::PROTOBUF_NAMESPACE_ID::uint32 tag; - ptr = ::PROTOBUF_NAMESPACE_ID::internal::ReadTag(ptr, &tag); - CHK_(ptr); - switch (tag >> 3) { - // int32 index_type = 1; - case 1: - if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 8)) { - index_type_ = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint(&ptr); - CHK_(ptr); - } else goto handle_unusual; - continue; - // int32 nlist = 2; - case 2: - if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 16)) { - nlist_ = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint(&ptr); - CHK_(ptr); - } else goto handle_unusual; - continue; - default: { - handle_unusual: - if ((tag & 7) == 4 || tag == 0) { - ctx->SetLastTag(tag); - goto success; - } - ptr = UnknownFieldParse(tag, &_internal_metadata_, ptr, ctx); - CHK_(ptr != nullptr); - continue; - } - } // switch - } // while -success: - return ptr; -failure: - ptr = nullptr; - goto success; -#undef CHK_ -} -#else // GOOGLE_PROTOBUF_ENABLE_EXPERIMENTAL_PARSER -bool Index::MergePartialFromCodedStream( - ::PROTOBUF_NAMESPACE_ID::io::CodedInputStream* input) { -#define DO_(EXPRESSION) if (!PROTOBUF_PREDICT_TRUE(EXPRESSION)) goto failure - ::PROTOBUF_NAMESPACE_ID::uint32 tag; - // @@protoc_insertion_point(parse_start:milvus.grpc.Index) - for (;;) { - ::std::pair<::PROTOBUF_NAMESPACE_ID::uint32, bool> p = input->ReadTagWithCutoffNoLastTag(127u); - tag = p.first; - if (!p.second) goto handle_unusual; - switch (::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::GetTagFieldNumber(tag)) { - // int32 index_type = 1; - case 1: { - if (static_cast< ::PROTOBUF_NAMESPACE_ID::uint8>(tag) == (8 & 0xFF)) { - - DO_((::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::ReadPrimitive< - ::PROTOBUF_NAMESPACE_ID::int32, ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::TYPE_INT32>( - input, &index_type_))); - } else { - goto handle_unusual; - } - break; - } - - // int32 nlist = 2; - case 2: { - if (static_cast< ::PROTOBUF_NAMESPACE_ID::uint8>(tag) == (16 & 0xFF)) { - - DO_((::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::ReadPrimitive< - ::PROTOBUF_NAMESPACE_ID::int32, ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::TYPE_INT32>( - input, &nlist_))); - } else { - goto handle_unusual; - } - break; - } - - default: { - handle_unusual: - if (tag == 0) { - goto success; - } - DO_(::PROTOBUF_NAMESPACE_ID::internal::WireFormat::SkipField( - input, tag, _internal_metadata_.mutable_unknown_fields())); - break; - } - } - } -success: - // @@protoc_insertion_point(parse_success:milvus.grpc.Index) - return true; -failure: - // @@protoc_insertion_point(parse_failure:milvus.grpc.Index) - return false; -#undef DO_ -} -#endif // GOOGLE_PROTOBUF_ENABLE_EXPERIMENTAL_PARSER - -void Index::SerializeWithCachedSizes( - ::PROTOBUF_NAMESPACE_ID::io::CodedOutputStream* output) const { - // @@protoc_insertion_point(serialize_start:milvus.grpc.Index) - ::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0; - (void) cached_has_bits; - - // int32 index_type = 1; - if (this->index_type() != 0) { - ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt32(1, this->index_type(), output); - } - - // int32 nlist = 2; - if (this->nlist() != 0) { - ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt32(2, this->nlist(), output); - } - - if (_internal_metadata_.have_unknown_fields()) { - ::PROTOBUF_NAMESPACE_ID::internal::WireFormat::SerializeUnknownFields( - _internal_metadata_.unknown_fields(), output); - } - // @@protoc_insertion_point(serialize_end:milvus.grpc.Index) -} - -::PROTOBUF_NAMESPACE_ID::uint8* Index::InternalSerializeWithCachedSizesToArray( - ::PROTOBUF_NAMESPACE_ID::uint8* target) const { - // @@protoc_insertion_point(serialize_to_array_start:milvus.grpc.Index) - ::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0; - (void) cached_has_bits; - - // int32 index_type = 1; - if (this->index_type() != 0) { - target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt32ToArray(1, this->index_type(), target); - } - - // int32 nlist = 2; - if (this->nlist() != 0) { - target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt32ToArray(2, this->nlist(), target); - } - - if (_internal_metadata_.have_unknown_fields()) { - target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormat::SerializeUnknownFieldsToArray( - _internal_metadata_.unknown_fields(), target); - } - // @@protoc_insertion_point(serialize_to_array_end:milvus.grpc.Index) - return target; -} - -size_t Index::ByteSizeLong() const { -// @@protoc_insertion_point(message_byte_size_start:milvus.grpc.Index) - size_t total_size = 0; - - if (_internal_metadata_.have_unknown_fields()) { - total_size += - ::PROTOBUF_NAMESPACE_ID::internal::WireFormat::ComputeUnknownFieldsSize( - _internal_metadata_.unknown_fields()); - } - ::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0; - // Prevent compiler warnings about cached_has_bits being unused - (void) cached_has_bits; - - // int32 index_type = 1; - if (this->index_type() != 0) { - total_size += 1 + - ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::Int32Size( - this->index_type()); - } - - // int32 nlist = 2; - if (this->nlist() != 0) { - total_size += 1 + - ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::Int32Size( - this->nlist()); - } - - int cached_size = ::PROTOBUF_NAMESPACE_ID::internal::ToCachedSize(total_size); - SetCachedSize(cached_size); - return total_size; -} - -void Index::MergeFrom(const ::PROTOBUF_NAMESPACE_ID::Message& from) { -// @@protoc_insertion_point(generalized_merge_from_start:milvus.grpc.Index) - GOOGLE_DCHECK_NE(&from, this); - const Index* source = - ::PROTOBUF_NAMESPACE_ID::DynamicCastToGenerated( - &from); - if (source == nullptr) { - // @@protoc_insertion_point(generalized_merge_from_cast_fail:milvus.grpc.Index) - ::PROTOBUF_NAMESPACE_ID::internal::ReflectionOps::Merge(from, this); - } else { - // @@protoc_insertion_point(generalized_merge_from_cast_success:milvus.grpc.Index) - MergeFrom(*source); - } -} - -void Index::MergeFrom(const Index& from) { -// @@protoc_insertion_point(class_specific_merge_from_start:milvus.grpc.Index) - GOOGLE_DCHECK_NE(&from, this); - _internal_metadata_.MergeFrom(from._internal_metadata_); - ::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0; - (void) cached_has_bits; - - if (from.index_type() != 0) { - set_index_type(from.index_type()); - } - if (from.nlist() != 0) { - set_nlist(from.nlist()); - } -} - -void Index::CopyFrom(const ::PROTOBUF_NAMESPACE_ID::Message& from) { -// @@protoc_insertion_point(generalized_copy_from_start:milvus.grpc.Index) - if (&from == this) return; - Clear(); - MergeFrom(from); -} - -void Index::CopyFrom(const Index& from) { -// @@protoc_insertion_point(class_specific_copy_from_start:milvus.grpc.Index) - if (&from == this) return; - Clear(); - MergeFrom(from); -} - -bool Index::IsInitialized() const { - return true; -} - -void Index::InternalSwap(Index* other) { - using std::swap; - _internal_metadata_.Swap(&other->_internal_metadata_); - swap(index_type_, other->index_type_); - swap(nlist_, other->nlist_); -} - -::PROTOBUF_NAMESPACE_ID::Metadata Index::GetMetadata() const { - return GetMetadataStatic(); -} - - // =================================================================== void IndexParam::InitAsDefaultInstance() { ::milvus::grpc::_IndexParam_default_instance_._instance.get_mutable()->status_ = const_cast< ::milvus::grpc::Status*>( ::milvus::grpc::Status::internal_default_instance()); - ::milvus::grpc::_IndexParam_default_instance_._instance.get_mutable()->index_ = const_cast< ::milvus::grpc::Index*>( - ::milvus::grpc::Index::internal_default_instance()); } class IndexParam::_Internal { public: static const ::milvus::grpc::Status& status(const IndexParam* msg); - static const ::milvus::grpc::Index& index(const IndexParam* msg); }; const ::milvus::grpc::Status& IndexParam::_Internal::status(const IndexParam* msg) { return *msg->status_; } -const ::milvus::grpc::Index& -IndexParam::_Internal::index(const IndexParam* msg) { - return *msg->index_; -} void IndexParam::clear_status() { if (GetArenaNoVirtual() == nullptr && status_ != nullptr) { delete status_; @@ -7048,7 +7221,8 @@ IndexParam::IndexParam() } IndexParam::IndexParam(const IndexParam& from) : ::PROTOBUF_NAMESPACE_ID::Message(), - _internal_metadata_(nullptr) { + _internal_metadata_(nullptr), + extra_params_(from.extra_params_) { _internal_metadata_.MergeFrom(from._internal_metadata_); table_name_.UnsafeSetDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); if (!from.table_name().empty()) { @@ -7059,11 +7233,7 @@ IndexParam::IndexParam(const IndexParam& from) } else { status_ = nullptr; } - if (from.has_index()) { - index_ = new ::milvus::grpc::Index(*from.index_); - } else { - index_ = nullptr; - } + index_type_ = from.index_type_; // @@protoc_insertion_point(copy_constructor:milvus.grpc.IndexParam) } @@ -7071,8 +7241,8 @@ void IndexParam::SharedCtor() { ::PROTOBUF_NAMESPACE_ID::internal::InitSCC(&scc_info_IndexParam_milvus_2eproto.base); table_name_.UnsafeSetDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); ::memset(&status_, 0, static_cast( - reinterpret_cast(&index_) - - reinterpret_cast(&status_)) + sizeof(index_)); + reinterpret_cast(&index_type_) - + reinterpret_cast(&status_)) + sizeof(index_type_)); } IndexParam::~IndexParam() { @@ -7083,7 +7253,6 @@ IndexParam::~IndexParam() { void IndexParam::SharedDtor() { table_name_.DestroyNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); if (this != internal_default_instance()) delete status_; - if (this != internal_default_instance()) delete index_; } void IndexParam::SetCachedSize(int size) const { @@ -7101,15 +7270,13 @@ void IndexParam::Clear() { // Prevent compiler warnings about cached_has_bits being unused (void) cached_has_bits; + extra_params_.Clear(); table_name_.ClearToEmptyNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); if (GetArenaNoVirtual() == nullptr && status_ != nullptr) { delete status_; } status_ = nullptr; - if (GetArenaNoVirtual() == nullptr && index_ != nullptr) { - delete index_; - } - index_ = nullptr; + index_type_ = 0; _internal_metadata_.Clear(); } @@ -7135,13 +7302,25 @@ const char* IndexParam::_InternalParse(const char* ptr, ::PROTOBUF_NAMESPACE_ID: CHK_(ptr); } else goto handle_unusual; continue; - // .milvus.grpc.Index index = 3; + // int32 index_type = 3; case 3: - if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 26)) { - ptr = ctx->ParseMessage(mutable_index(), ptr); + if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 24)) { + index_type_ = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint(&ptr); CHK_(ptr); } else goto handle_unusual; continue; + // repeated .milvus.grpc.KeyValuePair extra_params = 4; + case 4: + if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 34)) { + ptr -= 1; + do { + ptr += 1; + ptr = ctx->ParseMessage(add_extra_params(), ptr); + CHK_(ptr); + if (!ctx->DataAvailable(ptr)) break; + } while (::PROTOBUF_NAMESPACE_ID::internal::UnalignedLoad<::PROTOBUF_NAMESPACE_ID::uint8>(ptr) == 34); + } else goto handle_unusual; + continue; default: { handle_unusual: if ((tag & 7) == 4 || tag == 0) { @@ -7198,11 +7377,24 @@ bool IndexParam::MergePartialFromCodedStream( break; } - // .milvus.grpc.Index index = 3; + // int32 index_type = 3; case 3: { - if (static_cast< ::PROTOBUF_NAMESPACE_ID::uint8>(tag) == (26 & 0xFF)) { + if (static_cast< ::PROTOBUF_NAMESPACE_ID::uint8>(tag) == (24 & 0xFF)) { + + DO_((::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::ReadPrimitive< + ::PROTOBUF_NAMESPACE_ID::int32, ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::TYPE_INT32>( + input, &index_type_))); + } else { + goto handle_unusual; + } + break; + } + + // repeated .milvus.grpc.KeyValuePair extra_params = 4; + case 4: { + if (static_cast< ::PROTOBUF_NAMESPACE_ID::uint8>(tag) == (34 & 0xFF)) { DO_(::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::ReadMessage( - input, mutable_index())); + input, add_extra_params())); } else { goto handle_unusual; } @@ -7252,10 +7444,18 @@ void IndexParam::SerializeWithCachedSizes( 2, this->table_name(), output); } - // .milvus.grpc.Index index = 3; - if (this->has_index()) { + // int32 index_type = 3; + if (this->index_type() != 0) { + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt32(3, this->index_type(), output); + } + + // repeated .milvus.grpc.KeyValuePair extra_params = 4; + for (unsigned int i = 0, + n = static_cast(this->extra_params_size()); i < n; i++) { ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteMessageMaybeToArray( - 3, _Internal::index(this), output); + 4, + this->extra_params(static_cast(i)), + output); } if (_internal_metadata_.have_unknown_fields()) { @@ -7289,11 +7489,17 @@ void IndexParam::SerializeWithCachedSizes( 2, this->table_name(), target); } - // .milvus.grpc.Index index = 3; - if (this->has_index()) { + // int32 index_type = 3; + if (this->index_type() != 0) { + target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt32ToArray(3, this->index_type(), target); + } + + // repeated .milvus.grpc.KeyValuePair extra_params = 4; + for (unsigned int i = 0, + n = static_cast(this->extra_params_size()); i < n; i++) { target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite:: InternalWriteMessageToArray( - 3, _Internal::index(this), target); + 4, this->extra_params(static_cast(i)), target); } if (_internal_metadata_.have_unknown_fields()) { @@ -7317,6 +7523,17 @@ size_t IndexParam::ByteSizeLong() const { // Prevent compiler warnings about cached_has_bits being unused (void) cached_has_bits; + // repeated .milvus.grpc.KeyValuePair extra_params = 4; + { + unsigned int count = static_cast(this->extra_params_size()); + total_size += 1UL * count; + for (unsigned int i = 0; i < count; i++) { + total_size += + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::MessageSize( + this->extra_params(static_cast(i))); + } + } + // string table_name = 2; if (this->table_name().size() > 0) { total_size += 1 + @@ -7331,11 +7548,11 @@ size_t IndexParam::ByteSizeLong() const { *status_); } - // .milvus.grpc.Index index = 3; - if (this->has_index()) { + // int32 index_type = 3; + if (this->index_type() != 0) { total_size += 1 + - ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::MessageSize( - *index_); + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::Int32Size( + this->index_type()); } int cached_size = ::PROTOBUF_NAMESPACE_ID::internal::ToCachedSize(total_size); @@ -7365,6 +7582,7 @@ void IndexParam::MergeFrom(const IndexParam& from) { ::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0; (void) cached_has_bits; + extra_params_.MergeFrom(from.extra_params_); if (from.table_name().size() > 0) { table_name_.AssignWithDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), from.table_name_); @@ -7372,8 +7590,8 @@ void IndexParam::MergeFrom(const IndexParam& from) { if (from.has_status()) { mutable_status()->::milvus::grpc::Status::MergeFrom(from.status()); } - if (from.has_index()) { - mutable_index()->::milvus::grpc::Index::MergeFrom(from.index()); + if (from.index_type() != 0) { + set_index_type(from.index_type()); } } @@ -7398,10 +7616,11 @@ bool IndexParam::IsInitialized() const { void IndexParam::InternalSwap(IndexParam* other) { using std::swap; _internal_metadata_.Swap(&other->_internal_metadata_); + CastToBase(&extra_params_)->InternalSwap(CastToBase(&other->extra_params_)); table_name_.Swap(&other->table_name_, &::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), GetArenaNoVirtual()); swap(status_, other->status_); - swap(index_, other->index_); + swap(index_type_, other->index_type_); } ::PROTOBUF_NAMESPACE_ID::Metadata IndexParam::GetMetadata() const { @@ -10158,6 +10377,9 @@ void GetVectorIDsParam::InternalSwap(GetVectorIDsParam* other) { } // namespace grpc } // namespace milvus PROTOBUF_NAMESPACE_OPEN +template<> PROTOBUF_NOINLINE ::milvus::grpc::KeyValuePair* Arena::CreateMaybeMessage< ::milvus::grpc::KeyValuePair >(Arena* arena) { + return Arena::CreateInternal< ::milvus::grpc::KeyValuePair >(arena); +} template<> PROTOBUF_NOINLINE ::milvus::grpc::TableName* Arena::CreateMaybeMessage< ::milvus::grpc::TableName >(Arena* arena) { return Arena::CreateInternal< ::milvus::grpc::TableName >(arena); } @@ -10206,9 +10428,6 @@ template<> PROTOBUF_NOINLINE ::milvus::grpc::TableRowCount* Arena::CreateMaybeMe template<> PROTOBUF_NOINLINE ::milvus::grpc::Command* Arena::CreateMaybeMessage< ::milvus::grpc::Command >(Arena* arena) { return Arena::CreateInternal< ::milvus::grpc::Command >(arena); } -template<> PROTOBUF_NOINLINE ::milvus::grpc::Index* Arena::CreateMaybeMessage< ::milvus::grpc::Index >(Arena* arena) { - return Arena::CreateInternal< ::milvus::grpc::Index >(arena); -} template<> PROTOBUF_NOINLINE ::milvus::grpc::IndexParam* Arena::CreateMaybeMessage< ::milvus::grpc::IndexParam >(Arena* arena) { return Arena::CreateInternal< ::milvus::grpc::IndexParam >(arena); } diff --git a/sdk/grpc-gen/gen-milvus/milvus.pb.h b/sdk/grpc-gen/gen-milvus/milvus.pb.h index cc03c9bd26..4528f278e1 100644 --- a/sdk/grpc-gen/gen-milvus/milvus.pb.h +++ b/sdk/grpc-gen/gen-milvus/milvus.pb.h @@ -72,15 +72,15 @@ extern FlushParamDefaultTypeInternal _FlushParam_default_instance_; class GetVectorIDsParam; class GetVectorIDsParamDefaultTypeInternal; extern GetVectorIDsParamDefaultTypeInternal _GetVectorIDsParam_default_instance_; -class Index; -class IndexDefaultTypeInternal; -extern IndexDefaultTypeInternal _Index_default_instance_; class IndexParam; class IndexParamDefaultTypeInternal; extern IndexParamDefaultTypeInternal _IndexParam_default_instance_; class InsertParam; class InsertParamDefaultTypeInternal; extern InsertParamDefaultTypeInternal _InsertParam_default_instance_; +class KeyValuePair; +class KeyValuePairDefaultTypeInternal; +extern KeyValuePairDefaultTypeInternal _KeyValuePair_default_instance_; class PartitionList; class PartitionListDefaultTypeInternal; extern PartitionListDefaultTypeInternal _PartitionList_default_instance_; @@ -143,9 +143,9 @@ template<> ::milvus::grpc::Command* Arena::CreateMaybeMessage<::milvus::grpc::Co template<> ::milvus::grpc::DeleteByIDParam* Arena::CreateMaybeMessage<::milvus::grpc::DeleteByIDParam>(Arena*); template<> ::milvus::grpc::FlushParam* Arena::CreateMaybeMessage<::milvus::grpc::FlushParam>(Arena*); template<> ::milvus::grpc::GetVectorIDsParam* Arena::CreateMaybeMessage<::milvus::grpc::GetVectorIDsParam>(Arena*); -template<> ::milvus::grpc::Index* Arena::CreateMaybeMessage<::milvus::grpc::Index>(Arena*); template<> ::milvus::grpc::IndexParam* Arena::CreateMaybeMessage<::milvus::grpc::IndexParam>(Arena*); template<> ::milvus::grpc::InsertParam* Arena::CreateMaybeMessage<::milvus::grpc::InsertParam>(Arena*); +template<> ::milvus::grpc::KeyValuePair* Arena::CreateMaybeMessage<::milvus::grpc::KeyValuePair>(Arena*); template<> ::milvus::grpc::PartitionList* Arena::CreateMaybeMessage<::milvus::grpc::PartitionList>(Arena*); template<> ::milvus::grpc::PartitionParam* Arena::CreateMaybeMessage<::milvus::grpc::PartitionParam>(Arena*); template<> ::milvus::grpc::PartitionStat* Arena::CreateMaybeMessage<::milvus::grpc::PartitionStat>(Arena*); @@ -170,6 +170,156 @@ namespace grpc { // =================================================================== +class KeyValuePair : + public ::PROTOBUF_NAMESPACE_ID::Message /* @@protoc_insertion_point(class_definition:milvus.grpc.KeyValuePair) */ { + public: + KeyValuePair(); + virtual ~KeyValuePair(); + + KeyValuePair(const KeyValuePair& from); + KeyValuePair(KeyValuePair&& from) noexcept + : KeyValuePair() { + *this = ::std::move(from); + } + + inline KeyValuePair& operator=(const KeyValuePair& from) { + CopyFrom(from); + return *this; + } + inline KeyValuePair& operator=(KeyValuePair&& from) noexcept { + if (GetArenaNoVirtual() == from.GetArenaNoVirtual()) { + if (this != &from) InternalSwap(&from); + } else { + CopyFrom(from); + } + return *this; + } + + static const ::PROTOBUF_NAMESPACE_ID::Descriptor* descriptor() { + return GetDescriptor(); + } + static const ::PROTOBUF_NAMESPACE_ID::Descriptor* GetDescriptor() { + return GetMetadataStatic().descriptor; + } + static const ::PROTOBUF_NAMESPACE_ID::Reflection* GetReflection() { + return GetMetadataStatic().reflection; + } + static const KeyValuePair& default_instance(); + + static void InitAsDefaultInstance(); // FOR INTERNAL USE ONLY + static inline const KeyValuePair* internal_default_instance() { + return reinterpret_cast( + &_KeyValuePair_default_instance_); + } + static constexpr int kIndexInFileMessages = + 0; + + friend void swap(KeyValuePair& a, KeyValuePair& b) { + a.Swap(&b); + } + inline void Swap(KeyValuePair* other) { + if (other == this) return; + InternalSwap(other); + } + + // implements Message ---------------------------------------------- + + inline KeyValuePair* New() const final { + return CreateMaybeMessage(nullptr); + } + + KeyValuePair* New(::PROTOBUF_NAMESPACE_ID::Arena* arena) const final { + return CreateMaybeMessage(arena); + } + void CopyFrom(const ::PROTOBUF_NAMESPACE_ID::Message& from) final; + void MergeFrom(const ::PROTOBUF_NAMESPACE_ID::Message& from) final; + void CopyFrom(const KeyValuePair& from); + void MergeFrom(const KeyValuePair& from); + PROTOBUF_ATTRIBUTE_REINITIALIZES void Clear() final; + bool IsInitialized() const final; + + size_t ByteSizeLong() const final; + #if GOOGLE_PROTOBUF_ENABLE_EXPERIMENTAL_PARSER + const char* _InternalParse(const char* ptr, ::PROTOBUF_NAMESPACE_ID::internal::ParseContext* ctx) final; + #else + bool MergePartialFromCodedStream( + ::PROTOBUF_NAMESPACE_ID::io::CodedInputStream* input) final; + #endif // GOOGLE_PROTOBUF_ENABLE_EXPERIMENTAL_PARSER + void SerializeWithCachedSizes( + ::PROTOBUF_NAMESPACE_ID::io::CodedOutputStream* output) const final; + ::PROTOBUF_NAMESPACE_ID::uint8* InternalSerializeWithCachedSizesToArray( + ::PROTOBUF_NAMESPACE_ID::uint8* target) const final; + int GetCachedSize() const final { return _cached_size_.Get(); } + + private: + inline void SharedCtor(); + inline void SharedDtor(); + void SetCachedSize(int size) const final; + void InternalSwap(KeyValuePair* other); + friend class ::PROTOBUF_NAMESPACE_ID::internal::AnyMetadata; + static ::PROTOBUF_NAMESPACE_ID::StringPiece FullMessageName() { + return "milvus.grpc.KeyValuePair"; + } + private: + inline ::PROTOBUF_NAMESPACE_ID::Arena* GetArenaNoVirtual() const { + return nullptr; + } + inline void* MaybeArenaPtr() const { + return nullptr; + } + public: + + ::PROTOBUF_NAMESPACE_ID::Metadata GetMetadata() const final; + private: + static ::PROTOBUF_NAMESPACE_ID::Metadata GetMetadataStatic() { + ::PROTOBUF_NAMESPACE_ID::internal::AssignDescriptors(&::descriptor_table_milvus_2eproto); + return ::descriptor_table_milvus_2eproto.file_level_metadata[kIndexInFileMessages]; + } + + public: + + // nested types ---------------------------------------------------- + + // accessors ------------------------------------------------------- + + enum : int { + kKeyFieldNumber = 1, + kValueFieldNumber = 2, + }; + // string key = 1; + void clear_key(); + const std::string& key() const; + void set_key(const std::string& value); + void set_key(std::string&& value); + void set_key(const char* value); + void set_key(const char* value, size_t size); + std::string* mutable_key(); + std::string* release_key(); + void set_allocated_key(std::string* key); + + // string value = 2; + void clear_value(); + const std::string& value() const; + void set_value(const std::string& value); + void set_value(std::string&& value); + void set_value(const char* value); + void set_value(const char* value, size_t size); + std::string* mutable_value(); + std::string* release_value(); + void set_allocated_value(std::string* value); + + // @@protoc_insertion_point(class_scope:milvus.grpc.KeyValuePair) + private: + class _Internal; + + ::PROTOBUF_NAMESPACE_ID::internal::InternalMetadataWithArena _internal_metadata_; + ::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr key_; + ::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr value_; + mutable ::PROTOBUF_NAMESPACE_ID::internal::CachedSize _cached_size_; + friend struct ::TableStruct_milvus_2eproto; +}; +// ------------------------------------------------------------------- + class TableName : public ::PROTOBUF_NAMESPACE_ID::Message /* @@protoc_insertion_point(class_definition:milvus.grpc.TableName) */ { public: @@ -212,7 +362,7 @@ class TableName : &_TableName_default_instance_); } static constexpr int kIndexInFileMessages = - 0; + 1; friend void swap(TableName& a, TableName& b) { a.Swap(&b); @@ -349,7 +499,7 @@ class TableNameList : &_TableNameList_default_instance_); } static constexpr int kIndexInFileMessages = - 1; + 2; friend void swap(TableNameList& a, TableNameList& b) { a.Swap(&b); @@ -502,7 +652,7 @@ class TableSchema : &_TableSchema_default_instance_); } static constexpr int kIndexInFileMessages = - 2; + 3; friend void swap(TableSchema& a, TableSchema& b) { a.Swap(&b); @@ -573,12 +723,24 @@ class TableSchema : // accessors ------------------------------------------------------- enum : int { + kExtraParamsFieldNumber = 6, kTableNameFieldNumber = 2, kStatusFieldNumber = 1, kDimensionFieldNumber = 3, kIndexFileSizeFieldNumber = 4, kMetricTypeFieldNumber = 5, }; + // repeated .milvus.grpc.KeyValuePair extra_params = 6; + int extra_params_size() const; + void clear_extra_params(); + ::milvus::grpc::KeyValuePair* mutable_extra_params(int index); + ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::milvus::grpc::KeyValuePair >* + mutable_extra_params(); + const ::milvus::grpc::KeyValuePair& extra_params(int index) const; + ::milvus::grpc::KeyValuePair* add_extra_params(); + const ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::milvus::grpc::KeyValuePair >& + extra_params() const; + // string table_name = 2; void clear_table_name(); const std::string& table_name() const; @@ -618,6 +780,7 @@ class TableSchema : class _Internal; ::PROTOBUF_NAMESPACE_ID::internal::InternalMetadataWithArena _internal_metadata_; + ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::milvus::grpc::KeyValuePair > extra_params_; ::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr table_name_; ::milvus::grpc::Status* status_; ::PROTOBUF_NAMESPACE_ID::int64 dimension_; @@ -670,7 +833,7 @@ class PartitionParam : &_PartitionParam_default_instance_); } static constexpr int kIndexInFileMessages = - 3; + 4; friend void swap(PartitionParam& a, PartitionParam& b) { a.Swap(&b); @@ -820,7 +983,7 @@ class PartitionList : &_PartitionList_default_instance_); } static constexpr int kIndexInFileMessages = - 4; + 5; friend void swap(PartitionList& a, PartitionList& b) { a.Swap(&b); @@ -973,7 +1136,7 @@ class RowRecord : &_RowRecord_default_instance_); } static constexpr int kIndexInFileMessages = - 5; + 6; friend void swap(RowRecord& a, RowRecord& b) { a.Swap(&b); @@ -1124,7 +1287,7 @@ class InsertParam : &_InsertParam_default_instance_); } static constexpr int kIndexInFileMessages = - 6; + 7; friend void swap(InsertParam& a, InsertParam& b) { a.Swap(&b); @@ -1197,6 +1360,7 @@ class InsertParam : enum : int { kRowRecordArrayFieldNumber = 2, kRowIdArrayFieldNumber = 3, + kExtraParamsFieldNumber = 5, kTableNameFieldNumber = 1, kPartitionTagFieldNumber = 4, }; @@ -1222,6 +1386,17 @@ class InsertParam : ::PROTOBUF_NAMESPACE_ID::RepeatedField< ::PROTOBUF_NAMESPACE_ID::int64 >* mutable_row_id_array(); + // repeated .milvus.grpc.KeyValuePair extra_params = 5; + int extra_params_size() const; + void clear_extra_params(); + ::milvus::grpc::KeyValuePair* mutable_extra_params(int index); + ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::milvus::grpc::KeyValuePair >* + mutable_extra_params(); + const ::milvus::grpc::KeyValuePair& extra_params(int index) const; + ::milvus::grpc::KeyValuePair* add_extra_params(); + const ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::milvus::grpc::KeyValuePair >& + extra_params() const; + // string table_name = 1; void clear_table_name(); const std::string& table_name() const; @@ -1252,6 +1427,7 @@ class InsertParam : ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::milvus::grpc::RowRecord > row_record_array_; ::PROTOBUF_NAMESPACE_ID::RepeatedField< ::PROTOBUF_NAMESPACE_ID::int64 > row_id_array_; mutable std::atomic _row_id_array_cached_byte_size_; + ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::milvus::grpc::KeyValuePair > extra_params_; ::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr table_name_; ::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr partition_tag_; mutable ::PROTOBUF_NAMESPACE_ID::internal::CachedSize _cached_size_; @@ -1301,7 +1477,7 @@ class VectorIds : &_VectorIds_default_instance_); } static constexpr int kIndexInFileMessages = - 7; + 8; friend void swap(VectorIds& a, VectorIds& b) { a.Swap(&b); @@ -1449,7 +1625,7 @@ class SearchParam : &_SearchParam_default_instance_); } static constexpr int kIndexInFileMessages = - 8; + 9; friend void swap(SearchParam& a, SearchParam& b) { a.Swap(&b); @@ -1520,24 +1696,13 @@ class SearchParam : // accessors ------------------------------------------------------- enum : int { - kQueryRecordArrayFieldNumber = 2, - kPartitionTagArrayFieldNumber = 5, + kPartitionTagArrayFieldNumber = 2, + kQueryRecordArrayFieldNumber = 3, + kExtraParamsFieldNumber = 5, kTableNameFieldNumber = 1, - kTopkFieldNumber = 3, - kNprobeFieldNumber = 4, + kTopkFieldNumber = 4, }; - // repeated .milvus.grpc.RowRecord query_record_array = 2; - int query_record_array_size() const; - void clear_query_record_array(); - ::milvus::grpc::RowRecord* mutable_query_record_array(int index); - ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::milvus::grpc::RowRecord >* - mutable_query_record_array(); - const ::milvus::grpc::RowRecord& query_record_array(int index) const; - ::milvus::grpc::RowRecord* add_query_record_array(); - const ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::milvus::grpc::RowRecord >& - query_record_array() const; - - // repeated string partition_tag_array = 5; + // repeated string partition_tag_array = 2; int partition_tag_array_size() const; void clear_partition_tag_array(); const std::string& partition_tag_array(int index) const; @@ -1554,6 +1719,28 @@ class SearchParam : const ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField& partition_tag_array() const; ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField* mutable_partition_tag_array(); + // repeated .milvus.grpc.RowRecord query_record_array = 3; + int query_record_array_size() const; + void clear_query_record_array(); + ::milvus::grpc::RowRecord* mutable_query_record_array(int index); + ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::milvus::grpc::RowRecord >* + mutable_query_record_array(); + const ::milvus::grpc::RowRecord& query_record_array(int index) const; + ::milvus::grpc::RowRecord* add_query_record_array(); + const ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::milvus::grpc::RowRecord >& + query_record_array() const; + + // repeated .milvus.grpc.KeyValuePair extra_params = 5; + int extra_params_size() const; + void clear_extra_params(); + ::milvus::grpc::KeyValuePair* mutable_extra_params(int index); + ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::milvus::grpc::KeyValuePair >* + mutable_extra_params(); + const ::milvus::grpc::KeyValuePair& extra_params(int index) const; + ::milvus::grpc::KeyValuePair* add_extra_params(); + const ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::milvus::grpc::KeyValuePair >& + extra_params() const; + // string table_name = 1; void clear_table_name(); const std::string& table_name() const; @@ -1565,26 +1752,21 @@ class SearchParam : std::string* release_table_name(); void set_allocated_table_name(std::string* table_name); - // int64 topk = 3; + // int64 topk = 4; void clear_topk(); ::PROTOBUF_NAMESPACE_ID::int64 topk() const; void set_topk(::PROTOBUF_NAMESPACE_ID::int64 value); - // int64 nprobe = 4; - void clear_nprobe(); - ::PROTOBUF_NAMESPACE_ID::int64 nprobe() const; - void set_nprobe(::PROTOBUF_NAMESPACE_ID::int64 value); - // @@protoc_insertion_point(class_scope:milvus.grpc.SearchParam) private: class _Internal; ::PROTOBUF_NAMESPACE_ID::internal::InternalMetadataWithArena _internal_metadata_; - ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::milvus::grpc::RowRecord > query_record_array_; ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField partition_tag_array_; + ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::milvus::grpc::RowRecord > query_record_array_; + ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::milvus::grpc::KeyValuePair > extra_params_; ::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr table_name_; ::PROTOBUF_NAMESPACE_ID::int64 topk_; - ::PROTOBUF_NAMESPACE_ID::int64 nprobe_; mutable ::PROTOBUF_NAMESPACE_ID::internal::CachedSize _cached_size_; friend struct ::TableStruct_milvus_2eproto; }; @@ -1632,7 +1814,7 @@ class SearchInFilesParam : &_SearchInFilesParam_default_instance_); } static constexpr int kIndexInFileMessages = - 9; + 10; friend void swap(SearchInFilesParam& a, SearchInFilesParam& b) { a.Swap(&b); @@ -1785,7 +1967,7 @@ class SearchByIDParam : &_SearchByIDParam_default_instance_); } static constexpr int kIndexInFileMessages = - 10; + 11; friend void swap(SearchByIDParam& a, SearchByIDParam& b) { a.Swap(&b); @@ -1856,13 +2038,13 @@ class SearchByIDParam : // accessors ------------------------------------------------------- enum : int { - kPartitionTagArrayFieldNumber = 5, + kPartitionTagArrayFieldNumber = 2, + kExtraParamsFieldNumber = 5, kTableNameFieldNumber = 1, - kIdFieldNumber = 2, - kTopkFieldNumber = 3, - kNprobeFieldNumber = 4, + kIdFieldNumber = 3, + kTopkFieldNumber = 4, }; - // repeated string partition_tag_array = 5; + // repeated string partition_tag_array = 2; int partition_tag_array_size() const; void clear_partition_tag_array(); const std::string& partition_tag_array(int index) const; @@ -1879,6 +2061,17 @@ class SearchByIDParam : const ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField& partition_tag_array() const; ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField* mutable_partition_tag_array(); + // repeated .milvus.grpc.KeyValuePair extra_params = 5; + int extra_params_size() const; + void clear_extra_params(); + ::milvus::grpc::KeyValuePair* mutable_extra_params(int index); + ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::milvus::grpc::KeyValuePair >* + mutable_extra_params(); + const ::milvus::grpc::KeyValuePair& extra_params(int index) const; + ::milvus::grpc::KeyValuePair* add_extra_params(); + const ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::milvus::grpc::KeyValuePair >& + extra_params() const; + // string table_name = 1; void clear_table_name(); const std::string& table_name() const; @@ -1890,31 +2083,26 @@ class SearchByIDParam : std::string* release_table_name(); void set_allocated_table_name(std::string* table_name); - // int64 id = 2; + // int64 id = 3; void clear_id(); ::PROTOBUF_NAMESPACE_ID::int64 id() const; void set_id(::PROTOBUF_NAMESPACE_ID::int64 value); - // int64 topk = 3; + // int64 topk = 4; void clear_topk(); ::PROTOBUF_NAMESPACE_ID::int64 topk() const; void set_topk(::PROTOBUF_NAMESPACE_ID::int64 value); - // int64 nprobe = 4; - void clear_nprobe(); - ::PROTOBUF_NAMESPACE_ID::int64 nprobe() const; - void set_nprobe(::PROTOBUF_NAMESPACE_ID::int64 value); - // @@protoc_insertion_point(class_scope:milvus.grpc.SearchByIDParam) private: class _Internal; ::PROTOBUF_NAMESPACE_ID::internal::InternalMetadataWithArena _internal_metadata_; ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField partition_tag_array_; + ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::milvus::grpc::KeyValuePair > extra_params_; ::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr table_name_; ::PROTOBUF_NAMESPACE_ID::int64 id_; ::PROTOBUF_NAMESPACE_ID::int64 topk_; - ::PROTOBUF_NAMESPACE_ID::int64 nprobe_; mutable ::PROTOBUF_NAMESPACE_ID::internal::CachedSize _cached_size_; friend struct ::TableStruct_milvus_2eproto; }; @@ -1962,7 +2150,7 @@ class TopKQueryResult : &_TopKQueryResult_default_instance_); } static constexpr int kIndexInFileMessages = - 11; + 12; friend void swap(TopKQueryResult& a, TopKQueryResult& b) { a.Swap(&b); @@ -2131,7 +2319,7 @@ class StringReply : &_StringReply_default_instance_); } static constexpr int kIndexInFileMessages = - 12; + 13; friend void swap(StringReply& a, StringReply& b) { a.Swap(&b); @@ -2278,7 +2466,7 @@ class BoolReply : &_BoolReply_default_instance_); } static constexpr int kIndexInFileMessages = - 13; + 14; friend void swap(BoolReply& a, BoolReply& b) { a.Swap(&b); @@ -2419,7 +2607,7 @@ class TableRowCount : &_TableRowCount_default_instance_); } static constexpr int kIndexInFileMessages = - 14; + 15; friend void swap(TableRowCount& a, TableRowCount& b) { a.Swap(&b); @@ -2560,7 +2748,7 @@ class Command : &_Command_default_instance_); } static constexpr int kIndexInFileMessages = - 15; + 16; friend void swap(Command& a, Command& b) { a.Swap(&b); @@ -2655,144 +2843,6 @@ class Command : }; // ------------------------------------------------------------------- -class Index : - public ::PROTOBUF_NAMESPACE_ID::Message /* @@protoc_insertion_point(class_definition:milvus.grpc.Index) */ { - public: - Index(); - virtual ~Index(); - - Index(const Index& from); - Index(Index&& from) noexcept - : Index() { - *this = ::std::move(from); - } - - inline Index& operator=(const Index& from) { - CopyFrom(from); - return *this; - } - inline Index& operator=(Index&& from) noexcept { - if (GetArenaNoVirtual() == from.GetArenaNoVirtual()) { - if (this != &from) InternalSwap(&from); - } else { - CopyFrom(from); - } - return *this; - } - - static const ::PROTOBUF_NAMESPACE_ID::Descriptor* descriptor() { - return GetDescriptor(); - } - static const ::PROTOBUF_NAMESPACE_ID::Descriptor* GetDescriptor() { - return GetMetadataStatic().descriptor; - } - static const ::PROTOBUF_NAMESPACE_ID::Reflection* GetReflection() { - return GetMetadataStatic().reflection; - } - static const Index& default_instance(); - - static void InitAsDefaultInstance(); // FOR INTERNAL USE ONLY - static inline const Index* internal_default_instance() { - return reinterpret_cast( - &_Index_default_instance_); - } - static constexpr int kIndexInFileMessages = - 16; - - friend void swap(Index& a, Index& b) { - a.Swap(&b); - } - inline void Swap(Index* other) { - if (other == this) return; - InternalSwap(other); - } - - // implements Message ---------------------------------------------- - - inline Index* New() const final { - return CreateMaybeMessage(nullptr); - } - - Index* New(::PROTOBUF_NAMESPACE_ID::Arena* arena) const final { - return CreateMaybeMessage(arena); - } - void CopyFrom(const ::PROTOBUF_NAMESPACE_ID::Message& from) final; - void MergeFrom(const ::PROTOBUF_NAMESPACE_ID::Message& from) final; - void CopyFrom(const Index& from); - void MergeFrom(const Index& from); - PROTOBUF_ATTRIBUTE_REINITIALIZES void Clear() final; - bool IsInitialized() const final; - - size_t ByteSizeLong() const final; - #if GOOGLE_PROTOBUF_ENABLE_EXPERIMENTAL_PARSER - const char* _InternalParse(const char* ptr, ::PROTOBUF_NAMESPACE_ID::internal::ParseContext* ctx) final; - #else - bool MergePartialFromCodedStream( - ::PROTOBUF_NAMESPACE_ID::io::CodedInputStream* input) final; - #endif // GOOGLE_PROTOBUF_ENABLE_EXPERIMENTAL_PARSER - void SerializeWithCachedSizes( - ::PROTOBUF_NAMESPACE_ID::io::CodedOutputStream* output) const final; - ::PROTOBUF_NAMESPACE_ID::uint8* InternalSerializeWithCachedSizesToArray( - ::PROTOBUF_NAMESPACE_ID::uint8* target) const final; - int GetCachedSize() const final { return _cached_size_.Get(); } - - private: - inline void SharedCtor(); - inline void SharedDtor(); - void SetCachedSize(int size) const final; - void InternalSwap(Index* other); - friend class ::PROTOBUF_NAMESPACE_ID::internal::AnyMetadata; - static ::PROTOBUF_NAMESPACE_ID::StringPiece FullMessageName() { - return "milvus.grpc.Index"; - } - private: - inline ::PROTOBUF_NAMESPACE_ID::Arena* GetArenaNoVirtual() const { - return nullptr; - } - inline void* MaybeArenaPtr() const { - return nullptr; - } - public: - - ::PROTOBUF_NAMESPACE_ID::Metadata GetMetadata() const final; - private: - static ::PROTOBUF_NAMESPACE_ID::Metadata GetMetadataStatic() { - ::PROTOBUF_NAMESPACE_ID::internal::AssignDescriptors(&::descriptor_table_milvus_2eproto); - return ::descriptor_table_milvus_2eproto.file_level_metadata[kIndexInFileMessages]; - } - - public: - - // nested types ---------------------------------------------------- - - // accessors ------------------------------------------------------- - - enum : int { - kIndexTypeFieldNumber = 1, - kNlistFieldNumber = 2, - }; - // int32 index_type = 1; - void clear_index_type(); - ::PROTOBUF_NAMESPACE_ID::int32 index_type() const; - void set_index_type(::PROTOBUF_NAMESPACE_ID::int32 value); - - // int32 nlist = 2; - void clear_nlist(); - ::PROTOBUF_NAMESPACE_ID::int32 nlist() const; - void set_nlist(::PROTOBUF_NAMESPACE_ID::int32 value); - - // @@protoc_insertion_point(class_scope:milvus.grpc.Index) - private: - class _Internal; - - ::PROTOBUF_NAMESPACE_ID::internal::InternalMetadataWithArena _internal_metadata_; - ::PROTOBUF_NAMESPACE_ID::int32 index_type_; - ::PROTOBUF_NAMESPACE_ID::int32 nlist_; - mutable ::PROTOBUF_NAMESPACE_ID::internal::CachedSize _cached_size_; - friend struct ::TableStruct_milvus_2eproto; -}; -// ------------------------------------------------------------------- - class IndexParam : public ::PROTOBUF_NAMESPACE_ID::Message /* @@protoc_insertion_point(class_definition:milvus.grpc.IndexParam) */ { public: @@ -2906,10 +2956,22 @@ class IndexParam : // accessors ------------------------------------------------------- enum : int { + kExtraParamsFieldNumber = 4, kTableNameFieldNumber = 2, kStatusFieldNumber = 1, - kIndexFieldNumber = 3, + kIndexTypeFieldNumber = 3, }; + // repeated .milvus.grpc.KeyValuePair extra_params = 4; + int extra_params_size() const; + void clear_extra_params(); + ::milvus::grpc::KeyValuePair* mutable_extra_params(int index); + ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::milvus::grpc::KeyValuePair >* + mutable_extra_params(); + const ::milvus::grpc::KeyValuePair& extra_params(int index) const; + ::milvus::grpc::KeyValuePair* add_extra_params(); + const ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::milvus::grpc::KeyValuePair >& + extra_params() const; + // string table_name = 2; void clear_table_name(); const std::string& table_name() const; @@ -2929,22 +2991,20 @@ class IndexParam : ::milvus::grpc::Status* mutable_status(); void set_allocated_status(::milvus::grpc::Status* status); - // .milvus.grpc.Index index = 3; - bool has_index() const; - void clear_index(); - const ::milvus::grpc::Index& index() const; - ::milvus::grpc::Index* release_index(); - ::milvus::grpc::Index* mutable_index(); - void set_allocated_index(::milvus::grpc::Index* index); + // int32 index_type = 3; + void clear_index_type(); + ::PROTOBUF_NAMESPACE_ID::int32 index_type() const; + void set_index_type(::PROTOBUF_NAMESPACE_ID::int32 value); // @@protoc_insertion_point(class_scope:milvus.grpc.IndexParam) private: class _Internal; ::PROTOBUF_NAMESPACE_ID::internal::InternalMetadataWithArena _internal_metadata_; + ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::milvus::grpc::KeyValuePair > extra_params_; ::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr table_name_; ::milvus::grpc::Status* status_; - ::milvus::grpc::Index* index_; + ::PROTOBUF_NAMESPACE_ID::int32 index_type_; mutable ::PROTOBUF_NAMESPACE_ID::internal::CachedSize _cached_size_; friend struct ::TableStruct_milvus_2eproto; }; @@ -4164,6 +4224,112 @@ class GetVectorIDsParam : #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wstrict-aliasing" #endif // __GNUC__ +// KeyValuePair + +// string key = 1; +inline void KeyValuePair::clear_key() { + key_.ClearToEmptyNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); +} +inline const std::string& KeyValuePair::key() const { + // @@protoc_insertion_point(field_get:milvus.grpc.KeyValuePair.key) + return key_.GetNoArena(); +} +inline void KeyValuePair::set_key(const std::string& value) { + + key_.SetNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), value); + // @@protoc_insertion_point(field_set:milvus.grpc.KeyValuePair.key) +} +inline void KeyValuePair::set_key(std::string&& value) { + + key_.SetNoArena( + &::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), ::std::move(value)); + // @@protoc_insertion_point(field_set_rvalue:milvus.grpc.KeyValuePair.key) +} +inline void KeyValuePair::set_key(const char* value) { + GOOGLE_DCHECK(value != nullptr); + + key_.SetNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), ::std::string(value)); + // @@protoc_insertion_point(field_set_char:milvus.grpc.KeyValuePair.key) +} +inline void KeyValuePair::set_key(const char* value, size_t size) { + + key_.SetNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), + ::std::string(reinterpret_cast(value), size)); + // @@protoc_insertion_point(field_set_pointer:milvus.grpc.KeyValuePair.key) +} +inline std::string* KeyValuePair::mutable_key() { + + // @@protoc_insertion_point(field_mutable:milvus.grpc.KeyValuePair.key) + return key_.MutableNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); +} +inline std::string* KeyValuePair::release_key() { + // @@protoc_insertion_point(field_release:milvus.grpc.KeyValuePair.key) + + return key_.ReleaseNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); +} +inline void KeyValuePair::set_allocated_key(std::string* key) { + if (key != nullptr) { + + } else { + + } + key_.SetAllocatedNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), key); + // @@protoc_insertion_point(field_set_allocated:milvus.grpc.KeyValuePair.key) +} + +// string value = 2; +inline void KeyValuePair::clear_value() { + value_.ClearToEmptyNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); +} +inline const std::string& KeyValuePair::value() const { + // @@protoc_insertion_point(field_get:milvus.grpc.KeyValuePair.value) + return value_.GetNoArena(); +} +inline void KeyValuePair::set_value(const std::string& value) { + + value_.SetNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), value); + // @@protoc_insertion_point(field_set:milvus.grpc.KeyValuePair.value) +} +inline void KeyValuePair::set_value(std::string&& value) { + + value_.SetNoArena( + &::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), ::std::move(value)); + // @@protoc_insertion_point(field_set_rvalue:milvus.grpc.KeyValuePair.value) +} +inline void KeyValuePair::set_value(const char* value) { + GOOGLE_DCHECK(value != nullptr); + + value_.SetNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), ::std::string(value)); + // @@protoc_insertion_point(field_set_char:milvus.grpc.KeyValuePair.value) +} +inline void KeyValuePair::set_value(const char* value, size_t size) { + + value_.SetNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), + ::std::string(reinterpret_cast(value), size)); + // @@protoc_insertion_point(field_set_pointer:milvus.grpc.KeyValuePair.value) +} +inline std::string* KeyValuePair::mutable_value() { + + // @@protoc_insertion_point(field_mutable:milvus.grpc.KeyValuePair.value) + return value_.MutableNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); +} +inline std::string* KeyValuePair::release_value() { + // @@protoc_insertion_point(field_release:milvus.grpc.KeyValuePair.value) + + return value_.ReleaseNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); +} +inline void KeyValuePair::set_allocated_value(std::string* value) { + if (value != nullptr) { + + } else { + + } + value_.SetAllocatedNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), value); + // @@protoc_insertion_point(field_set_allocated:milvus.grpc.KeyValuePair.value) +} + +// ------------------------------------------------------------------- + // TableName // string table_name = 1; @@ -4473,6 +4639,36 @@ inline void TableSchema::set_metric_type(::PROTOBUF_NAMESPACE_ID::int32 value) { // @@protoc_insertion_point(field_set:milvus.grpc.TableSchema.metric_type) } +// repeated .milvus.grpc.KeyValuePair extra_params = 6; +inline int TableSchema::extra_params_size() const { + return extra_params_.size(); +} +inline void TableSchema::clear_extra_params() { + extra_params_.Clear(); +} +inline ::milvus::grpc::KeyValuePair* TableSchema::mutable_extra_params(int index) { + // @@protoc_insertion_point(field_mutable:milvus.grpc.TableSchema.extra_params) + return extra_params_.Mutable(index); +} +inline ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::milvus::grpc::KeyValuePair >* +TableSchema::mutable_extra_params() { + // @@protoc_insertion_point(field_mutable_list:milvus.grpc.TableSchema.extra_params) + return &extra_params_; +} +inline const ::milvus::grpc::KeyValuePair& TableSchema::extra_params(int index) const { + // @@protoc_insertion_point(field_get:milvus.grpc.TableSchema.extra_params) + return extra_params_.Get(index); +} +inline ::milvus::grpc::KeyValuePair* TableSchema::add_extra_params() { + // @@protoc_insertion_point(field_add:milvus.grpc.TableSchema.extra_params) + return extra_params_.Add(); +} +inline const ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::milvus::grpc::KeyValuePair >& +TableSchema::extra_params() const { + // @@protoc_insertion_point(field_list:milvus.grpc.TableSchema.extra_params) + return extra_params_; +} + // ------------------------------------------------------------------- // PartitionParam @@ -4944,6 +5140,36 @@ inline void InsertParam::set_allocated_partition_tag(std::string* partition_tag) // @@protoc_insertion_point(field_set_allocated:milvus.grpc.InsertParam.partition_tag) } +// repeated .milvus.grpc.KeyValuePair extra_params = 5; +inline int InsertParam::extra_params_size() const { + return extra_params_.size(); +} +inline void InsertParam::clear_extra_params() { + extra_params_.Clear(); +} +inline ::milvus::grpc::KeyValuePair* InsertParam::mutable_extra_params(int index) { + // @@protoc_insertion_point(field_mutable:milvus.grpc.InsertParam.extra_params) + return extra_params_.Mutable(index); +} +inline ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::milvus::grpc::KeyValuePair >* +InsertParam::mutable_extra_params() { + // @@protoc_insertion_point(field_mutable_list:milvus.grpc.InsertParam.extra_params) + return &extra_params_; +} +inline const ::milvus::grpc::KeyValuePair& InsertParam::extra_params(int index) const { + // @@protoc_insertion_point(field_get:milvus.grpc.InsertParam.extra_params) + return extra_params_.Get(index); +} +inline ::milvus::grpc::KeyValuePair* InsertParam::add_extra_params() { + // @@protoc_insertion_point(field_add:milvus.grpc.InsertParam.extra_params) + return extra_params_.Add(); +} +inline const ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::milvus::grpc::KeyValuePair >& +InsertParam::extra_params() const { + // @@protoc_insertion_point(field_list:milvus.grpc.InsertParam.extra_params) + return extra_params_; +} + // ------------------------------------------------------------------- // VectorIds @@ -5078,65 +5304,7 @@ inline void SearchParam::set_allocated_table_name(std::string* table_name) { // @@protoc_insertion_point(field_set_allocated:milvus.grpc.SearchParam.table_name) } -// repeated .milvus.grpc.RowRecord query_record_array = 2; -inline int SearchParam::query_record_array_size() const { - return query_record_array_.size(); -} -inline void SearchParam::clear_query_record_array() { - query_record_array_.Clear(); -} -inline ::milvus::grpc::RowRecord* SearchParam::mutable_query_record_array(int index) { - // @@protoc_insertion_point(field_mutable:milvus.grpc.SearchParam.query_record_array) - return query_record_array_.Mutable(index); -} -inline ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::milvus::grpc::RowRecord >* -SearchParam::mutable_query_record_array() { - // @@protoc_insertion_point(field_mutable_list:milvus.grpc.SearchParam.query_record_array) - return &query_record_array_; -} -inline const ::milvus::grpc::RowRecord& SearchParam::query_record_array(int index) const { - // @@protoc_insertion_point(field_get:milvus.grpc.SearchParam.query_record_array) - return query_record_array_.Get(index); -} -inline ::milvus::grpc::RowRecord* SearchParam::add_query_record_array() { - // @@protoc_insertion_point(field_add:milvus.grpc.SearchParam.query_record_array) - return query_record_array_.Add(); -} -inline const ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::milvus::grpc::RowRecord >& -SearchParam::query_record_array() const { - // @@protoc_insertion_point(field_list:milvus.grpc.SearchParam.query_record_array) - return query_record_array_; -} - -// int64 topk = 3; -inline void SearchParam::clear_topk() { - topk_ = PROTOBUF_LONGLONG(0); -} -inline ::PROTOBUF_NAMESPACE_ID::int64 SearchParam::topk() const { - // @@protoc_insertion_point(field_get:milvus.grpc.SearchParam.topk) - return topk_; -} -inline void SearchParam::set_topk(::PROTOBUF_NAMESPACE_ID::int64 value) { - - topk_ = value; - // @@protoc_insertion_point(field_set:milvus.grpc.SearchParam.topk) -} - -// int64 nprobe = 4; -inline void SearchParam::clear_nprobe() { - nprobe_ = PROTOBUF_LONGLONG(0); -} -inline ::PROTOBUF_NAMESPACE_ID::int64 SearchParam::nprobe() const { - // @@protoc_insertion_point(field_get:milvus.grpc.SearchParam.nprobe) - return nprobe_; -} -inline void SearchParam::set_nprobe(::PROTOBUF_NAMESPACE_ID::int64 value) { - - nprobe_ = value; - // @@protoc_insertion_point(field_set:milvus.grpc.SearchParam.nprobe) -} - -// repeated string partition_tag_array = 5; +// repeated string partition_tag_array = 2; inline int SearchParam::partition_tag_array_size() const { return partition_tag_array_.size(); } @@ -5201,6 +5369,80 @@ SearchParam::mutable_partition_tag_array() { return &partition_tag_array_; } +// repeated .milvus.grpc.RowRecord query_record_array = 3; +inline int SearchParam::query_record_array_size() const { + return query_record_array_.size(); +} +inline void SearchParam::clear_query_record_array() { + query_record_array_.Clear(); +} +inline ::milvus::grpc::RowRecord* SearchParam::mutable_query_record_array(int index) { + // @@protoc_insertion_point(field_mutable:milvus.grpc.SearchParam.query_record_array) + return query_record_array_.Mutable(index); +} +inline ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::milvus::grpc::RowRecord >* +SearchParam::mutable_query_record_array() { + // @@protoc_insertion_point(field_mutable_list:milvus.grpc.SearchParam.query_record_array) + return &query_record_array_; +} +inline const ::milvus::grpc::RowRecord& SearchParam::query_record_array(int index) const { + // @@protoc_insertion_point(field_get:milvus.grpc.SearchParam.query_record_array) + return query_record_array_.Get(index); +} +inline ::milvus::grpc::RowRecord* SearchParam::add_query_record_array() { + // @@protoc_insertion_point(field_add:milvus.grpc.SearchParam.query_record_array) + return query_record_array_.Add(); +} +inline const ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::milvus::grpc::RowRecord >& +SearchParam::query_record_array() const { + // @@protoc_insertion_point(field_list:milvus.grpc.SearchParam.query_record_array) + return query_record_array_; +} + +// int64 topk = 4; +inline void SearchParam::clear_topk() { + topk_ = PROTOBUF_LONGLONG(0); +} +inline ::PROTOBUF_NAMESPACE_ID::int64 SearchParam::topk() const { + // @@protoc_insertion_point(field_get:milvus.grpc.SearchParam.topk) + return topk_; +} +inline void SearchParam::set_topk(::PROTOBUF_NAMESPACE_ID::int64 value) { + + topk_ = value; + // @@protoc_insertion_point(field_set:milvus.grpc.SearchParam.topk) +} + +// repeated .milvus.grpc.KeyValuePair extra_params = 5; +inline int SearchParam::extra_params_size() const { + return extra_params_.size(); +} +inline void SearchParam::clear_extra_params() { + extra_params_.Clear(); +} +inline ::milvus::grpc::KeyValuePair* SearchParam::mutable_extra_params(int index) { + // @@protoc_insertion_point(field_mutable:milvus.grpc.SearchParam.extra_params) + return extra_params_.Mutable(index); +} +inline ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::milvus::grpc::KeyValuePair >* +SearchParam::mutable_extra_params() { + // @@protoc_insertion_point(field_mutable_list:milvus.grpc.SearchParam.extra_params) + return &extra_params_; +} +inline const ::milvus::grpc::KeyValuePair& SearchParam::extra_params(int index) const { + // @@protoc_insertion_point(field_get:milvus.grpc.SearchParam.extra_params) + return extra_params_.Get(index); +} +inline ::milvus::grpc::KeyValuePair* SearchParam::add_extra_params() { + // @@protoc_insertion_point(field_add:milvus.grpc.SearchParam.extra_params) + return extra_params_.Add(); +} +inline const ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::milvus::grpc::KeyValuePair >& +SearchParam::extra_params() const { + // @@protoc_insertion_point(field_list:milvus.grpc.SearchParam.extra_params) + return extra_params_; +} + // ------------------------------------------------------------------- // SearchInFilesParam @@ -5376,49 +5618,7 @@ inline void SearchByIDParam::set_allocated_table_name(std::string* table_name) { // @@protoc_insertion_point(field_set_allocated:milvus.grpc.SearchByIDParam.table_name) } -// int64 id = 2; -inline void SearchByIDParam::clear_id() { - id_ = PROTOBUF_LONGLONG(0); -} -inline ::PROTOBUF_NAMESPACE_ID::int64 SearchByIDParam::id() const { - // @@protoc_insertion_point(field_get:milvus.grpc.SearchByIDParam.id) - return id_; -} -inline void SearchByIDParam::set_id(::PROTOBUF_NAMESPACE_ID::int64 value) { - - id_ = value; - // @@protoc_insertion_point(field_set:milvus.grpc.SearchByIDParam.id) -} - -// int64 topk = 3; -inline void SearchByIDParam::clear_topk() { - topk_ = PROTOBUF_LONGLONG(0); -} -inline ::PROTOBUF_NAMESPACE_ID::int64 SearchByIDParam::topk() const { - // @@protoc_insertion_point(field_get:milvus.grpc.SearchByIDParam.topk) - return topk_; -} -inline void SearchByIDParam::set_topk(::PROTOBUF_NAMESPACE_ID::int64 value) { - - topk_ = value; - // @@protoc_insertion_point(field_set:milvus.grpc.SearchByIDParam.topk) -} - -// int64 nprobe = 4; -inline void SearchByIDParam::clear_nprobe() { - nprobe_ = PROTOBUF_LONGLONG(0); -} -inline ::PROTOBUF_NAMESPACE_ID::int64 SearchByIDParam::nprobe() const { - // @@protoc_insertion_point(field_get:milvus.grpc.SearchByIDParam.nprobe) - return nprobe_; -} -inline void SearchByIDParam::set_nprobe(::PROTOBUF_NAMESPACE_ID::int64 value) { - - nprobe_ = value; - // @@protoc_insertion_point(field_set:milvus.grpc.SearchByIDParam.nprobe) -} - -// repeated string partition_tag_array = 5; +// repeated string partition_tag_array = 2; inline int SearchByIDParam::partition_tag_array_size() const { return partition_tag_array_.size(); } @@ -5483,6 +5683,64 @@ SearchByIDParam::mutable_partition_tag_array() { return &partition_tag_array_; } +// int64 id = 3; +inline void SearchByIDParam::clear_id() { + id_ = PROTOBUF_LONGLONG(0); +} +inline ::PROTOBUF_NAMESPACE_ID::int64 SearchByIDParam::id() const { + // @@protoc_insertion_point(field_get:milvus.grpc.SearchByIDParam.id) + return id_; +} +inline void SearchByIDParam::set_id(::PROTOBUF_NAMESPACE_ID::int64 value) { + + id_ = value; + // @@protoc_insertion_point(field_set:milvus.grpc.SearchByIDParam.id) +} + +// int64 topk = 4; +inline void SearchByIDParam::clear_topk() { + topk_ = PROTOBUF_LONGLONG(0); +} +inline ::PROTOBUF_NAMESPACE_ID::int64 SearchByIDParam::topk() const { + // @@protoc_insertion_point(field_get:milvus.grpc.SearchByIDParam.topk) + return topk_; +} +inline void SearchByIDParam::set_topk(::PROTOBUF_NAMESPACE_ID::int64 value) { + + topk_ = value; + // @@protoc_insertion_point(field_set:milvus.grpc.SearchByIDParam.topk) +} + +// repeated .milvus.grpc.KeyValuePair extra_params = 5; +inline int SearchByIDParam::extra_params_size() const { + return extra_params_.size(); +} +inline void SearchByIDParam::clear_extra_params() { + extra_params_.Clear(); +} +inline ::milvus::grpc::KeyValuePair* SearchByIDParam::mutable_extra_params(int index) { + // @@protoc_insertion_point(field_mutable:milvus.grpc.SearchByIDParam.extra_params) + return extra_params_.Mutable(index); +} +inline ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::milvus::grpc::KeyValuePair >* +SearchByIDParam::mutable_extra_params() { + // @@protoc_insertion_point(field_mutable_list:milvus.grpc.SearchByIDParam.extra_params) + return &extra_params_; +} +inline const ::milvus::grpc::KeyValuePair& SearchByIDParam::extra_params(int index) const { + // @@protoc_insertion_point(field_get:milvus.grpc.SearchByIDParam.extra_params) + return extra_params_.Get(index); +} +inline ::milvus::grpc::KeyValuePair* SearchByIDParam::add_extra_params() { + // @@protoc_insertion_point(field_add:milvus.grpc.SearchByIDParam.extra_params) + return extra_params_.Add(); +} +inline const ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::milvus::grpc::KeyValuePair >& +SearchByIDParam::extra_params() const { + // @@protoc_insertion_point(field_list:milvus.grpc.SearchByIDParam.extra_params) + return extra_params_; +} + // ------------------------------------------------------------------- // TopKQueryResult @@ -5889,38 +6147,6 @@ inline void Command::set_allocated_cmd(std::string* cmd) { // ------------------------------------------------------------------- -// Index - -// int32 index_type = 1; -inline void Index::clear_index_type() { - index_type_ = 0; -} -inline ::PROTOBUF_NAMESPACE_ID::int32 Index::index_type() const { - // @@protoc_insertion_point(field_get:milvus.grpc.Index.index_type) - return index_type_; -} -inline void Index::set_index_type(::PROTOBUF_NAMESPACE_ID::int32 value) { - - index_type_ = value; - // @@protoc_insertion_point(field_set:milvus.grpc.Index.index_type) -} - -// int32 nlist = 2; -inline void Index::clear_nlist() { - nlist_ = 0; -} -inline ::PROTOBUF_NAMESPACE_ID::int32 Index::nlist() const { - // @@protoc_insertion_point(field_get:milvus.grpc.Index.nlist) - return nlist_; -} -inline void Index::set_nlist(::PROTOBUF_NAMESPACE_ID::int32 value) { - - nlist_ = value; - // @@protoc_insertion_point(field_set:milvus.grpc.Index.nlist) -} - -// ------------------------------------------------------------------- - // IndexParam // .milvus.grpc.Status status = 1; @@ -6019,55 +6245,48 @@ inline void IndexParam::set_allocated_table_name(std::string* table_name) { // @@protoc_insertion_point(field_set_allocated:milvus.grpc.IndexParam.table_name) } -// .milvus.grpc.Index index = 3; -inline bool IndexParam::has_index() const { - return this != internal_default_instance() && index_ != nullptr; +// int32 index_type = 3; +inline void IndexParam::clear_index_type() { + index_type_ = 0; } -inline void IndexParam::clear_index() { - if (GetArenaNoVirtual() == nullptr && index_ != nullptr) { - delete index_; - } - index_ = nullptr; +inline ::PROTOBUF_NAMESPACE_ID::int32 IndexParam::index_type() const { + // @@protoc_insertion_point(field_get:milvus.grpc.IndexParam.index_type) + return index_type_; } -inline const ::milvus::grpc::Index& IndexParam::index() const { - const ::milvus::grpc::Index* p = index_; - // @@protoc_insertion_point(field_get:milvus.grpc.IndexParam.index) - return p != nullptr ? *p : *reinterpret_cast( - &::milvus::grpc::_Index_default_instance_); -} -inline ::milvus::grpc::Index* IndexParam::release_index() { - // @@protoc_insertion_point(field_release:milvus.grpc.IndexParam.index) +inline void IndexParam::set_index_type(::PROTOBUF_NAMESPACE_ID::int32 value) { - ::milvus::grpc::Index* temp = index_; - index_ = nullptr; - return temp; + index_type_ = value; + // @@protoc_insertion_point(field_set:milvus.grpc.IndexParam.index_type) } -inline ::milvus::grpc::Index* IndexParam::mutable_index() { - - if (index_ == nullptr) { - auto* p = CreateMaybeMessage<::milvus::grpc::Index>(GetArenaNoVirtual()); - index_ = p; - } - // @@protoc_insertion_point(field_mutable:milvus.grpc.IndexParam.index) - return index_; + +// repeated .milvus.grpc.KeyValuePair extra_params = 4; +inline int IndexParam::extra_params_size() const { + return extra_params_.size(); } -inline void IndexParam::set_allocated_index(::milvus::grpc::Index* index) { - ::PROTOBUF_NAMESPACE_ID::Arena* message_arena = GetArenaNoVirtual(); - if (message_arena == nullptr) { - delete index_; - } - if (index) { - ::PROTOBUF_NAMESPACE_ID::Arena* submessage_arena = nullptr; - if (message_arena != submessage_arena) { - index = ::PROTOBUF_NAMESPACE_ID::internal::GetOwnedMessage( - message_arena, index, submessage_arena); - } - - } else { - - } - index_ = index; - // @@protoc_insertion_point(field_set_allocated:milvus.grpc.IndexParam.index) +inline void IndexParam::clear_extra_params() { + extra_params_.Clear(); +} +inline ::milvus::grpc::KeyValuePair* IndexParam::mutable_extra_params(int index) { + // @@protoc_insertion_point(field_mutable:milvus.grpc.IndexParam.extra_params) + return extra_params_.Mutable(index); +} +inline ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::milvus::grpc::KeyValuePair >* +IndexParam::mutable_extra_params() { + // @@protoc_insertion_point(field_mutable_list:milvus.grpc.IndexParam.extra_params) + return &extra_params_; +} +inline const ::milvus::grpc::KeyValuePair& IndexParam::extra_params(int index) const { + // @@protoc_insertion_point(field_get:milvus.grpc.IndexParam.extra_params) + return extra_params_.Get(index); +} +inline ::milvus::grpc::KeyValuePair* IndexParam::add_extra_params() { + // @@protoc_insertion_point(field_add:milvus.grpc.IndexParam.extra_params) + return extra_params_.Add(); +} +inline const ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::milvus::grpc::KeyValuePair >& +IndexParam::extra_params() const { + // @@protoc_insertion_point(field_list:milvus.grpc.IndexParam.extra_params) + return extra_params_; } // ------------------------------------------------------------------- diff --git a/sdk/grpc/ClientProxy.cpp b/sdk/grpc/ClientProxy.cpp index b2addc5024..ad23f9b954 100644 --- a/sdk/grpc/ClientProxy.cpp +++ b/sdk/grpc/ClientProxy.cpp @@ -17,16 +17,36 @@ #include "grpc-gen/gen-milvus/milvus.grpc.pb.h" -//#define GRPC_MULTIPLE_THREAD; #define MILVUS_SDK_VERSION "0.7.0"; namespace milvus { + +static const char* EXTRA_PARAM_KEY = "params"; + bool UriCheck(const std::string& uri) { size_t index = uri.find_first_of(':', 0); return (index != std::string::npos); } +template +void +ConstructSearchParam(const std::string& table_name, + const std::vector& partition_tag_array, + int64_t topk, + const std::string& extra_params, + T& search_param) { + search_param.set_table_name(table_name); + search_param.set_topk(topk); + milvus::grpc::KeyValuePair* kv = search_param.add_extra_params(); + kv->set_key(EXTRA_PARAM_KEY); + kv->set_value(extra_params); + + for (auto& tag : partition_tag_array) { + search_param.add_partition_tag_array(tag); + } +} + void CopyRowRecord(::milvus::grpc::RowRecord* target, const RowRecord& src) { if (!src.float_data.empty()) { @@ -152,8 +172,10 @@ ClientProxy::CreateIndex(const IndexParam& index_param) { try { ::milvus::grpc::IndexParam grpc_index_param; grpc_index_param.set_table_name(index_param.table_name); - grpc_index_param.mutable_index()->set_index_type(static_cast(index_param.index_type)); - grpc_index_param.mutable_index()->set_nlist(index_param.nlist); + grpc_index_param.set_index_type(static_cast(index_param.index_type)); + milvus::grpc::KeyValuePair* kv = grpc_index_param.add_extra_params(); + kv->set_key(EXTRA_PARAM_KEY); + kv->set_value(index_param.extra_params); return client_ptr_->CreateIndex(grpc_index_param); } catch (std::exception& ex) { return Status(StatusCode::UnknownError, "Failed to build index: " + std::string(ex.what())); @@ -165,47 +187,6 @@ ClientProxy::Insert(const std::string& table_name, const std::string& partition_ const std::vector& record_array, std::vector& id_array) { Status status = Status::OK(); try { -//////////////////////////////////////////////////////////////////////////// -#ifdef GRPC_MULTIPLE_THREAD - std::vector threads; - int thread_count = 10; - - std::shared_ptr<::milvus::grpc::InsertInfos> insert_info_array( - new ::milvus::grpc::InsertInfos[thread_count], std::default_delete<::milvus::grpc::InsertInfos[]>()); - - std::shared_ptr<::milvus::grpc::VectorIds> vector_ids_array(new ::milvus::grpc::VectorIds[thread_count], - std::default_delete<::milvus::grpc::VectorIds[]>()); - - int64_t record_count = record_array.size() / thread_count; - - for (size_t i = 0; i < thread_count; i++) { - insert_info_array.get()[i].set_table_name(table_name); - for (size_t j = i * record_count; j < record_count * (i + 1); j++) { - ::milvus::grpc::RowRecord* grpc_record = insert_info_array.get()[i].add_row_record_array(); - for (size_t k = 0; k < record_array[j].data.size(); k++) { - grpc_record->add_vector_data(record_array[j].data[k]); - } - } - } - - std::cout << "*****************************************************\n"; - auto start = std::chrono::high_resolution_clock::now(); - for (size_t j = 0; j < thread_count; j++) { - threads.push_back(std::thread(&GrpcClient::InsertVector, client_ptr_, std::ref(vector_ids_array.get()[j]), - std::ref(insert_info_array.get()[j]), std::ref(status))); - } - std::for_each(threads.begin(), threads.end(), std::mem_fn(&std::thread::join)); - auto finish = std::chrono::high_resolution_clock::now(); - std::cout << "InsertVector cost: " - << std::chrono::duration_cast>(finish - start).count() << "s\n"; - std::cout << "*****************************************************\n"; - - for (size_t i = 0; i < thread_count; i++) { - for (size_t j = 0; j < vector_ids_array.get()[i].vector_id_array_size(); j++) { - id_array.push_back(vector_ids_array.get()[i].vector_id_array(j)); - } - } -#else ::milvus::grpc::InsertParam insert_param; insert_param.set_table_name(table_name); insert_param.set_partition_tag(partition_tag); @@ -228,7 +209,6 @@ ClientProxy::Insert(const std::string& table_name, const std::string& partition_ /* return Milvus generated ids back to user */ id_array.insert(id_array.end(), vector_ids.vector_id_array().begin(), vector_ids.vector_id_array().end()); } -#endif } catch (std::exception& ex) { return Status(StatusCode::UnknownError, "Failed to add vector: " + std::string(ex.what())); } @@ -291,18 +271,18 @@ ClientProxy::GetIDsInSegment(const std::string& table_name, const std::string& s } Status -ClientProxy::Search(const std::string& table_name, const std::vector& partition_tags, - const std::vector& query_record_array, int64_t topk, int64_t nprobe, +ClientProxy::Search(const std::string& table_name, const std::vector& partition_tag_array, + const std::vector& query_record_array, int64_t topk, const std::string& extra_params, TopKQueryResult& topk_query_result) { try { // step 1: convert vectors data ::milvus::grpc::SearchParam search_param; - search_param.set_table_name(table_name); - search_param.set_topk(topk); - search_param.set_nprobe(nprobe); - for (auto& tag : partition_tags) { - search_param.add_partition_tag_array(tag); - } + ConstructSearchParam(table_name, + partition_tag_array, + topk, + extra_params, + search_param); + for (auto& record : query_record_array) { ::milvus::grpc::RowRecord* row_record = search_param.add_query_record_array(); CopyRowRecord(row_record, record); @@ -334,46 +314,6 @@ ClientProxy::Search(const std::string& table_name, const std::vector& partition_tags, int64_t query_id, - int64_t topk, int64_t nprobe, TopKQueryResult& topk_query_result) { - try { - // step 1: convert vector id array - ::milvus::grpc::SearchByIDParam search_param; - search_param.set_table_name(table_name); - search_param.set_topk(topk); - search_param.set_nprobe(nprobe); - for (auto& tag : partition_tags) { - search_param.add_partition_tag_array(tag); - } - search_param.set_id(query_id); - - // step 2: search vectors - ::milvus::grpc::TopKQueryResult result; - Status status = client_ptr_->SearchByID(search_param, result); - if (result.row_num() == 0) { - return status; - } - - // step 4: convert result array - topk_query_result.reserve(result.row_num()); - int64_t nq = result.row_num(); - int64_t topk = result.ids().size() / nq; - for (int64_t i = 0; i < result.row_num(); i++) { - milvus::QueryResult one_result; - one_result.ids.resize(topk); - one_result.distances.resize(topk); - memcpy(one_result.ids.data(), result.ids().data() + topk * i, topk * sizeof(int64_t)); - memcpy(one_result.distances.data(), result.distances().data() + topk * i, topk * sizeof(float)); - topk_query_result.emplace_back(one_result); - } - - return status; - } catch (std::exception& ex) { - return Status(StatusCode::UnknownError, "Failed to search vectors: " + std::string(ex.what())); - } -} - Status ClientProxy::DescribeTable(const std::string& table_name, TableSchema& table_schema) { try { @@ -523,10 +463,17 @@ ClientProxy::DescribeIndex(const std::string& table_name, IndexParam& index_para try { ::milvus::grpc::TableName grpc_table_name; grpc_table_name.set_table_name(table_name); + ::milvus::grpc::IndexParam grpc_index_param; Status status = client_ptr_->DescribeIndex(grpc_table_name, grpc_index_param); - index_param.index_type = static_cast(grpc_index_param.mutable_index()->index_type()); - index_param.nlist = grpc_index_param.mutable_index()->nlist(); + index_param.index_type = static_cast(grpc_index_param.index_type()); + + for (int i = 0; i < grpc_index_param.extra_params_size(); i++) { + const milvus::grpc::KeyValuePair& kv = grpc_index_param.extra_params(i); + if (kv.key() == EXTRA_PARAM_KEY) { + index_param.extra_params = kv.value(); + } + } return status; } catch (std::exception& ex) { @@ -560,15 +507,15 @@ ClientProxy::CreatePartition(const PartitionParam& partition_param) { } Status -ClientProxy::ShowPartitions(const std::string& table_name, PartitionTagList& partition_array) const { +ClientProxy::ShowPartitions(const std::string& table_name, PartitionTagList& partition_tag_array) const { try { ::milvus::grpc::TableName grpc_table_name; grpc_table_name.set_table_name(table_name); ::milvus::grpc::PartitionList grpc_partition_list; Status status = client_ptr_->ShowPartitions(grpc_table_name, grpc_partition_list); - partition_array.resize(grpc_partition_list.partition_tag_array_size()); + partition_tag_array.resize(grpc_partition_list.partition_tag_array_size()); for (uint64_t i = 0; i < grpc_partition_list.partition_tag_array_size(); ++i) { - partition_array[i] = grpc_partition_list.partition_tag_array(i); + partition_tag_array[i] = grpc_partition_list.partition_tag_array(i); } return status; } catch (std::exception& ex) { diff --git a/sdk/grpc/ClientProxy.h b/sdk/grpc/ClientProxy.h index fab399331b..0fd493dcd2 100644 --- a/sdk/grpc/ClientProxy.h +++ b/sdk/grpc/ClientProxy.h @@ -59,15 +59,10 @@ class ClientProxy : public Connection { std::vector& id_array) override; Status - Search(const std::string& table_name, const std::vector& partition_tags, - const std::vector& query_record_array, int64_t topk, int64_t nprobe, + Search(const std::string& table_name, const std::vector& partition_tag_array, + const std::vector& query_record_array, int64_t topk, const std::string& extra_params, TopKQueryResult& topk_query_result) override; - Status - SearchByID(const std::string& table_name, const std::vector& partition_tags, - int64_t query_id, int64_t topk, - int64_t nprobe, TopKQueryResult& topk_query_result) override; - Status DescribeTable(const std::string& table_name, TableSchema& table_schema) override; @@ -108,7 +103,7 @@ class ClientProxy : public Connection { CreatePartition(const PartitionParam& partition_param) override; Status - ShowPartitions(const std::string& table_name, PartitionTagList& partition_array) const override; + ShowPartitions(const std::string& table_name, PartitionTagList& partition_tag_array) const override; Status DropPartition(const PartitionParam& partition_param) override; diff --git a/sdk/grpc/GrpcClient.cpp b/sdk/grpc/GrpcClient.cpp index 47c6c9e8ab..96773f9c75 100644 --- a/sdk/grpc/GrpcClient.cpp +++ b/sdk/grpc/GrpcClient.cpp @@ -178,26 +178,6 @@ GrpcClient::Search( return Status::OK(); } -Status -GrpcClient::SearchByID(const ::milvus::grpc::SearchByIDParam& search_param, - ::milvus::grpc::TopKQueryResult& topk_query_result) { - ::milvus::grpc::TopKQueryResult query_result; - ClientContext context; - ::grpc::Status grpc_status = stub_->SearchByID(&context, search_param, &topk_query_result); - - if (!grpc_status.ok()) { - std::cerr << "SearchByID rpc failed!" << std::endl; - std::cerr << grpc_status.error_message() << std::endl; - return Status(StatusCode::RPCFailed, grpc_status.error_message()); - } - if (topk_query_result.status().error_code() != grpc::SUCCESS) { - std::cerr << topk_query_result.status().reason() << std::endl; - return Status(StatusCode::ServerFailed, topk_query_result.status().reason()); - } - - return Status::OK(); -} - Status GrpcClient::DescribeTable(const std::string& table_name, ::milvus::grpc::TableSchema& grpc_schema) { ClientContext context; diff --git a/sdk/grpc/GrpcClient.h b/sdk/grpc/GrpcClient.h index 51af08a2c7..d7b7ae42c5 100644 --- a/sdk/grpc/GrpcClient.h +++ b/sdk/grpc/GrpcClient.h @@ -59,9 +59,6 @@ class GrpcClient { Status Search(const grpc::SearchParam& search_param, ::milvus::grpc::TopKQueryResult& topk_query_result); - Status - SearchByID(const grpc::SearchByIDParam& search_param, ::milvus::grpc::TopKQueryResult& topk_query_result); - Status DescribeTable(const std::string& table_name, grpc::TableSchema& grpc_schema); diff --git a/sdk/include/MilvusApi.h b/sdk/include/MilvusApi.h index 353dcef43f..111cc79dc3 100644 --- a/sdk/include/MilvusApi.h +++ b/sdk/include/MilvusApi.h @@ -59,18 +59,10 @@ struct ConnectParam { struct TableSchema { std::string table_name; ///< Table name int64_t dimension = 0; ///< Vector dimension, must be a positive value - int64_t index_file_size = 0; ///< Index file size, must be a positive value + int64_t index_file_size = 1024; ///< Index file size, must be a positive value, unit: MB MetricType metric_type = MetricType::L2; ///< Index metric type }; -/** - * @brief Range information - * for DATE range, the format is like: 'year-month-day' - */ -struct Range { - std::string start_value; ///< Range start - std::string end_value; ///< Range stop -}; /** * @brief Record inserted @@ -90,12 +82,18 @@ struct QueryResult { using TopKQueryResult = std::vector; ///< Topk query result /** - * @brief index parameters + * @brief Index parameters + * Note: extra_params is extra parameters list, it must be json format + * For different index type, parameter list is different accordingly, for example: + * FLAT/IVFLAT/SQ8: "{nlist: '16384'}" + * IVFPQ: "{nlist: '16384', nbits: "16"}" + * NSG: "{search_length: '100', out_degree:'40', pool_size:'66'}" + * HNSW "{M: '16', ef_construct:'500'}" */ struct IndexParam { - std::string table_name; ///< Table name for create index - IndexType index_type; ///< Create index type - int32_t nlist; ///< Index nlist + std::string table_name; ///< Table name for create index + IndexType index_type; ///< Index type + std::string extra_params; ///< Extra parameters according to different index type, must be json format }; /** @@ -131,7 +129,7 @@ struct PartitionStat { * @brief table info */ struct TableInfo { - int64_t total_row_count; ///< Table total row count + int64_t total_row_count; ///< Table total row count std::vector partitions_stat; ///< Table's partitions statistics }; @@ -269,7 +267,7 @@ class Connection { * This method is used to insert vector array to table. * * @param table_name, target table's name. - * @param partition_tag, target partition's tag, keep empty if no partition. + * @param partition_tag, target partition's tag, keep empty if no partition specified. * @param record_array, vector array is inserted. * @param id_array, * specify id for each vector, @@ -318,37 +316,23 @@ class Connection { * This method is used to query vector in table. * * @param table_name, target table's name. - * @param partition_tags, target partitions, keep empty if no partition. + * @param partition_tag_array, target partitions, keep empty if no partition specified. * @param query_record_array, vectors to be queried. - * @param query_range_array, [deprecated] time ranges, if not specified, will search in whole table - * @param topk, how many similarity vectors will be searched. - * @param nprobe, the number of centroids choose to search. - * @param topk_query_result_array, result array. + * @param topk, how many similarity vectors will be returned. + * @param extra_params, extra search parameters according to different index type, must be json format. + * Note: extra_params is extra parameters list, it must be json format, for example: + * For different index type, parameter list is different accordingly + * FLAT/IVFLAT/SQ8/IVFPQ: "{nprobe: '32'}" + * NSG: "{search_length:'100'} + * HNSW "{ef: '64'} + * @param topk_query_result, result array. * * @return Indicate if query is successful. */ virtual Status - Search(const std::string& table_name, const std::vector& partition_tags, + Search(const std::string& table_name, const PartitionTagList& partition_tag_array, const std::vector& query_record_array, int64_t topk, - int64_t nprobe, TopKQueryResult& topk_query_result) = 0; - - /** - * @brief Search vector by ID - * - * This method is used to query vector in table. - * - * @param table_name, target table's name. - * @param partition_tags, target partitions, keep empty if no partition. - * @param query_id_array, vector ids to be queried. - * @param topk, how many similarity vectors will be searched. - * @param nprobe, the number of centroids choose to search. - * @param topk_query_result_array, result array. - * - * @return Indicate if query is successful. - */ - virtual Status - SearchByID(const std::string& table_name, const std::vector& partition_tags, int64_t query_id, - int64_t topk, int64_t nprobe, TopKQueryResult& topk_query_result) = 0; + const std::string& extra_params, TopKQueryResult& topk_query_result) = 0; /** * @brief Show table description @@ -510,12 +494,12 @@ class Connection { * This method is used to create table * * @param table_name, table name is going to be tested. - * @param partition_array, partition tag array of the table. + * @param partition_tag_array, partition tag array of the table. * * @return Indicate if this operation is successful */ virtual Status - ShowPartitions(const std::string& table_name, PartitionTagList& partition_array) const = 0; + ShowPartitions(const std::string& table_name, PartitionTagList& partition_tag_array) const = 0; /** * @brief Delete partition method diff --git a/sdk/interface/ConnectionImpl.cpp b/sdk/interface/ConnectionImpl.cpp index 7e76836ab3..4acbcec7b8 100644 --- a/sdk/interface/ConnectionImpl.cpp +++ b/sdk/interface/ConnectionImpl.cpp @@ -89,21 +89,15 @@ ConnectionImpl::GetVectorByID(const std::string& table_name, int64_t vector_id, Status ConnectionImpl::GetIDsInSegment(const std::string& table_name, const std::string& segment_name, - std::vector& id_array) { + std::vector& id_array) { return client_proxy_->GetIDsInSegment(table_name, segment_name, id_array); } Status ConnectionImpl::Search(const std::string& table_name, const std::vector& partition_tags, - const std::vector& query_record_array, int64_t topk, int64_t nprobe, + const std::vector& query_record_array, int64_t topk, const std::string& extra_params, TopKQueryResult& topk_query_result) { - return client_proxy_->Search(table_name, partition_tags, query_record_array, topk, nprobe, topk_query_result); -} - -Status -ConnectionImpl::SearchByID(const std::string& table_name, const std::vector& partition_tags, - int64_t query_id, int64_t topk, int64_t nprobe, TopKQueryResult& topk_query_result) { - return client_proxy_->SearchByID(table_name, partition_tags, query_id, topk, nprobe, topk_query_result); + return client_proxy_->Search(table_name, partition_tags, query_record_array, topk, extra_params, topk_query_result); } Status diff --git a/sdk/interface/ConnectionImpl.h b/sdk/interface/ConnectionImpl.h index c06afda346..ec0f423857 100644 --- a/sdk/interface/ConnectionImpl.h +++ b/sdk/interface/ConnectionImpl.h @@ -61,13 +61,9 @@ class ConnectionImpl : public Connection { std::vector& id_array) override; Status - Search(const std::string& table_name, const std::vector& partition_tags, + Search(const std::string& table_name, const std::vector& partition_tag_array, const std::vector& query_record_array, int64_t topk, - int64_t nprobe, TopKQueryResult& topk_query_result) override; - - Status - SearchByID(const std::string& table_name, const std::vector& partition_tags, int64_t query_id, - int64_t topk, int64_t nprobe, TopKQueryResult& topk_query_result) override; + const std::string& extra_params, TopKQueryResult& topk_query_result) override; Status DescribeTable(const std::string& table_name, TableSchema& table_schema) override; @@ -109,7 +105,7 @@ class ConnectionImpl : public Connection { CreatePartition(const PartitionParam& param) override; Status - ShowPartitions(const std::string& table_name, PartitionTagList& partition_array) const override; + ShowPartitions(const std::string& table_name, PartitionTagList& partition_tag_array) const override; Status DropPartition(const PartitionParam& param) override; diff --git a/sdk/thirdparty/nlohmann/json.hpp b/sdk/thirdparty/nlohmann/json.hpp new file mode 100644 index 0000000000..4acdcd3aea --- /dev/null +++ b/sdk/thirdparty/nlohmann/json.hpp @@ -0,0 +1,21006 @@ +/* + __ _____ _____ _____ + __| | __| | | | JSON for Modern C++ +| | |__ | | | | | | version 3.7.0 +|_____|_____|_____|_|___| https://github.com/nlohmann/json + +Licensed under the MIT License . +SPDX-License-Identifier: MIT +Copyright (c) 2013-2019 Niels Lohmann . + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +#ifndef INCLUDE_NLOHMANN_JSON_HPP_ +#define INCLUDE_NLOHMANN_JSON_HPP_ + +#define NLOHMANN_JSON_VERSION_MAJOR 3 +#define NLOHMANN_JSON_VERSION_MINOR 7 +#define NLOHMANN_JSON_VERSION_PATCH 0 + +#include // all_of, find, for_each +#include // assert +#include // and, not, or +#include // nullptr_t, ptrdiff_t, size_t +#include // hash, less +#include // initializer_list +#include // istream, ostream +#include // random_access_iterator_tag +#include // unique_ptr +#include // accumulate +#include // string, stoi, to_string +#include // declval, forward, move, pair, swap +#include // vector + +// #include + +#include + +// #include + +#include // transform +#include // array +#include // and, not +#include // forward_list +#include // inserter, front_inserter, end +#include // map +#include // string +#include // tuple, make_tuple +#include // is_arithmetic, is_same, is_enum, underlying_type, is_convertible +#include // unordered_map +#include // pair, declval +#include // valarray + +// #include + +#include // exception +#include // runtime_error +#include // to_string + +// #include + +#include // size_t + +namespace nlohmann { +namespace detail { +/// struct to capture the start position of the current token +struct position_t { + /// the total number of characters read + std::size_t chars_read_total = 0; + /// the number of characters read in the current line + std::size_t chars_read_current_line = 0; + /// the number of lines read + std::size_t lines_read = 0; + + /// conversion to size_t to preserve SAX interface + constexpr operator size_t() const { + return chars_read_total; + } +}; + +} // namespace detail +} // namespace nlohmann + +// #include + +#include // pair +// #include +/* Hedley - https://nemequ.github.io/hedley + * Created by Evan Nemerson + * + * To the extent possible under law, the author(s) have dedicated all + * copyright and related and neighboring rights to this software to + * the public domain worldwide. This software is distributed without + * any warranty. + * + * For details, see . + * SPDX-License-Identifier: CC0-1.0 + */ + +#if !defined(JSON_HEDLEY_VERSION) || (JSON_HEDLEY_VERSION < 9) +#if defined(JSON_HEDLEY_VERSION) +#undef JSON_HEDLEY_VERSION +#endif +#define JSON_HEDLEY_VERSION 9 + +#if defined(JSON_HEDLEY_STRINGIFY_EX) +#undef JSON_HEDLEY_STRINGIFY_EX +#endif +#define JSON_HEDLEY_STRINGIFY_EX(x) #x + +#if defined(JSON_HEDLEY_STRINGIFY) +#undef JSON_HEDLEY_STRINGIFY +#endif +#define JSON_HEDLEY_STRINGIFY(x) JSON_HEDLEY_STRINGIFY_EX(x) + +#if defined(JSON_HEDLEY_CONCAT_EX) +#undef JSON_HEDLEY_CONCAT_EX +#endif +#define JSON_HEDLEY_CONCAT_EX(a, b) a##b + +#if defined(JSON_HEDLEY_CONCAT) +#undef JSON_HEDLEY_CONCAT +#endif +#define JSON_HEDLEY_CONCAT(a, b) JSON_HEDLEY_CONCAT_EX(a, b) + +#if defined(JSON_HEDLEY_VERSION_ENCODE) +#undef JSON_HEDLEY_VERSION_ENCODE +#endif +#define JSON_HEDLEY_VERSION_ENCODE(major, minor, revision) (((major)*1000000) + ((minor)*1000) + (revision)) + +#if defined(JSON_HEDLEY_VERSION_DECODE_MAJOR) +#undef JSON_HEDLEY_VERSION_DECODE_MAJOR +#endif +#define JSON_HEDLEY_VERSION_DECODE_MAJOR(version) ((version) / 1000000) + +#if defined(JSON_HEDLEY_VERSION_DECODE_MINOR) +#undef JSON_HEDLEY_VERSION_DECODE_MINOR +#endif +#define JSON_HEDLEY_VERSION_DECODE_MINOR(version) (((version) % 1000000) / 1000) + +#if defined(JSON_HEDLEY_VERSION_DECODE_REVISION) +#undef JSON_HEDLEY_VERSION_DECODE_REVISION +#endif +#define JSON_HEDLEY_VERSION_DECODE_REVISION(version) ((version) % 1000) + +#if defined(JSON_HEDLEY_GNUC_VERSION) +#undef JSON_HEDLEY_GNUC_VERSION +#endif +#if defined(__GNUC__) && defined(__GNUC_PATCHLEVEL__) +#define JSON_HEDLEY_GNUC_VERSION JSON_HEDLEY_VERSION_ENCODE(__GNUC__, __GNUC_MINOR__, __GNUC_PATCHLEVEL__) +#elif defined(__GNUC__) +#define JSON_HEDLEY_GNUC_VERSION JSON_HEDLEY_VERSION_ENCODE(__GNUC__, __GNUC_MINOR__, 0) +#endif + +#if defined(JSON_HEDLEY_GNUC_VERSION_CHECK) +#undef JSON_HEDLEY_GNUC_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_GNUC_VERSION) +#define JSON_HEDLEY_GNUC_VERSION_CHECK(major, minor, patch) \ + (JSON_HEDLEY_GNUC_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else +#define JSON_HEDLEY_GNUC_VERSION_CHECK(major, minor, patch) (0) +#endif + +#if defined(JSON_HEDLEY_MSVC_VERSION) +#undef JSON_HEDLEY_MSVC_VERSION +#endif +#if defined(_MSC_FULL_VER) && (_MSC_FULL_VER >= 140000000) +#define JSON_HEDLEY_MSVC_VERSION \ + JSON_HEDLEY_VERSION_ENCODE(_MSC_FULL_VER / 10000000, (_MSC_FULL_VER % 10000000) / 100000, \ + (_MSC_FULL_VER % 100000) / 100) +#elif defined(_MSC_FULL_VER) +#define JSON_HEDLEY_MSVC_VERSION \ + JSON_HEDLEY_VERSION_ENCODE(_MSC_FULL_VER / 1000000, (_MSC_FULL_VER % 1000000) / 10000, (_MSC_FULL_VER % 10000) / 10) +#elif defined(_MSC_VER) +#define JSON_HEDLEY_MSVC_VERSION JSON_HEDLEY_VERSION_ENCODE(_MSC_VER / 100, _MSC_VER % 100, 0) +#endif + +#if defined(JSON_HEDLEY_MSVC_VERSION_CHECK) +#undef JSON_HEDLEY_MSVC_VERSION_CHECK +#endif +#if !defined(_MSC_VER) +#define JSON_HEDLEY_MSVC_VERSION_CHECK(major, minor, patch) (0) +#elif defined(_MSC_VER) && (_MSC_VER >= 1400) +#define JSON_HEDLEY_MSVC_VERSION_CHECK(major, minor, patch) \ + (_MSC_FULL_VER >= ((major * 10000000) + (minor * 100000) + (patch))) +#elif defined(_MSC_VER) && (_MSC_VER >= 1200) +#define JSON_HEDLEY_MSVC_VERSION_CHECK(major, minor, patch) \ + (_MSC_FULL_VER >= ((major * 1000000) + (minor * 10000) + (patch))) +#else +#define JSON_HEDLEY_MSVC_VERSION_CHECK(major, minor, patch) (_MSC_VER >= ((major * 100) + (minor))) +#endif + +#if defined(JSON_HEDLEY_INTEL_VERSION) +#undef JSON_HEDLEY_INTEL_VERSION +#endif +#if defined(__INTEL_COMPILER) && defined(__INTEL_COMPILER_UPDATE) +#define JSON_HEDLEY_INTEL_VERSION \ + JSON_HEDLEY_VERSION_ENCODE(__INTEL_COMPILER / 100, __INTEL_COMPILER % 100, __INTEL_COMPILER_UPDATE) +#elif defined(__INTEL_COMPILER) +#define JSON_HEDLEY_INTEL_VERSION JSON_HEDLEY_VERSION_ENCODE(__INTEL_COMPILER / 100, __INTEL_COMPILER % 100, 0) +#endif + +#if defined(JSON_HEDLEY_INTEL_VERSION_CHECK) +#undef JSON_HEDLEY_INTEL_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_INTEL_VERSION) +#define JSON_HEDLEY_INTEL_VERSION_CHECK(major, minor, patch) \ + (JSON_HEDLEY_INTEL_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else +#define JSON_HEDLEY_INTEL_VERSION_CHECK(major, minor, patch) (0) +#endif + +#if defined(JSON_HEDLEY_PGI_VERSION) +#undef JSON_HEDLEY_PGI_VERSION +#endif +#if defined(__PGI) && defined(__PGIC__) && defined(__PGIC_MINOR__) && defined(__PGIC_PATCHLEVEL__) +#define JSON_HEDLEY_PGI_VERSION JSON_HEDLEY_VERSION_ENCODE(__PGIC__, __PGIC_MINOR__, __PGIC_PATCHLEVEL__) +#endif + +#if defined(JSON_HEDLEY_PGI_VERSION_CHECK) +#undef JSON_HEDLEY_PGI_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_PGI_VERSION) +#define JSON_HEDLEY_PGI_VERSION_CHECK(major, minor, patch) \ + (JSON_HEDLEY_PGI_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else +#define JSON_HEDLEY_PGI_VERSION_CHECK(major, minor, patch) (0) +#endif + +#if defined(JSON_HEDLEY_SUNPRO_VERSION) +#undef JSON_HEDLEY_SUNPRO_VERSION +#endif +#if defined(__SUNPRO_C) && (__SUNPRO_C > 0x1000) +#define JSON_HEDLEY_SUNPRO_VERSION \ + JSON_HEDLEY_VERSION_ENCODE((((__SUNPRO_C >> 16) & 0xf) * 10) + ((__SUNPRO_C >> 12) & 0xf), \ + (((__SUNPRO_C >> 8) & 0xf) * 10) + ((__SUNPRO_C >> 4) & 0xf), (__SUNPRO_C & 0xf) * 10) +#elif defined(__SUNPRO_C) +#define JSON_HEDLEY_SUNPRO_VERSION \ + JSON_HEDLEY_VERSION_ENCODE((__SUNPRO_C >> 8) & 0xf, (__SUNPRO_C >> 4) & 0xf, (__SUNPRO_C)&0xf) +#elif defined(__SUNPRO_CC) && (__SUNPRO_CC > 0x1000) +#define JSON_HEDLEY_SUNPRO_VERSION \ + JSON_HEDLEY_VERSION_ENCODE((((__SUNPRO_CC >> 16) & 0xf) * 10) + ((__SUNPRO_CC >> 12) & 0xf), \ + (((__SUNPRO_CC >> 8) & 0xf) * 10) + ((__SUNPRO_CC >> 4) & 0xf), \ + (__SUNPRO_CC & 0xf) * 10) +#elif defined(__SUNPRO_CC) +#define JSON_HEDLEY_SUNPRO_VERSION \ + JSON_HEDLEY_VERSION_ENCODE((__SUNPRO_CC >> 8) & 0xf, (__SUNPRO_CC >> 4) & 0xf, (__SUNPRO_CC)&0xf) +#endif + +#if defined(JSON_HEDLEY_SUNPRO_VERSION_CHECK) +#undef JSON_HEDLEY_SUNPRO_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_SUNPRO_VERSION) +#define JSON_HEDLEY_SUNPRO_VERSION_CHECK(major, minor, patch) \ + (JSON_HEDLEY_SUNPRO_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else +#define JSON_HEDLEY_SUNPRO_VERSION_CHECK(major, minor, patch) (0) +#endif + +#if defined(JSON_HEDLEY_EMSCRIPTEN_VERSION) +#undef JSON_HEDLEY_EMSCRIPTEN_VERSION +#endif +#if defined(__EMSCRIPTEN__) +#define JSON_HEDLEY_EMSCRIPTEN_VERSION \ + JSON_HEDLEY_VERSION_ENCODE(__EMSCRIPTEN_major__, __EMSCRIPTEN_minor__, __EMSCRIPTEN_tiny__) +#endif + +#if defined(JSON_HEDLEY_EMSCRIPTEN_VERSION_CHECK) +#undef JSON_HEDLEY_EMSCRIPTEN_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_EMSCRIPTEN_VERSION) +#define JSON_HEDLEY_EMSCRIPTEN_VERSION_CHECK(major, minor, patch) \ + (JSON_HEDLEY_EMSCRIPTEN_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else +#define JSON_HEDLEY_EMSCRIPTEN_VERSION_CHECK(major, minor, patch) (0) +#endif + +#if defined(JSON_HEDLEY_ARM_VERSION) +#undef JSON_HEDLEY_ARM_VERSION +#endif +#if defined(__CC_ARM) && defined(__ARMCOMPILER_VERSION) +#define JSON_HEDLEY_ARM_VERSION \ + JSON_HEDLEY_VERSION_ENCODE(__ARMCOMPILER_VERSION / 1000000, (__ARMCOMPILER_VERSION % 1000000) / 10000, \ + (__ARMCOMPILER_VERSION % 10000) / 100) +#elif defined(__CC_ARM) && defined(__ARMCC_VERSION) +#define JSON_HEDLEY_ARM_VERSION \ + JSON_HEDLEY_VERSION_ENCODE(__ARMCC_VERSION / 1000000, (__ARMCC_VERSION % 1000000) / 10000, \ + (__ARMCC_VERSION % 10000) / 100) +#endif + +#if defined(JSON_HEDLEY_ARM_VERSION_CHECK) +#undef JSON_HEDLEY_ARM_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_ARM_VERSION) +#define JSON_HEDLEY_ARM_VERSION_CHECK(major, minor, patch) \ + (JSON_HEDLEY_ARM_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else +#define JSON_HEDLEY_ARM_VERSION_CHECK(major, minor, patch) (0) +#endif + +#if defined(JSON_HEDLEY_IBM_VERSION) +#undef JSON_HEDLEY_IBM_VERSION +#endif +#if defined(__ibmxl__) +#define JSON_HEDLEY_IBM_VERSION JSON_HEDLEY_VERSION_ENCODE(__ibmxl_version__, __ibmxl_release__, __ibmxl_modification__) +#elif defined(__xlC__) && defined(__xlC_ver__) +#define JSON_HEDLEY_IBM_VERSION JSON_HEDLEY_VERSION_ENCODE(__xlC__ >> 8, __xlC__ & 0xff, (__xlC_ver__ >> 8) & 0xff) +#elif defined(__xlC__) +#define JSON_HEDLEY_IBM_VERSION JSON_HEDLEY_VERSION_ENCODE(__xlC__ >> 8, __xlC__ & 0xff, 0) +#endif + +#if defined(JSON_HEDLEY_IBM_VERSION_CHECK) +#undef JSON_HEDLEY_IBM_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_IBM_VERSION) +#define JSON_HEDLEY_IBM_VERSION_CHECK(major, minor, patch) \ + (JSON_HEDLEY_IBM_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else +#define JSON_HEDLEY_IBM_VERSION_CHECK(major, minor, patch) (0) +#endif + +#if defined(JSON_HEDLEY_TI_VERSION) +#undef JSON_HEDLEY_TI_VERSION +#endif +#if defined(__TI_COMPILER_VERSION__) +#define JSON_HEDLEY_TI_VERSION \ + JSON_HEDLEY_VERSION_ENCODE(__TI_COMPILER_VERSION__ / 1000000, (__TI_COMPILER_VERSION__ % 1000000) / 1000, \ + (__TI_COMPILER_VERSION__ % 1000)) +#endif + +#if defined(JSON_HEDLEY_TI_VERSION_CHECK) +#undef JSON_HEDLEY_TI_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_TI_VERSION) +#define JSON_HEDLEY_TI_VERSION_CHECK(major, minor, patch) \ + (JSON_HEDLEY_TI_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else +#define JSON_HEDLEY_TI_VERSION_CHECK(major, minor, patch) (0) +#endif + +#if defined(JSON_HEDLEY_CRAY_VERSION) +#undef JSON_HEDLEY_CRAY_VERSION +#endif +#if defined(_CRAYC) +#if defined(_RELEASE_PATCHLEVEL) +#define JSON_HEDLEY_CRAY_VERSION JSON_HEDLEY_VERSION_ENCODE(_RELEASE_MAJOR, _RELEASE_MINOR, _RELEASE_PATCHLEVEL) +#else +#define JSON_HEDLEY_CRAY_VERSION JSON_HEDLEY_VERSION_ENCODE(_RELEASE_MAJOR, _RELEASE_MINOR, 0) +#endif +#endif + +#if defined(JSON_HEDLEY_CRAY_VERSION_CHECK) +#undef JSON_HEDLEY_CRAY_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_CRAY_VERSION) +#define JSON_HEDLEY_CRAY_VERSION_CHECK(major, minor, patch) \ + (JSON_HEDLEY_CRAY_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else +#define JSON_HEDLEY_CRAY_VERSION_CHECK(major, minor, patch) (0) +#endif + +#if defined(JSON_HEDLEY_IAR_VERSION) +#undef JSON_HEDLEY_IAR_VERSION +#endif +#if defined(__IAR_SYSTEMS_ICC__) +#if __VER__ > 1000 +#define JSON_HEDLEY_IAR_VERSION \ + JSON_HEDLEY_VERSION_ENCODE((__VER__ / 1000000), ((__VER__ / 1000) % 1000), (__VER__ % 1000)) +#else +#define JSON_HEDLEY_IAR_VERSION JSON_HEDLEY_VERSION_ENCODE(VER / 100, __VER__ % 100, 0) +#endif +#endif + +#if defined(JSON_HEDLEY_IAR_VERSION_CHECK) +#undef JSON_HEDLEY_IAR_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_IAR_VERSION) +#define JSON_HEDLEY_IAR_VERSION_CHECK(major, minor, patch) \ + (JSON_HEDLEY_IAR_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else +#define JSON_HEDLEY_IAR_VERSION_CHECK(major, minor, patch) (0) +#endif + +#if defined(JSON_HEDLEY_TINYC_VERSION) +#undef JSON_HEDLEY_TINYC_VERSION +#endif +#if defined(__TINYC__) +#define JSON_HEDLEY_TINYC_VERSION JSON_HEDLEY_VERSION_ENCODE(__TINYC__ / 1000, (__TINYC__ / 100) % 10, __TINYC__ % 100) +#endif + +#if defined(JSON_HEDLEY_TINYC_VERSION_CHECK) +#undef JSON_HEDLEY_TINYC_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_TINYC_VERSION) +#define JSON_HEDLEY_TINYC_VERSION_CHECK(major, minor, patch) \ + (JSON_HEDLEY_TINYC_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else +#define JSON_HEDLEY_TINYC_VERSION_CHECK(major, minor, patch) (0) +#endif + +#if defined(JSON_HEDLEY_DMC_VERSION) +#undef JSON_HEDLEY_DMC_VERSION +#endif +#if defined(__DMC__) +#define JSON_HEDLEY_DMC_VERSION JSON_HEDLEY_VERSION_ENCODE(__DMC__ >> 8, (__DMC__ >> 4) & 0xf, __DMC__ & 0xf) +#endif + +#if defined(JSON_HEDLEY_DMC_VERSION_CHECK) +#undef JSON_HEDLEY_DMC_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_DMC_VERSION) +#define JSON_HEDLEY_DMC_VERSION_CHECK(major, minor, patch) \ + (JSON_HEDLEY_DMC_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else +#define JSON_HEDLEY_DMC_VERSION_CHECK(major, minor, patch) (0) +#endif + +#if defined(JSON_HEDLEY_COMPCERT_VERSION) +#undef JSON_HEDLEY_COMPCERT_VERSION +#endif +#if defined(__COMPCERT_VERSION__) +#define JSON_HEDLEY_COMPCERT_VERSION \ + JSON_HEDLEY_VERSION_ENCODE(__COMPCERT_VERSION__ / 10000, (__COMPCERT_VERSION__ / 100) % 100, \ + __COMPCERT_VERSION__ % 100) +#endif + +#if defined(JSON_HEDLEY_COMPCERT_VERSION_CHECK) +#undef JSON_HEDLEY_COMPCERT_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_COMPCERT_VERSION) +#define JSON_HEDLEY_COMPCERT_VERSION_CHECK(major, minor, patch) \ + (JSON_HEDLEY_COMPCERT_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else +#define JSON_HEDLEY_COMPCERT_VERSION_CHECK(major, minor, patch) (0) +#endif + +#if defined(JSON_HEDLEY_PELLES_VERSION) +#undef JSON_HEDLEY_PELLES_VERSION +#endif +#if defined(__POCC__) +#define JSON_HEDLEY_PELLES_VERSION JSON_HEDLEY_VERSION_ENCODE(__POCC__ / 100, __POCC__ % 100, 0) +#endif + +#if defined(JSON_HEDLEY_PELLES_VERSION_CHECK) +#undef JSON_HEDLEY_PELLES_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_PELLES_VERSION) +#define JSON_HEDLEY_PELLES_VERSION_CHECK(major, minor, patch) \ + (JSON_HEDLEY_PELLES_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else +#define JSON_HEDLEY_PELLES_VERSION_CHECK(major, minor, patch) (0) +#endif + +#if defined(JSON_HEDLEY_GCC_VERSION) +#undef JSON_HEDLEY_GCC_VERSION +#endif +#if defined(JSON_HEDLEY_GNUC_VERSION) && !defined(__clang__) && !defined(JSON_HEDLEY_INTEL_VERSION) && \ + !defined(JSON_HEDLEY_PGI_VERSION) && !defined(JSON_HEDLEY_ARM_VERSION) && !defined(JSON_HEDLEY_TI_VERSION) && \ + !defined(__COMPCERT__) +#define JSON_HEDLEY_GCC_VERSION JSON_HEDLEY_GNUC_VERSION +#endif + +#if defined(JSON_HEDLEY_GCC_VERSION_CHECK) +#undef JSON_HEDLEY_GCC_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_GCC_VERSION) +#define JSON_HEDLEY_GCC_VERSION_CHECK(major, minor, patch) \ + (JSON_HEDLEY_GCC_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else +#define JSON_HEDLEY_GCC_VERSION_CHECK(major, minor, patch) (0) +#endif + +#if defined(JSON_HEDLEY_HAS_ATTRIBUTE) +#undef JSON_HEDLEY_HAS_ATTRIBUTE +#endif +#if defined(__has_attribute) +#define JSON_HEDLEY_HAS_ATTRIBUTE(attribute) __has_attribute(attribute) +#else +#define JSON_HEDLEY_HAS_ATTRIBUTE(attribute) (0) +#endif + +#if defined(JSON_HEDLEY_GNUC_HAS_ATTRIBUTE) +#undef JSON_HEDLEY_GNUC_HAS_ATTRIBUTE +#endif +#if defined(__has_attribute) +#define JSON_HEDLEY_GNUC_HAS_ATTRIBUTE(attribute, major, minor, patch) __has_attribute(attribute) +#else +#define JSON_HEDLEY_GNUC_HAS_ATTRIBUTE(attribute, major, minor, patch) \ + JSON_HEDLEY_GNUC_VERSION_CHECK(major, minor, patch) +#endif + +#if defined(JSON_HEDLEY_GCC_HAS_ATTRIBUTE) +#undef JSON_HEDLEY_GCC_HAS_ATTRIBUTE +#endif +#if defined(__has_attribute) +#define JSON_HEDLEY_GCC_HAS_ATTRIBUTE(attribute, major, minor, patch) __has_attribute(attribute) +#else +#define JSON_HEDLEY_GCC_HAS_ATTRIBUTE(attribute, major, minor, patch) JSON_HEDLEY_GCC_VERSION_CHECK(major, minor, patch) +#endif + +#if defined(JSON_HEDLEY_HAS_CPP_ATTRIBUTE) +#undef JSON_HEDLEY_HAS_CPP_ATTRIBUTE +#endif +#if defined(__has_cpp_attribute) && defined(__cplusplus) +#define JSON_HEDLEY_HAS_CPP_ATTRIBUTE(attribute) __has_cpp_attribute(attribute) +#else +#define JSON_HEDLEY_HAS_CPP_ATTRIBUTE(attribute) (0) +#endif + +#if defined(JSON_HEDLEY_GNUC_HAS_CPP_ATTRIBUTE) +#undef JSON_HEDLEY_GNUC_HAS_CPP_ATTRIBUTE +#endif +#if defined(__has_cpp_attribute) && defined(__cplusplus) +#define JSON_HEDLEY_GNUC_HAS_CPP_ATTRIBUTE(attribute, major, minor, patch) __has_cpp_attribute(attribute) +#else +#define JSON_HEDLEY_GNUC_HAS_CPP_ATTRIBUTE(attribute, major, minor, patch) \ + JSON_HEDLEY_GNUC_VERSION_CHECK(major, minor, patch) +#endif + +#if defined(JSON_HEDLEY_GCC_HAS_CPP_ATTRIBUTE) +#undef JSON_HEDLEY_GCC_HAS_CPP_ATTRIBUTE +#endif +#if defined(__has_cpp_attribute) && defined(__cplusplus) +#define JSON_HEDLEY_GCC_HAS_CPP_ATTRIBUTE(attribute, major, minor, patch) __has_cpp_attribute(attribute) +#else +#define JSON_HEDLEY_GCC_HAS_CPP_ATTRIBUTE(attribute, major, minor, patch) \ + JSON_HEDLEY_GCC_VERSION_CHECK(major, minor, patch) +#endif + +#if defined(JSON_HEDLEY_HAS_BUILTIN) +#undef JSON_HEDLEY_HAS_BUILTIN +#endif +#if defined(__has_builtin) +#define JSON_HEDLEY_HAS_BUILTIN(builtin) __has_builtin(builtin) +#else +#define JSON_HEDLEY_HAS_BUILTIN(builtin) (0) +#endif + +#if defined(JSON_HEDLEY_GNUC_HAS_BUILTIN) +#undef JSON_HEDLEY_GNUC_HAS_BUILTIN +#endif +#if defined(__has_builtin) +#define JSON_HEDLEY_GNUC_HAS_BUILTIN(builtin, major, minor, patch) __has_builtin(builtin) +#else +#define JSON_HEDLEY_GNUC_HAS_BUILTIN(builtin, major, minor, patch) JSON_HEDLEY_GNUC_VERSION_CHECK(major, minor, patch) +#endif + +#if defined(JSON_HEDLEY_GCC_HAS_BUILTIN) +#undef JSON_HEDLEY_GCC_HAS_BUILTIN +#endif +#if defined(__has_builtin) +#define JSON_HEDLEY_GCC_HAS_BUILTIN(builtin, major, minor, patch) __has_builtin(builtin) +#else +#define JSON_HEDLEY_GCC_HAS_BUILTIN(builtin, major, minor, patch) JSON_HEDLEY_GCC_VERSION_CHECK(major, minor, patch) +#endif + +#if defined(JSON_HEDLEY_HAS_FEATURE) +#undef JSON_HEDLEY_HAS_FEATURE +#endif +#if defined(__has_feature) +#define JSON_HEDLEY_HAS_FEATURE(feature) __has_feature(feature) +#else +#define JSON_HEDLEY_HAS_FEATURE(feature) (0) +#endif + +#if defined(JSON_HEDLEY_GNUC_HAS_FEATURE) +#undef JSON_HEDLEY_GNUC_HAS_FEATURE +#endif +#if defined(__has_feature) +#define JSON_HEDLEY_GNUC_HAS_FEATURE(feature, major, minor, patch) __has_feature(feature) +#else +#define JSON_HEDLEY_GNUC_HAS_FEATURE(feature, major, minor, patch) JSON_HEDLEY_GNUC_VERSION_CHECK(major, minor, patch) +#endif + +#if defined(JSON_HEDLEY_GCC_HAS_FEATURE) +#undef JSON_HEDLEY_GCC_HAS_FEATURE +#endif +#if defined(__has_feature) +#define JSON_HEDLEY_GCC_HAS_FEATURE(feature, major, minor, patch) __has_feature(feature) +#else +#define JSON_HEDLEY_GCC_HAS_FEATURE(feature, major, minor, patch) JSON_HEDLEY_GCC_VERSION_CHECK(major, minor, patch) +#endif + +#if defined(JSON_HEDLEY_HAS_EXTENSION) +#undef JSON_HEDLEY_HAS_EXTENSION +#endif +#if defined(__has_extension) +#define JSON_HEDLEY_HAS_EXTENSION(extension) __has_extension(extension) +#else +#define JSON_HEDLEY_HAS_EXTENSION(extension) (0) +#endif + +#if defined(JSON_HEDLEY_GNUC_HAS_EXTENSION) +#undef JSON_HEDLEY_GNUC_HAS_EXTENSION +#endif +#if defined(__has_extension) +#define JSON_HEDLEY_GNUC_HAS_EXTENSION(extension, major, minor, patch) __has_extension(extension) +#else +#define JSON_HEDLEY_GNUC_HAS_EXTENSION(extension, major, minor, patch) \ + JSON_HEDLEY_GNUC_VERSION_CHECK(major, minor, patch) +#endif + +#if defined(JSON_HEDLEY_GCC_HAS_EXTENSION) +#undef JSON_HEDLEY_GCC_HAS_EXTENSION +#endif +#if defined(__has_extension) +#define JSON_HEDLEY_GCC_HAS_EXTENSION(extension, major, minor, patch) __has_extension(extension) +#else +#define JSON_HEDLEY_GCC_HAS_EXTENSION(extension, major, minor, patch) JSON_HEDLEY_GCC_VERSION_CHECK(major, minor, patch) +#endif + +#if defined(JSON_HEDLEY_HAS_DECLSPEC_ATTRIBUTE) +#undef JSON_HEDLEY_HAS_DECLSPEC_ATTRIBUTE +#endif +#if defined(__has_declspec_attribute) +#define JSON_HEDLEY_HAS_DECLSPEC_ATTRIBUTE(attribute) __has_declspec_attribute(attribute) +#else +#define JSON_HEDLEY_HAS_DECLSPEC_ATTRIBUTE(attribute) (0) +#endif + +#if defined(JSON_HEDLEY_GNUC_HAS_DECLSPEC_ATTRIBUTE) +#undef JSON_HEDLEY_GNUC_HAS_DECLSPEC_ATTRIBUTE +#endif +#if defined(__has_declspec_attribute) +#define JSON_HEDLEY_GNUC_HAS_DECLSPEC_ATTRIBUTE(attribute, major, minor, patch) __has_declspec_attribute(attribute) +#else +#define JSON_HEDLEY_GNUC_HAS_DECLSPEC_ATTRIBUTE(attribute, major, minor, patch) \ + JSON_HEDLEY_GNUC_VERSION_CHECK(major, minor, patch) +#endif + +#if defined(JSON_HEDLEY_GCC_HAS_DECLSPEC_ATTRIBUTE) +#undef JSON_HEDLEY_GCC_HAS_DECLSPEC_ATTRIBUTE +#endif +#if defined(__has_declspec_attribute) +#define JSON_HEDLEY_GCC_HAS_DECLSPEC_ATTRIBUTE(attribute, major, minor, patch) __has_declspec_attribute(attribute) +#else +#define JSON_HEDLEY_GCC_HAS_DECLSPEC_ATTRIBUTE(attribute, major, minor, patch) \ + JSON_HEDLEY_GCC_VERSION_CHECK(major, minor, patch) +#endif + +#if defined(JSON_HEDLEY_HAS_WARNING) +#undef JSON_HEDLEY_HAS_WARNING +#endif +#if defined(__has_warning) +#define JSON_HEDLEY_HAS_WARNING(warning) __has_warning(warning) +#else +#define JSON_HEDLEY_HAS_WARNING(warning) (0) +#endif + +#if defined(JSON_HEDLEY_GNUC_HAS_WARNING) +#undef JSON_HEDLEY_GNUC_HAS_WARNING +#endif +#if defined(__has_warning) +#define JSON_HEDLEY_GNUC_HAS_WARNING(warning, major, minor, patch) __has_warning(warning) +#else +#define JSON_HEDLEY_GNUC_HAS_WARNING(warning, major, minor, patch) JSON_HEDLEY_GNUC_VERSION_CHECK(major, minor, patch) +#endif + +#if defined(JSON_HEDLEY_GCC_HAS_WARNING) +#undef JSON_HEDLEY_GCC_HAS_WARNING +#endif +#if defined(__has_warning) +#define JSON_HEDLEY_GCC_HAS_WARNING(warning, major, minor, patch) __has_warning(warning) +#else +#define JSON_HEDLEY_GCC_HAS_WARNING(warning, major, minor, patch) JSON_HEDLEY_GCC_VERSION_CHECK(major, minor, patch) +#endif + +#if (defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L)) || defined(__clang__) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(3, 0, 0) || JSON_HEDLEY_INTEL_VERSION_CHECK(13, 0, 0) || \ + JSON_HEDLEY_IAR_VERSION_CHECK(8, 0, 0) || JSON_HEDLEY_PGI_VERSION_CHECK(18, 4, 0) || \ + JSON_HEDLEY_ARM_VERSION_CHECK(4, 1, 0) || JSON_HEDLEY_TI_VERSION_CHECK(6, 0, 0) || \ + JSON_HEDLEY_CRAY_VERSION_CHECK(5, 0, 0) || JSON_HEDLEY_TINYC_VERSION_CHECK(0, 9, 17) || \ + JSON_HEDLEY_SUNPRO_VERSION_CHECK(8, 0, 0) || \ + (JSON_HEDLEY_IBM_VERSION_CHECK(10, 1, 0) && defined(__C99_PRAGMA_OPERATOR)) +#define JSON_HEDLEY_PRAGMA(value) _Pragma(#value) +#elif JSON_HEDLEY_MSVC_VERSION_CHECK(15, 0, 0) +#define JSON_HEDLEY_PRAGMA(value) __pragma(value) +#else +#define JSON_HEDLEY_PRAGMA(value) +#endif + +#if defined(JSON_HEDLEY_DIAGNOSTIC_PUSH) +#undef JSON_HEDLEY_DIAGNOSTIC_PUSH +#endif +#if defined(JSON_HEDLEY_DIAGNOSTIC_POP) +#undef JSON_HEDLEY_DIAGNOSTIC_POP +#endif +#if defined(__clang__) +#define JSON_HEDLEY_DIAGNOSTIC_PUSH _Pragma("clang diagnostic push") +#define JSON_HEDLEY_DIAGNOSTIC_POP _Pragma("clang diagnostic pop") +#elif JSON_HEDLEY_INTEL_VERSION_CHECK(13, 0, 0) +#define JSON_HEDLEY_DIAGNOSTIC_PUSH _Pragma("warning(push)") +#define JSON_HEDLEY_DIAGNOSTIC_POP _Pragma("warning(pop)") +#elif JSON_HEDLEY_GCC_VERSION_CHECK(4, 6, 0) +#define JSON_HEDLEY_DIAGNOSTIC_PUSH _Pragma("GCC diagnostic push") +#define JSON_HEDLEY_DIAGNOSTIC_POP _Pragma("GCC diagnostic pop") +#elif JSON_HEDLEY_MSVC_VERSION_CHECK(15, 0, 0) +#define JSON_HEDLEY_DIAGNOSTIC_PUSH __pragma(warning(push)) +#define JSON_HEDLEY_DIAGNOSTIC_POP __pragma(warning(pop)) +#elif JSON_HEDLEY_ARM_VERSION_CHECK(5, 6, 0) +#define JSON_HEDLEY_DIAGNOSTIC_PUSH _Pragma("push") +#define JSON_HEDLEY_DIAGNOSTIC_POP _Pragma("pop") +#elif JSON_HEDLEY_TI_VERSION_CHECK(8, 1, 0) +#define JSON_HEDLEY_DIAGNOSTIC_PUSH _Pragma("diag_push") +#define JSON_HEDLEY_DIAGNOSTIC_POP _Pragma("diag_pop") +#elif JSON_HEDLEY_PELLES_VERSION_CHECK(2, 90, 0) +#define JSON_HEDLEY_DIAGNOSTIC_PUSH _Pragma("warning(push)") +#define JSON_HEDLEY_DIAGNOSTIC_POP _Pragma("warning(pop)") +#else +#define JSON_HEDLEY_DIAGNOSTIC_PUSH +#define JSON_HEDLEY_DIAGNOSTIC_POP +#endif + +#if defined(JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED) +#undef JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED +#endif +#if JSON_HEDLEY_HAS_WARNING("-Wdeprecated-declarations") +#define JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED _Pragma("clang diagnostic ignored \"-Wdeprecated-declarations\"") +#elif JSON_HEDLEY_INTEL_VERSION_CHECK(13, 0, 0) +#define JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED _Pragma("warning(disable:1478 1786)") +#elif JSON_HEDLEY_PGI_VERSION_CHECK(17, 10, 0) +#define JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED _Pragma("diag_suppress 1215,1444") +#elif JSON_HEDLEY_GCC_VERSION_CHECK(4, 3, 0) +#define JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED _Pragma("GCC diagnostic ignored \"-Wdeprecated-declarations\"") +#elif JSON_HEDLEY_MSVC_VERSION_CHECK(15, 0, 0) +#define JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED __pragma(warning(disable : 4996)) +#elif JSON_HEDLEY_TI_VERSION_CHECK(8, 0, 0) +#define JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED _Pragma("diag_suppress 1291,1718") +#elif JSON_HEDLEY_SUNPRO_VERSION_CHECK(5, 13, 0) && !defined(__cplusplus) +#define JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED _Pragma("error_messages(off,E_DEPRECATED_ATT,E_DEPRECATED_ATT_MESS)") +#elif JSON_HEDLEY_SUNPRO_VERSION_CHECK(5, 13, 0) && defined(__cplusplus) +#define JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED _Pragma("error_messages(off,symdeprecated,symdeprecated2)") +#elif JSON_HEDLEY_IAR_VERSION_CHECK(8, 0, 0) +#define JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED _Pragma("diag_suppress=Pe1444,Pe1215") +#elif JSON_HEDLEY_PELLES_VERSION_CHECK(2, 90, 0) +#define JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED _Pragma("warn(disable:2241)") +#else +#define JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED +#endif + +#if defined(JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS) +#undef JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS +#endif +#if JSON_HEDLEY_HAS_WARNING("-Wunknown-pragmas") +#define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS _Pragma("clang diagnostic ignored \"-Wunknown-pragmas\"") +#elif JSON_HEDLEY_INTEL_VERSION_CHECK(13, 0, 0) +#define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS _Pragma("warning(disable:161)") +#elif JSON_HEDLEY_PGI_VERSION_CHECK(17, 10, 0) +#define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS _Pragma("diag_suppress 1675") +#elif JSON_HEDLEY_GCC_VERSION_CHECK(4, 3, 0) +#define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS _Pragma("GCC diagnostic ignored \"-Wunknown-pragmas\"") +#elif JSON_HEDLEY_MSVC_VERSION_CHECK(15, 0, 0) +#define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS __pragma(warning(disable : 4068)) +#elif JSON_HEDLEY_TI_VERSION_CHECK(8, 0, 0) +#define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS _Pragma("diag_suppress 163") +#elif JSON_HEDLEY_IAR_VERSION_CHECK(8, 0, 0) +#define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS _Pragma("diag_suppress=Pe161") +#else +#define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS +#endif + +#if defined(JSON_HEDLEY_DIAGNOSTIC_DISABLE_CAST_QUAL) +#undef JSON_HEDLEY_DIAGNOSTIC_DISABLE_CAST_QUAL +#endif +#if JSON_HEDLEY_HAS_WARNING("-Wcast-qual") +#define JSON_HEDLEY_DIAGNOSTIC_DISABLE_CAST_QUAL _Pragma("clang diagnostic ignored \"-Wcast-qual\"") +#elif JSON_HEDLEY_INTEL_VERSION_CHECK(13, 0, 0) +#define JSON_HEDLEY_DIAGNOSTIC_DISABLE_CAST_QUAL _Pragma("warning(disable:2203 2331)") +#elif JSON_HEDLEY_GCC_VERSION_CHECK(3, 0, 0) +#define JSON_HEDLEY_DIAGNOSTIC_DISABLE_CAST_QUAL _Pragma("GCC diagnostic ignored \"-Wcast-qual\"") +#else +#define JSON_HEDLEY_DIAGNOSTIC_DISABLE_CAST_QUAL +#endif + +#if defined(JSON_HEDLEY_DEPRECATED) +#undef JSON_HEDLEY_DEPRECATED +#endif +#if defined(JSON_HEDLEY_DEPRECATED_FOR) +#undef JSON_HEDLEY_DEPRECATED_FOR +#endif +#if defined(__cplusplus) && (__cplusplus >= 201402L) +#define JSON_HEDLEY_DEPRECATED(since) [[deprecated("Since " #since)]] +#define JSON_HEDLEY_DEPRECATED_FOR(since, replacement) [[deprecated("Since " #since "; use " #replacement)]] +#elif JSON_HEDLEY_HAS_EXTENSION(attribute_deprecated_with_message) || JSON_HEDLEY_GCC_VERSION_CHECK(4, 5, 0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13, 0, 0) || JSON_HEDLEY_ARM_VERSION_CHECK(5, 6, 0) || \ + JSON_HEDLEY_SUNPRO_VERSION_CHECK(5, 13, 0) || JSON_HEDLEY_PGI_VERSION_CHECK(17, 10, 0) || \ + JSON_HEDLEY_TI_VERSION_CHECK(8, 3, 0) +#define JSON_HEDLEY_DEPRECATED(since) __attribute__((__deprecated__("Since " #since))) +#define JSON_HEDLEY_DEPRECATED_FOR(since, replacement) \ + __attribute__((__deprecated__("Since " #since "; use " #replacement))) +#elif JSON_HEDLEY_HAS_ATTRIBUTE(deprecated) || JSON_HEDLEY_GCC_VERSION_CHECK(3, 1, 0) || \ + JSON_HEDLEY_ARM_VERSION_CHECK(4, 1, 0) || JSON_HEDLEY_TI_VERSION_CHECK(8, 0, 0) || \ + (JSON_HEDLEY_TI_VERSION_CHECK(7, 3, 0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) +#define JSON_HEDLEY_DEPRECATED(since) __attribute__((__deprecated__)) +#define JSON_HEDLEY_DEPRECATED_FOR(since, replacement) __attribute__((__deprecated__)) +#elif JSON_HEDLEY_MSVC_VERSION_CHECK(14, 0, 0) +#define JSON_HEDLEY_DEPRECATED(since) __declspec(deprecated("Since " #since)) +#define JSON_HEDLEY_DEPRECATED_FOR(since, replacement) __declspec(deprecated("Since " #since "; use " #replacement)) +#elif JSON_HEDLEY_MSVC_VERSION_CHECK(13, 10, 0) || JSON_HEDLEY_PELLES_VERSION_CHECK(6, 50, 0) +#define JSON_HEDLEY_DEPRECATED(since) _declspec(deprecated) +#define JSON_HEDLEY_DEPRECATED_FOR(since, replacement) __declspec(deprecated) +#elif JSON_HEDLEY_IAR_VERSION_CHECK(8, 0, 0) +#define JSON_HEDLEY_DEPRECATED(since) _Pragma("deprecated") +#define JSON_HEDLEY_DEPRECATED_FOR(since, replacement) _Pragma("deprecated") +#else +#define JSON_HEDLEY_DEPRECATED(since) +#define JSON_HEDLEY_DEPRECATED_FOR(since, replacement) +#endif + +#if defined(JSON_HEDLEY_UNAVAILABLE) +#undef JSON_HEDLEY_UNAVAILABLE +#endif +#if JSON_HEDLEY_HAS_ATTRIBUTE(warning) || JSON_HEDLEY_GCC_VERSION_CHECK(4, 3, 0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13, 0, 0) +#define JSON_HEDLEY_UNAVAILABLE(available_since) __attribute__((__warning__("Not available until " #available_since))) +#else +#define JSON_HEDLEY_UNAVAILABLE(available_since) +#endif + +#if defined(JSON_HEDLEY_WARN_UNUSED_RESULT) +#undef JSON_HEDLEY_WARN_UNUSED_RESULT +#endif +#if defined(__cplusplus) && (__cplusplus >= 201703L) +#define JSON_HEDLEY_WARN_UNUSED_RESULT [[nodiscard]] +#elif JSON_HEDLEY_HAS_ATTRIBUTE(warn_unused_result) || JSON_HEDLEY_GCC_VERSION_CHECK(3, 4, 0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13, 0, 0) || JSON_HEDLEY_TI_VERSION_CHECK(8, 0, 0) || \ + (JSON_HEDLEY_TI_VERSION_CHECK(7, 3, 0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + (JSON_HEDLEY_SUNPRO_VERSION_CHECK(5, 15, 0) && defined(__cplusplus)) || JSON_HEDLEY_PGI_VERSION_CHECK(17, 10, 0) +#define JSON_HEDLEY_WARN_UNUSED_RESULT __attribute__((__warn_unused_result__)) +#elif defined(_Check_return_) /* SAL */ +#define JSON_HEDLEY_WARN_UNUSED_RESULT _Check_return_ +#else +#define JSON_HEDLEY_WARN_UNUSED_RESULT +#endif + +#if defined(JSON_HEDLEY_SENTINEL) +#undef JSON_HEDLEY_SENTINEL +#endif +#if JSON_HEDLEY_HAS_ATTRIBUTE(sentinel) || JSON_HEDLEY_GCC_VERSION_CHECK(4, 0, 0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13, 0, 0) || JSON_HEDLEY_ARM_VERSION_CHECK(5, 4, 0) +#define JSON_HEDLEY_SENTINEL(position) __attribute__((__sentinel__(position))) +#else +#define JSON_HEDLEY_SENTINEL(position) +#endif + +#if defined(JSON_HEDLEY_NO_RETURN) +#undef JSON_HEDLEY_NO_RETURN +#endif +#if JSON_HEDLEY_IAR_VERSION_CHECK(8, 0, 0) +#define JSON_HEDLEY_NO_RETURN __noreturn +#elif JSON_HEDLEY_INTEL_VERSION_CHECK(13, 0, 0) +#define JSON_HEDLEY_NO_RETURN __attribute__((__noreturn__)) +#elif defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L +#define JSON_HEDLEY_NO_RETURN _Noreturn +#elif defined(__cplusplus) && (__cplusplus >= 201103L) +#define JSON_HEDLEY_NO_RETURN [[noreturn]] +#elif JSON_HEDLEY_HAS_ATTRIBUTE(noreturn) || JSON_HEDLEY_GCC_VERSION_CHECK(3, 2, 0) || \ + JSON_HEDLEY_SUNPRO_VERSION_CHECK(5, 11, 0) || JSON_HEDLEY_ARM_VERSION_CHECK(4, 1, 0) || \ + JSON_HEDLEY_IBM_VERSION_CHECK(10, 1, 0) || JSON_HEDLEY_TI_VERSION_CHECK(18, 0, 0) || \ + (JSON_HEDLEY_TI_VERSION_CHECK(17, 3, 0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) +#define JSON_HEDLEY_NO_RETURN __attribute__((__noreturn__)) +#elif JSON_HEDLEY_MSVC_VERSION_CHECK(13, 10, 0) +#define JSON_HEDLEY_NO_RETURN __declspec(noreturn) +#elif JSON_HEDLEY_TI_VERSION_CHECK(6, 0, 0) && defined(__cplusplus) +#define JSON_HEDLEY_NO_RETURN _Pragma("FUNC_NEVER_RETURNS;") +#elif JSON_HEDLEY_COMPCERT_VERSION_CHECK(3, 2, 0) +#define JSON_HEDLEY_NO_RETURN __attribute((noreturn)) +#elif JSON_HEDLEY_PELLES_VERSION_CHECK(9, 0, 0) +#define JSON_HEDLEY_NO_RETURN __declspec(noreturn) +#else +#define JSON_HEDLEY_NO_RETURN +#endif + +#if defined(JSON_HEDLEY_UNREACHABLE) +#undef JSON_HEDLEY_UNREACHABLE +#endif +#if defined(JSON_HEDLEY_UNREACHABLE_RETURN) +#undef JSON_HEDLEY_UNREACHABLE_RETURN +#endif +#if (JSON_HEDLEY_HAS_BUILTIN(__builtin_unreachable) && (!defined(JSON_HEDLEY_ARM_VERSION))) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(4, 5, 0) || JSON_HEDLEY_INTEL_VERSION_CHECK(13, 0, 0) || \ + JSON_HEDLEY_IBM_VERSION_CHECK(13, 1, 5) +#define JSON_HEDLEY_UNREACHABLE() __builtin_unreachable() +#elif JSON_HEDLEY_MSVC_VERSION_CHECK(13, 10, 0) +#define JSON_HEDLEY_UNREACHABLE() __assume(0) +#elif JSON_HEDLEY_TI_VERSION_CHECK(6, 0, 0) +#if defined(__cplusplus) +#define JSON_HEDLEY_UNREACHABLE() std::_nassert(0) +#else +#define JSON_HEDLEY_UNREACHABLE() _nassert(0) +#endif +#define JSON_HEDLEY_UNREACHABLE_RETURN(value) return value +#elif defined(EXIT_FAILURE) +#define JSON_HEDLEY_UNREACHABLE() abort() +#else +#define JSON_HEDLEY_UNREACHABLE() +#define JSON_HEDLEY_UNREACHABLE_RETURN(value) return value +#endif +#if !defined(JSON_HEDLEY_UNREACHABLE_RETURN) +#define JSON_HEDLEY_UNREACHABLE_RETURN(value) JSON_HEDLEY_UNREACHABLE() +#endif + +#if defined(JSON_HEDLEY_ASSUME) +#undef JSON_HEDLEY_ASSUME +#endif +#if JSON_HEDLEY_MSVC_VERSION_CHECK(13, 10, 0) || JSON_HEDLEY_INTEL_VERSION_CHECK(13, 0, 0) +#define JSON_HEDLEY_ASSUME(expr) __assume(expr) +#elif JSON_HEDLEY_HAS_BUILTIN(__builtin_assume) +#define JSON_HEDLEY_ASSUME(expr) __builtin_assume(expr) +#elif JSON_HEDLEY_TI_VERSION_CHECK(6, 0, 0) +#if defined(__cplusplus) +#define JSON_HEDLEY_ASSUME(expr) std::_nassert(expr) +#else +#define JSON_HEDLEY_ASSUME(expr) _nassert(expr) +#endif +#elif (JSON_HEDLEY_HAS_BUILTIN(__builtin_unreachable) && !defined(JSON_HEDLEY_ARM_VERSION)) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(4, 5, 0) || JSON_HEDLEY_INTEL_VERSION_CHECK(13, 0, 0) || \ + JSON_HEDLEY_IBM_VERSION_CHECK(13, 1, 5) +#define JSON_HEDLEY_ASSUME(expr) ((void)((expr) ? 1 : (__builtin_unreachable(), 1))) +#else +#define JSON_HEDLEY_ASSUME(expr) ((void)(expr)) +#endif + +JSON_HEDLEY_DIAGNOSTIC_PUSH +#if JSON_HEDLEY_HAS_WARNING("-Wvariadic-macros") || JSON_HEDLEY_GCC_VERSION_CHECK(4, 0, 0) +#if defined(__clang__) +#pragma clang diagnostic ignored "-Wvariadic-macros" +#elif defined(JSON_HEDLEY_GCC_VERSION) +#pragma GCC diagnostic ignored "-Wvariadic-macros" +#endif +#endif +#if defined(JSON_HEDLEY_NON_NULL) +#undef JSON_HEDLEY_NON_NULL +#endif +#if JSON_HEDLEY_HAS_ATTRIBUTE(nonnull) || JSON_HEDLEY_GCC_VERSION_CHECK(3, 3, 0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13, 0, 0) || JSON_HEDLEY_ARM_VERSION_CHECK(4, 1, 0) +#define JSON_HEDLEY_NON_NULL(...) __attribute__((__nonnull__(__VA_ARGS__))) +#else +#define JSON_HEDLEY_NON_NULL(...) +#endif +JSON_HEDLEY_DIAGNOSTIC_POP + +#if defined(JSON_HEDLEY_PRINTF_FORMAT) +#undef JSON_HEDLEY_PRINTF_FORMAT +#endif +#if defined(__MINGW32__) && JSON_HEDLEY_GCC_HAS_ATTRIBUTE(format, 4, 4, 0) && !defined(__USE_MINGW_ANSI_STDIO) +#define JSON_HEDLEY_PRINTF_FORMAT(string_idx, first_to_check) \ + __attribute__((__format__(ms_printf, string_idx, first_to_check))) +#elif defined(__MINGW32__) && JSON_HEDLEY_GCC_HAS_ATTRIBUTE(format, 4, 4, 0) && defined(__USE_MINGW_ANSI_STDIO) +#define JSON_HEDLEY_PRINTF_FORMAT(string_idx, first_to_check) \ + __attribute__((__format__(gnu_printf, string_idx, first_to_check))) +#elif JSON_HEDLEY_HAS_ATTRIBUTE(format) || JSON_HEDLEY_GCC_VERSION_CHECK(3, 1, 0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13, 0, 0) || JSON_HEDLEY_ARM_VERSION_CHECK(5, 6, 0) || \ + JSON_HEDLEY_IBM_VERSION_CHECK(10, 1, 0) || JSON_HEDLEY_TI_VERSION_CHECK(8, 0, 0) || \ + (JSON_HEDLEY_TI_VERSION_CHECK(7, 3, 0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) +#define JSON_HEDLEY_PRINTF_FORMAT(string_idx, first_to_check) \ + __attribute__((__format__(__printf__, string_idx, first_to_check))) +#elif JSON_HEDLEY_PELLES_VERSION_CHECK(6, 0, 0) +#define JSON_HEDLEY_PRINTF_FORMAT(string_idx, first_to_check) __declspec(vaformat(printf, string_idx, first_to_check)) +#else +#define JSON_HEDLEY_PRINTF_FORMAT(string_idx, first_to_check) +#endif + +#if defined(JSON_HEDLEY_CONSTEXPR) +#undef JSON_HEDLEY_CONSTEXPR +#endif +#if defined(__cplusplus) +#if __cplusplus >= 201103L +#define JSON_HEDLEY_CONSTEXPR constexpr +#endif +#endif +#if !defined(JSON_HEDLEY_CONSTEXPR) +#define JSON_HEDLEY_CONSTEXPR +#endif + +#if defined(JSON_HEDLEY_PREDICT) +#undef JSON_HEDLEY_PREDICT +#endif +#if defined(JSON_HEDLEY_LIKELY) +#undef JSON_HEDLEY_LIKELY +#endif +#if defined(JSON_HEDLEY_UNLIKELY) +#undef JSON_HEDLEY_UNLIKELY +#endif +#if defined(JSON_HEDLEY_UNPREDICTABLE) +#undef JSON_HEDLEY_UNPREDICTABLE +#endif +#if JSON_HEDLEY_HAS_BUILTIN(__builtin_unpredictable) +#define JSON_HEDLEY_UNPREDICTABLE(expr) __builtin_unpredictable(!!(expr)) +#endif +#if JSON_HEDLEY_HAS_BUILTIN(__builtin_expect_with_probability) || JSON_HEDLEY_GCC_VERSION_CHECK(9, 0, 0) +#define JSON_HEDLEY_PREDICT(expr, value, probability) __builtin_expect_with_probability(expr, value, probability) +#define JSON_HEDLEY_PREDICT_TRUE(expr, probability) __builtin_expect_with_probability(!!(expr), 1, probability) +#define JSON_HEDLEY_PREDICT_FALSE(expr, probability) __builtin_expect_with_probability(!!(expr), 0, probability) +#define JSON_HEDLEY_LIKELY(expr) __builtin_expect(!!(expr), 1) +#define JSON_HEDLEY_UNLIKELY(expr) __builtin_expect(!!(expr), 0) +#if !defined(JSON_HEDLEY_BUILTIN_UNPREDICTABLE) +#define JSON_HEDLEY_BUILTIN_UNPREDICTABLE(expr) __builtin_expect_with_probability(!!(expr), 1, 0.5) +#endif +#elif JSON_HEDLEY_HAS_BUILTIN(__builtin_expect) || JSON_HEDLEY_GCC_VERSION_CHECK(3, 0, 0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13, 0, 0) || \ + (JSON_HEDLEY_SUNPRO_VERSION_CHECK(5, 15, 0) && defined(__cplusplus)) || JSON_HEDLEY_ARM_VERSION_CHECK(4, 1, 0) || \ + JSON_HEDLEY_IBM_VERSION_CHECK(10, 1, 0) || JSON_HEDLEY_TI_VERSION_CHECK(6, 1, 0) || \ + JSON_HEDLEY_TINYC_VERSION_CHECK(0, 9, 27) +#define JSON_HEDLEY_PREDICT(expr, expected, probability) \ + (((probability) >= 0.9) ? __builtin_expect(!!(expr), (expected)) : (((void)(expected)), !!(expr))) +#define JSON_HEDLEY_PREDICT_TRUE(expr, probability) \ + (__extension__({ \ + JSON_HEDLEY_CONSTEXPR double hedley_probability_ = (probability); \ + ((hedley_probability_ >= 0.9) ? __builtin_expect(!!(expr), 1) \ + : ((hedley_probability_ <= 0.1) ? __builtin_expect(!!(expr), 0) : !!(expr))); \ + })) +#define JSON_HEDLEY_PREDICT_FALSE(expr, probability) \ + (__extension__({ \ + JSON_HEDLEY_CONSTEXPR double hedley_probability_ = (probability); \ + ((hedley_probability_ >= 0.9) ? __builtin_expect(!!(expr), 0) \ + : ((hedley_probability_ <= 0.1) ? __builtin_expect(!!(expr), 1) : !!(expr))); \ + })) +#define JSON_HEDLEY_LIKELY(expr) __builtin_expect(!!(expr), 1) +#define JSON_HEDLEY_UNLIKELY(expr) __builtin_expect(!!(expr), 0) +#else +#define JSON_HEDLEY_PREDICT(expr, expected, probability) (((void)(expected)), !!(expr)) +#define JSON_HEDLEY_PREDICT_TRUE(expr, probability) (!!(expr)) +#define JSON_HEDLEY_PREDICT_FALSE(expr, probability) (!!(expr)) +#define JSON_HEDLEY_LIKELY(expr) (!!(expr)) +#define JSON_HEDLEY_UNLIKELY(expr) (!!(expr)) +#endif +#if !defined(JSON_HEDLEY_UNPREDICTABLE) +#define JSON_HEDLEY_UNPREDICTABLE(expr) JSON_HEDLEY_PREDICT(expr, 1, 0.5) +#endif + +#if defined(JSON_HEDLEY_MALLOC) +#undef JSON_HEDLEY_MALLOC +#endif +#if JSON_HEDLEY_HAS_ATTRIBUTE(malloc) || JSON_HEDLEY_GCC_VERSION_CHECK(3, 1, 0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13, 0, 0) || JSON_HEDLEY_SUNPRO_VERSION_CHECK(5, 11, 0) || \ + JSON_HEDLEY_ARM_VERSION_CHECK(4, 1, 0) || JSON_HEDLEY_IBM_VERSION_CHECK(12, 1, 0) || \ + JSON_HEDLEY_TI_VERSION_CHECK(8, 0, 0) || \ + (JSON_HEDLEY_TI_VERSION_CHECK(7, 3, 0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) +#define JSON_HEDLEY_MALLOC __attribute__((__malloc__)) +#elif JSON_HEDLEY_MSVC_VERSION_CHECK(14, 0, 0) +#define JSON_HEDLEY_MALLOC __declspec(restrict) +#else +#define JSON_HEDLEY_MALLOC +#endif + +#if defined(JSON_HEDLEY_PURE) +#undef JSON_HEDLEY_PURE +#endif +#if JSON_HEDLEY_HAS_ATTRIBUTE(pure) || JSON_HEDLEY_GCC_VERSION_CHECK(2, 96, 0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13, 0, 0) || JSON_HEDLEY_SUNPRO_VERSION_CHECK(5, 11, 0) || \ + JSON_HEDLEY_ARM_VERSION_CHECK(4, 1, 0) || JSON_HEDLEY_IBM_VERSION_CHECK(10, 1, 0) || \ + JSON_HEDLEY_TI_VERSION_CHECK(8, 0, 0) || \ + (JSON_HEDLEY_TI_VERSION_CHECK(7, 3, 0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_PGI_VERSION_CHECK(17, 10, 0) +#define JSON_HEDLEY_PURE __attribute__((__pure__)) +#elif JSON_HEDLEY_TI_VERSION_CHECK(6, 0, 0) && defined(__cplusplus) +#define JSON_HEDLEY_PURE _Pragma("FUNC_IS_PURE;") +#else +#define JSON_HEDLEY_PURE +#endif + +#if defined(JSON_HEDLEY_CONST) +#undef JSON_HEDLEY_CONST +#endif +#if JSON_HEDLEY_HAS_ATTRIBUTE(const) || JSON_HEDLEY_GCC_VERSION_CHECK(2, 5, 0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13, 0, 0) || JSON_HEDLEY_SUNPRO_VERSION_CHECK(5, 11, 0) || \ + JSON_HEDLEY_ARM_VERSION_CHECK(4, 1, 0) || JSON_HEDLEY_IBM_VERSION_CHECK(10, 1, 0) || \ + JSON_HEDLEY_TI_VERSION_CHECK(8, 0, 0) || \ + (JSON_HEDLEY_TI_VERSION_CHECK(7, 3, 0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_PGI_VERSION_CHECK(17, 10, 0) +#define JSON_HEDLEY_CONST __attribute__((__const__)) +#else +#define JSON_HEDLEY_CONST JSON_HEDLEY_PURE +#endif + +#if defined(JSON_HEDLEY_RESTRICT) +#undef JSON_HEDLEY_RESTRICT +#endif +#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) && !defined(__cplusplus) +#define JSON_HEDLEY_RESTRICT restrict +#elif JSON_HEDLEY_GCC_VERSION_CHECK(3, 1, 0) || JSON_HEDLEY_MSVC_VERSION_CHECK(14, 0, 0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13, 0, 0) || JSON_HEDLEY_ARM_VERSION_CHECK(4, 1, 0) || \ + JSON_HEDLEY_IBM_VERSION_CHECK(10, 1, 0) || JSON_HEDLEY_PGI_VERSION_CHECK(17, 10, 0) || \ + JSON_HEDLEY_TI_VERSION_CHECK(8, 0, 0) || (JSON_HEDLEY_SUNPRO_VERSION_CHECK(5, 14, 0) && defined(__cplusplus)) || \ + JSON_HEDLEY_IAR_VERSION_CHECK(8, 0, 0) || defined(__clang__) +#define JSON_HEDLEY_RESTRICT __restrict +#elif JSON_HEDLEY_SUNPRO_VERSION_CHECK(5, 3, 0) && !defined(__cplusplus) +#define JSON_HEDLEY_RESTRICT _Restrict +#else +#define JSON_HEDLEY_RESTRICT +#endif + +#if defined(JSON_HEDLEY_INLINE) +#undef JSON_HEDLEY_INLINE +#endif +#if (defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L)) || (defined(__cplusplus) && (__cplusplus >= 199711L)) +#define JSON_HEDLEY_INLINE inline +#elif defined(JSON_HEDLEY_GCC_VERSION) || JSON_HEDLEY_ARM_VERSION_CHECK(6, 2, 0) +#define JSON_HEDLEY_INLINE __inline__ +#elif JSON_HEDLEY_MSVC_VERSION_CHECK(12, 0, 0) || JSON_HEDLEY_ARM_VERSION_CHECK(4, 1, 0) || \ + JSON_HEDLEY_TI_VERSION_CHECK(8, 0, 0) +#define JSON_HEDLEY_INLINE __inline +#else +#define JSON_HEDLEY_INLINE +#endif + +#if defined(JSON_HEDLEY_ALWAYS_INLINE) +#undef JSON_HEDLEY_ALWAYS_INLINE +#endif +#if JSON_HEDLEY_HAS_ATTRIBUTE(always_inline) || JSON_HEDLEY_GCC_VERSION_CHECK(4, 0, 0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13, 0, 0) || JSON_HEDLEY_SUNPRO_VERSION_CHECK(5, 11, 0) || \ + JSON_HEDLEY_ARM_VERSION_CHECK(4, 1, 0) || JSON_HEDLEY_IBM_VERSION_CHECK(10, 1, 0) || \ + JSON_HEDLEY_TI_VERSION_CHECK(8, 0, 0) || \ + (JSON_HEDLEY_TI_VERSION_CHECK(7, 3, 0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) +#define JSON_HEDLEY_ALWAYS_INLINE __attribute__((__always_inline__)) JSON_HEDLEY_INLINE +#elif JSON_HEDLEY_MSVC_VERSION_CHECK(12, 0, 0) +#define JSON_HEDLEY_ALWAYS_INLINE __forceinline +#elif JSON_HEDLEY_TI_VERSION_CHECK(7, 0, 0) && defined(__cplusplus) +#define JSON_HEDLEY_ALWAYS_INLINE _Pragma("FUNC_ALWAYS_INLINE;") +#elif JSON_HEDLEY_IAR_VERSION_CHECK(8, 0, 0) +#define JSON_HEDLEY_ALWAYS_INLINE _Pragma("inline=forced") +#else +#define JSON_HEDLEY_ALWAYS_INLINE JSON_HEDLEY_INLINE +#endif + +#if defined(JSON_HEDLEY_NEVER_INLINE) +#undef JSON_HEDLEY_NEVER_INLINE +#endif +#if JSON_HEDLEY_HAS_ATTRIBUTE(noinline) || JSON_HEDLEY_GCC_VERSION_CHECK(4, 0, 0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13, 0, 0) || JSON_HEDLEY_SUNPRO_VERSION_CHECK(5, 11, 0) || \ + JSON_HEDLEY_ARM_VERSION_CHECK(4, 1, 0) || JSON_HEDLEY_IBM_VERSION_CHECK(10, 1, 0) || \ + JSON_HEDLEY_TI_VERSION_CHECK(8, 0, 0) || \ + (JSON_HEDLEY_TI_VERSION_CHECK(7, 3, 0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) +#define JSON_HEDLEY_NEVER_INLINE __attribute__((__noinline__)) +#elif JSON_HEDLEY_MSVC_VERSION_CHECK(13, 10, 0) +#define JSON_HEDLEY_NEVER_INLINE __declspec(noinline) +#elif JSON_HEDLEY_PGI_VERSION_CHECK(10, 2, 0) +#define JSON_HEDLEY_NEVER_INLINE _Pragma("noinline") +#elif JSON_HEDLEY_TI_VERSION_CHECK(6, 0, 0) && defined(__cplusplus) +#define JSON_HEDLEY_NEVER_INLINE _Pragma("FUNC_CANNOT_INLINE;") +#elif JSON_HEDLEY_IAR_VERSION_CHECK(8, 0, 0) +#define JSON_HEDLEY_NEVER_INLINE _Pragma("inline=never") +#elif JSON_HEDLEY_COMPCERT_VERSION_CHECK(3, 2, 0) +#define JSON_HEDLEY_NEVER_INLINE __attribute((noinline)) +#elif JSON_HEDLEY_PELLES_VERSION_CHECK(9, 0, 0) +#define JSON_HEDLEY_NEVER_INLINE __declspec(noinline) +#else +#define JSON_HEDLEY_NEVER_INLINE +#endif + +#if defined(JSON_HEDLEY_PRIVATE) +#undef JSON_HEDLEY_PRIVATE +#endif +#if defined(JSON_HEDLEY_PUBLIC) +#undef JSON_HEDLEY_PUBLIC +#endif +#if defined(JSON_HEDLEY_IMPORT) +#undef JSON_HEDLEY_IMPORT +#endif +#if defined(_WIN32) || defined(__CYGWIN__) +#define JSON_HEDLEY_PRIVATE +#define JSON_HEDLEY_PUBLIC __declspec(dllexport) +#define JSON_HEDLEY_IMPORT __declspec(dllimport) +#else +#if JSON_HEDLEY_HAS_ATTRIBUTE(visibility) || JSON_HEDLEY_GCC_VERSION_CHECK(3, 3, 0) || \ + JSON_HEDLEY_SUNPRO_VERSION_CHECK(5, 11, 0) || JSON_HEDLEY_INTEL_VERSION_CHECK(13, 0, 0) || \ + JSON_HEDLEY_ARM_VERSION_CHECK(4, 1, 0) || JSON_HEDLEY_IBM_VERSION_CHECK(13, 1, 0) || \ + JSON_HEDLEY_TI_VERSION_CHECK(8, 0, 0) || \ + (JSON_HEDLEY_TI_VERSION_CHECK(7, 3, 0) && defined(__TI_EABI__) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) +#define JSON_HEDLEY_PRIVATE __attribute__((__visibility__("hidden"))) +#define JSON_HEDLEY_PUBLIC __attribute__((__visibility__("default"))) +#else +#define JSON_HEDLEY_PRIVATE +#define JSON_HEDLEY_PUBLIC +#endif +#define JSON_HEDLEY_IMPORT extern +#endif + +#if defined(JSON_HEDLEY_NO_THROW) +#undef JSON_HEDLEY_NO_THROW +#endif +#if JSON_HEDLEY_HAS_ATTRIBUTE(nothrow) || JSON_HEDLEY_GCC_VERSION_CHECK(3, 3, 0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13, 0, 0) +#define JSON_HEDLEY_NO_THROW __attribute__((__nothrow__)) +#elif JSON_HEDLEY_MSVC_VERSION_CHECK(13, 1, 0) || JSON_HEDLEY_ARM_VERSION_CHECK(4, 1, 0) +#define JSON_HEDLEY_NO_THROW __declspec(nothrow) +#else +#define JSON_HEDLEY_NO_THROW +#endif + +#if defined(JSON_HEDLEY_FALL_THROUGH) +#undef JSON_HEDLEY_FALL_THROUGH +#endif +#if defined(__cplusplus) && (!defined(JSON_HEDLEY_SUNPRO_VERSION) || JSON_HEDLEY_SUNPRO_VERSION_CHECK(5, 15, 0)) && \ + !defined(JSON_HEDLEY_PGI_VERSION) +#if (__cplusplus >= 201703L) || ((__cplusplus >= 201103L) && JSON_HEDLEY_HAS_CPP_ATTRIBUTE(fallthrough)) +#define JSON_HEDLEY_FALL_THROUGH [[fallthrough]] +#elif (__cplusplus >= 201103L) && JSON_HEDLEY_HAS_CPP_ATTRIBUTE(clang::fallthrough) +#define JSON_HEDLEY_FALL_THROUGH [[clang::fallthrough]] +#elif (__cplusplus >= 201103L) && JSON_HEDLEY_GCC_VERSION_CHECK(7, 0, 0) +#define JSON_HEDLEY_FALL_THROUGH [[gnu::fallthrough]] +#endif +#endif +#if !defined(JSON_HEDLEY_FALL_THROUGH) +#if JSON_HEDLEY_GNUC_HAS_ATTRIBUTE(fallthrough, 7, 0, 0) && !defined(JSON_HEDLEY_PGI_VERSION) +#define JSON_HEDLEY_FALL_THROUGH __attribute__((__fallthrough__)) +#elif defined(__fallthrough) /* SAL */ +#define JSON_HEDLEY_FALL_THROUGH __fallthrough +#else +#define JSON_HEDLEY_FALL_THROUGH +#endif +#endif + +#if defined(JSON_HEDLEY_RETURNS_NON_NULL) +#undef JSON_HEDLEY_RETURNS_NON_NULL +#endif +#if JSON_HEDLEY_HAS_ATTRIBUTE(returns_nonnull) || JSON_HEDLEY_GCC_VERSION_CHECK(4, 9, 0) +#define JSON_HEDLEY_RETURNS_NON_NULL __attribute__((__returns_nonnull__)) +#elif defined(_Ret_notnull_) /* SAL */ +#define JSON_HEDLEY_RETURNS_NON_NULL _Ret_notnull_ +#else +#define JSON_HEDLEY_RETURNS_NON_NULL +#endif + +#if defined(JSON_HEDLEY_ARRAY_PARAM) +#undef JSON_HEDLEY_ARRAY_PARAM +#endif +#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) && !defined(__STDC_NO_VLA__) && \ + !defined(__cplusplus) && !defined(JSON_HEDLEY_PGI_VERSION) && !defined(JSON_HEDLEY_TINYC_VERSION) +#define JSON_HEDLEY_ARRAY_PARAM(name) (name) +#else +#define JSON_HEDLEY_ARRAY_PARAM(name) +#endif + +#if defined(JSON_HEDLEY_IS_CONSTANT) +#undef JSON_HEDLEY_IS_CONSTANT +#endif +#if defined(JSON_HEDLEY_REQUIRE_CONSTEXPR) +#undef JSON_HEDLEY_REQUIRE_CONSTEXPR +#endif +/* Note the double-underscore. For internal use only; no API + * guarantees! */ +#if defined(JSON_HEDLEY__IS_CONSTEXPR) +#undef JSON_HEDLEY__IS_CONSTEXPR +#endif + +#if JSON_HEDLEY_HAS_BUILTIN(__builtin_constant_p) || JSON_HEDLEY_GCC_VERSION_CHECK(3, 4, 0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13, 0, 0) || JSON_HEDLEY_TINYC_VERSION_CHECK(0, 9, 19) || \ + JSON_HEDLEY_ARM_VERSION_CHECK(4, 1, 0) || JSON_HEDLEY_IBM_VERSION_CHECK(13, 1, 0) || \ + JSON_HEDLEY_TI_VERSION_CHECK(6, 1, 0) || JSON_HEDLEY_SUNPRO_VERSION_CHECK(5, 10, 0) || \ + JSON_HEDLEY_CRAY_VERSION_CHECK(8, 1, 0) +#define JSON_HEDLEY_IS_CONSTANT(expr) __builtin_constant_p(expr) +#endif +#if !defined(__cplusplus) +#if JSON_HEDLEY_HAS_BUILTIN(__builtin_types_compatible_p) || JSON_HEDLEY_GCC_VERSION_CHECK(3, 4, 0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13, 0, 0) || JSON_HEDLEY_IBM_VERSION_CHECK(13, 1, 0) || \ + JSON_HEDLEY_CRAY_VERSION_CHECK(8, 1, 0) || JSON_HEDLEY_ARM_VERSION_CHECK(5, 4, 0) || \ + JSON_HEDLEY_TINYC_VERSION_CHECK(0, 9, 24) +#if defined(__INTPTR_TYPE__) +#define JSON_HEDLEY__IS_CONSTEXPR(expr) \ + __builtin_types_compatible_p(__typeof__((1 ? (void*)((__INTPTR_TYPE__)((expr)*0)) : (int*)0)), int*) +#else +#include +#define JSON_HEDLEY__IS_CONSTEXPR(expr) \ + __builtin_types_compatible_p(__typeof__((1 ? (void*)((intptr_t)((expr)*0)) : (int*)0)), int*) +#endif +#elif (defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) && !defined(JSON_HEDLEY_SUNPRO_VERSION) && \ + !defined(JSON_HEDLEY_PGI_VERSION)) || \ + JSON_HEDLEY_HAS_EXTENSION(c_generic_selections) || JSON_HEDLEY_GCC_VERSION_CHECK(4, 9, 0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(17, 0, 0) || JSON_HEDLEY_IBM_VERSION_CHECK(12, 1, 0) || \ + JSON_HEDLEY_ARM_VERSION_CHECK(5, 3, 0) +#if defined(__INTPTR_TYPE__) +#define JSON_HEDLEY__IS_CONSTEXPR(expr) \ + _Generic((1 ? (void*)((__INTPTR_TYPE__)((expr)*0)) : (int*)0), int* : 1, void* : 0) +#else +#include +#define JSON_HEDLEY__IS_CONSTEXPR(expr) _Generic((1 ? (void*)((intptr_t)*0) : (int*)0), int* : 1, void* : 0) +#endif +#elif defined(JSON_HEDLEY_GCC_VERSION) || defined(JSON_HEDLEY_INTEL_VERSION) || defined(JSON_HEDLEY_TINYC_VERSION) || \ + defined(JSON_HEDLEY_TI_VERSION) || defined(__clang__) +#define JSON_HEDLEY__IS_CONSTEXPR(expr) \ + (sizeof(void) != sizeof(*(1 ? ((void*)((expr)*0L)) : ((struct { char v[sizeof(void) * 2]; }*)1)))) +#endif +#endif +#if defined(JSON_HEDLEY__IS_CONSTEXPR) +#if !defined(JSON_HEDLEY_IS_CONSTANT) +#define JSON_HEDLEY_IS_CONSTANT(expr) JSON_HEDLEY__IS_CONSTEXPR(expr) +#endif +#define JSON_HEDLEY_REQUIRE_CONSTEXPR(expr) (JSON_HEDLEY__IS_CONSTEXPR(expr) ? (expr) : (-1)) +#else +#if !defined(JSON_HEDLEY_IS_CONSTANT) +#define JSON_HEDLEY_IS_CONSTANT(expr) (0) +#endif +#define JSON_HEDLEY_REQUIRE_CONSTEXPR(expr) (expr) +#endif + +#if defined(JSON_HEDLEY_BEGIN_C_DECLS) +#undef JSON_HEDLEY_BEGIN_C_DECLS +#endif +#if defined(JSON_HEDLEY_END_C_DECLS) +#undef JSON_HEDLEY_END_C_DECLS +#endif +#if defined(JSON_HEDLEY_C_DECL) +#undef JSON_HEDLEY_C_DECL +#endif +#if defined(__cplusplus) +#define JSON_HEDLEY_BEGIN_C_DECLS extern "C" { +#define JSON_HEDLEY_END_C_DECLS } +#define JSON_HEDLEY_C_DECL extern "C" +#else +#define JSON_HEDLEY_BEGIN_C_DECLS +#define JSON_HEDLEY_END_C_DECLS +#define JSON_HEDLEY_C_DECL +#endif + +#if defined(JSON_HEDLEY_STATIC_ASSERT) +#undef JSON_HEDLEY_STATIC_ASSERT +#endif +#if !defined(__cplusplus) && \ + ((defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L)) || JSON_HEDLEY_HAS_FEATURE(c_static_assert) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(6, 0, 0) || JSON_HEDLEY_INTEL_VERSION_CHECK(13, 0, 0) || defined(_Static_assert)) +#define JSON_HEDLEY_STATIC_ASSERT(expr, message) _Static_assert(expr, message) +#elif (defined(__cplusplus) && (__cplusplus >= 201703L)) || JSON_HEDLEY_MSVC_VERSION_CHECK(16, 0, 0) || \ + (defined(__cplusplus) && JSON_HEDLEY_TI_VERSION_CHECK(8, 3, 0)) +#define JSON_HEDLEY_STATIC_ASSERT(expr, message) static_assert(expr, message) +#elif defined(__cplusplus) && (__cplusplus >= 201103L) +#define JSON_HEDLEY_STATIC_ASSERT(expr, message) static_assert(expr) +#else +#define JSON_HEDLEY_STATIC_ASSERT(expr, message) +#endif + +#if defined(JSON_HEDLEY_CONST_CAST) +#undef JSON_HEDLEY_CONST_CAST +#endif +#if defined(__cplusplus) +#define JSON_HEDLEY_CONST_CAST(T, expr) (const_cast(expr)) +#elif JSON_HEDLEY_HAS_WARNING("-Wcast-qual") || JSON_HEDLEY_GCC_VERSION_CHECK(4, 6, 0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13, 0, 0) +#define JSON_HEDLEY_CONST_CAST(T, expr) \ + (__extension__({ \ + JSON_HEDLEY_DIAGNOSTIC_PUSH \ + JSON_HEDLEY_DIAGNOSTIC_DISABLE_CAST_QUAL((T)(expr)); \ + JSON_HEDLEY_DIAGNOSTIC_POP \ + })) +#else +#define JSON_HEDLEY_CONST_CAST(T, expr) ((T)(expr)) +#endif + +#if defined(JSON_HEDLEY_REINTERPRET_CAST) +#undef JSON_HEDLEY_REINTERPRET_CAST +#endif +#if defined(__cplusplus) +#define JSON_HEDLEY_REINTERPRET_CAST(T, expr) (reinterpret_cast(expr)) +#else +#define JSON_HEDLEY_REINTERPRET_CAST(T, expr) (*((T*)&(expr))) +#endif + +#if defined(JSON_HEDLEY_STATIC_CAST) +#undef JSON_HEDLEY_STATIC_CAST +#endif +#if defined(__cplusplus) +#define JSON_HEDLEY_STATIC_CAST(T, expr) (static_cast(expr)) +#else +#define JSON_HEDLEY_STATIC_CAST(T, expr) ((T)(expr)) +#endif + +#if defined(JSON_HEDLEY_CPP_CAST) +#undef JSON_HEDLEY_CPP_CAST +#endif +#if defined(__cplusplus) +#define JSON_HEDLEY_CPP_CAST(T, expr) static_cast(expr) +#else +#define JSON_HEDLEY_CPP_CAST(T, expr) (expr) +#endif + +#if defined(JSON_HEDLEY_MESSAGE) +#undef JSON_HEDLEY_MESSAGE +#endif +#if JSON_HEDLEY_HAS_WARNING("-Wunknown-pragmas") +#define JSON_HEDLEY_MESSAGE(msg) \ + JSON_HEDLEY_DIAGNOSTIC_PUSH \ + JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS \ + JSON_HEDLEY_PRAGMA(message msg) \ + JSON_HEDLEY_DIAGNOSTIC_POP +#elif JSON_HEDLEY_GCC_VERSION_CHECK(4, 4, 0) || JSON_HEDLEY_INTEL_VERSION_CHECK(13, 0, 0) +#define JSON_HEDLEY_MESSAGE(msg) JSON_HEDLEY_PRAGMA(message msg) +#elif JSON_HEDLEY_CRAY_VERSION_CHECK(5, 0, 0) +#define JSON_HEDLEY_MESSAGE(msg) JSON_HEDLEY_PRAGMA(_CRI message msg) +#elif JSON_HEDLEY_IAR_VERSION_CHECK(8, 0, 0) +#define JSON_HEDLEY_MESSAGE(msg) JSON_HEDLEY_PRAGMA(message(msg)) +#elif JSON_HEDLEY_PELLES_VERSION_CHECK(2, 0, 0) +#define JSON_HEDLEY_MESSAGE(msg) JSON_HEDLEY_PRAGMA(message(msg)) +#else +#define JSON_HEDLEY_MESSAGE(msg) +#endif + +#if defined(JSON_HEDLEY_WARNING) +#undef JSON_HEDLEY_WARNING +#endif +#if JSON_HEDLEY_HAS_WARNING("-Wunknown-pragmas") +#define JSON_HEDLEY_WARNING(msg) \ + JSON_HEDLEY_DIAGNOSTIC_PUSH \ + JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS \ + JSON_HEDLEY_PRAGMA(clang warning msg) \ + JSON_HEDLEY_DIAGNOSTIC_POP +#elif JSON_HEDLEY_GCC_VERSION_CHECK(4, 8, 0) || JSON_HEDLEY_PGI_VERSION_CHECK(18, 4, 0) +#define JSON_HEDLEY_WARNING(msg) JSON_HEDLEY_PRAGMA(GCC warning msg) +#elif JSON_HEDLEY_MSVC_VERSION_CHECK(15, 0, 0) +#define JSON_HEDLEY_WARNING(msg) JSON_HEDLEY_PRAGMA(message(msg)) +#else +#define JSON_HEDLEY_WARNING(msg) JSON_HEDLEY_MESSAGE(msg) +#endif + +#if defined(JSON_HEDLEY_REQUIRE_MSG) +#undef JSON_HEDLEY_REQUIRE_MSG +#endif +#if JSON_HEDLEY_HAS_ATTRIBUTE(diagnose_if) +#if JSON_HEDLEY_HAS_WARNING("-Wgcc-compat") +#define JSON_HEDLEY_REQUIRE_MSG(expr, msg) \ + JSON_HEDLEY_DIAGNOSTIC_PUSH \ + _Pragma("clang diagnostic ignored \"-Wgcc-compat\"") __attribute__((__diagnose_if__(!(expr), msg, "error"))) \ + JSON_HEDLEY_DIAGNOSTIC_POP +#else +#define JSON_HEDLEY_REQUIRE_MSG(expr, msg) __attribute__((__diagnose_if__(!(expr), msg, "error"))) +#endif +#else +#define JSON_HEDLEY_REQUIRE_MSG(expr, msg) +#endif + +#if defined(JSON_HEDLEY_REQUIRE) +#undef JSON_HEDLEY_REQUIRE +#endif +#define JSON_HEDLEY_REQUIRE(expr) JSON_HEDLEY_REQUIRE_MSG(expr, #expr) + +#if defined(JSON_HEDLEY_FLAGS) +#undef JSON_HEDLEY_FLAGS +#endif +#if JSON_HEDLEY_HAS_ATTRIBUTE(flag_enum) +#define JSON_HEDLEY_FLAGS __attribute__((__flag_enum__)) +#endif + +#if defined(JSON_HEDLEY_FLAGS_CAST) +#undef JSON_HEDLEY_FLAGS_CAST +#endif +#if JSON_HEDLEY_INTEL_VERSION_CHECK(19, 0, 0) +#define JSON_HEDLEY_FLAGS_CAST(T, expr) \ + (__extension__({ \ + JSON_HEDLEY_DIAGNOSTIC_PUSH \ + _Pragma("warning(disable:188)")((T)(expr)); \ + JSON_HEDLEY_DIAGNOSTIC_POP \ + })) +#else +#define JSON_HEDLEY_FLAGS_CAST(T, expr) JSON_HEDLEY_STATIC_CAST(T, expr) +#endif + +/* Remaining macros are deprecated. */ + +#if defined(JSON_HEDLEY_GCC_NOT_CLANG_VERSION_CHECK) +#undef JSON_HEDLEY_GCC_NOT_CLANG_VERSION_CHECK +#endif +#if defined(__clang__) +#define JSON_HEDLEY_GCC_NOT_CLANG_VERSION_CHECK(major, minor, patch) (0) +#else +#define JSON_HEDLEY_GCC_NOT_CLANG_VERSION_CHECK(major, minor, patch) JSON_HEDLEY_GCC_VERSION_CHECK(major, minor, patch) +#endif + +#if defined(JSON_HEDLEY_CLANG_HAS_ATTRIBUTE) +#undef JSON_HEDLEY_CLANG_HAS_ATTRIBUTE +#endif +#define JSON_HEDLEY_CLANG_HAS_ATTRIBUTE(attribute) JSON_HEDLEY_HAS_ATTRIBUTE(attribute) + +#if defined(JSON_HEDLEY_CLANG_HAS_CPP_ATTRIBUTE) +#undef JSON_HEDLEY_CLANG_HAS_CPP_ATTRIBUTE +#endif +#define JSON_HEDLEY_CLANG_HAS_CPP_ATTRIBUTE(attribute) JSON_HEDLEY_HAS_CPP_ATTRIBUTE(attribute) + +#if defined(JSON_HEDLEY_CLANG_HAS_BUILTIN) +#undef JSON_HEDLEY_CLANG_HAS_BUILTIN +#endif +#define JSON_HEDLEY_CLANG_HAS_BUILTIN(builtin) JSON_HEDLEY_HAS_BUILTIN(builtin) + +#if defined(JSON_HEDLEY_CLANG_HAS_FEATURE) +#undef JSON_HEDLEY_CLANG_HAS_FEATURE +#endif +#define JSON_HEDLEY_CLANG_HAS_FEATURE(feature) JSON_HEDLEY_HAS_FEATURE(feature) + +#if defined(JSON_HEDLEY_CLANG_HAS_EXTENSION) +#undef JSON_HEDLEY_CLANG_HAS_EXTENSION +#endif +#define JSON_HEDLEY_CLANG_HAS_EXTENSION(extension) JSON_HEDLEY_HAS_EXTENSION(extension) + +#if defined(JSON_HEDLEY_CLANG_HAS_DECLSPEC_DECLSPEC_ATTRIBUTE) +#undef JSON_HEDLEY_CLANG_HAS_DECLSPEC_DECLSPEC_ATTRIBUTE +#endif +#define JSON_HEDLEY_CLANG_HAS_DECLSPEC_ATTRIBUTE(attribute) JSON_HEDLEY_HAS_DECLSPEC_ATTRIBUTE(attribute) + +#if defined(JSON_HEDLEY_CLANG_HAS_WARNING) +#undef JSON_HEDLEY_CLANG_HAS_WARNING +#endif +#define JSON_HEDLEY_CLANG_HAS_WARNING(warning) JSON_HEDLEY_HAS_WARNING(warning) + +#endif /* !defined(JSON_HEDLEY_VERSION) || (JSON_HEDLEY_VERSION < X) */ + +// This file contains all internal macro definitions +// You MUST include macro_unscope.hpp at the end of json.hpp to undef all of them + +// exclude unsupported compilers +#if !defined(JSON_SKIP_UNSUPPORTED_COMPILER_CHECK) +#if defined(__clang__) +#if (__clang_major__ * 10000 + __clang_minor__ * 100 + __clang_patchlevel__) < 30400 +#error "unsupported Clang version - see https://github.com/nlohmann/json#supported-compilers" +#endif +#elif defined(__GNUC__) && !(defined(__ICC) || defined(__INTEL_COMPILER)) +#if (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) < 40800 +#error "unsupported GCC version - see https://github.com/nlohmann/json#supported-compilers" +#endif +#endif +#endif + +// C++ language standard detection +#if (defined(__cplusplus) && __cplusplus >= 201703L) || (defined(_HAS_CXX17) && _HAS_CXX17 == 1) // fix for issue #464 +#define JSON_HAS_CPP_17 +#define JSON_HAS_CPP_14 +#elif (defined(__cplusplus) && __cplusplus >= 201402L) || (defined(_HAS_CXX14) && _HAS_CXX14 == 1) +#define JSON_HAS_CPP_14 +#endif + +// disable float-equal warnings on GCC/clang +#if defined(__clang__) || defined(__GNUC__) || defined(__GNUG__) +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wfloat-equal" +#endif + +// disable documentation warnings on clang +#if defined(__clang__) +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wdocumentation" +#endif + +// allow to disable exceptions +#if (defined(__cpp_exceptions) || defined(__EXCEPTIONS) || defined(_CPPUNWIND)) && !defined(JSON_NOEXCEPTION) +#define JSON_THROW(exception) throw exception +#define JSON_TRY try +#define JSON_CATCH(exception) catch (exception) +#define JSON_INTERNAL_CATCH(exception) catch (exception) +#else +#include +#define JSON_THROW(exception) std::abort() +#define JSON_TRY if (true) +#define JSON_CATCH(exception) if (false) +#define JSON_INTERNAL_CATCH(exception) if (false) +#endif + +// override exception macros +#if defined(JSON_THROW_USER) +#undef JSON_THROW +#define JSON_THROW JSON_THROW_USER +#endif +#if defined(JSON_TRY_USER) +#undef JSON_TRY +#define JSON_TRY JSON_TRY_USER +#endif +#if defined(JSON_CATCH_USER) +#undef JSON_CATCH +#define JSON_CATCH JSON_CATCH_USER +#undef JSON_INTERNAL_CATCH +#define JSON_INTERNAL_CATCH JSON_CATCH_USER +#endif +#if defined(JSON_INTERNAL_CATCH_USER) +#undef JSON_INTERNAL_CATCH +#define JSON_INTERNAL_CATCH JSON_INTERNAL_CATCH_USER +#endif + +/*! +@brief macro to briefly define a mapping between an enum and JSON +@def NLOHMANN_JSON_SERIALIZE_ENUM +@since version 3.4.0 +*/ +#define NLOHMANN_JSON_SERIALIZE_ENUM(ENUM_TYPE, ...) \ + template \ + inline void to_json(BasicJsonType& j, const ENUM_TYPE& e) { \ + static_assert(std::is_enum::value, #ENUM_TYPE " must be an enum!"); \ + static const std::pair m[] = __VA_ARGS__; \ + auto it = std::find_if( \ + std::begin(m), std::end(m), \ + [e](const std::pair& ej_pair) -> bool { return ej_pair.first == e; }); \ + j = ((it != std::end(m)) ? it : std::begin(m))->second; \ + } \ + template \ + inline void from_json(const BasicJsonType& j, ENUM_TYPE& e) { \ + static_assert(std::is_enum::value, #ENUM_TYPE " must be an enum!"); \ + static const std::pair m[] = __VA_ARGS__; \ + auto it = std::find_if( \ + std::begin(m), std::end(m), \ + [j](const std::pair& ej_pair) -> bool { return ej_pair.second == j; }); \ + e = ((it != std::end(m)) ? it : std::begin(m))->first; \ + } + +// Ugly macros to avoid uglier copy-paste when specializing basic_json. They +// may be removed in the future once the class is split. + +#define NLOHMANN_BASIC_JSON_TPL_DECLARATION \ + template