From f6b4ed333521d627a470e5ef772788e20a8241bc Mon Sep 17 00:00:00 2001 From: "yudong.cai" Date: Wed, 4 Mar 2020 11:01:05 +0800 Subject: [PATCH 01/58] #1463 enable config version check Signed-off-by: yudong.cai --- core/src/server/Config.cpp | 2 +- core/unittest/server/test_config.cpp | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/core/src/server/Config.cpp b/core/src/server/Config.cpp index 92032fd982..88e53ed844 100644 --- a/core/src/server/Config.cpp +++ b/core/src/server/Config.cpp @@ -617,7 +617,7 @@ Config::CheckConfigVersion(const std::string& value) { std::string msg = "Invalid config version: " + value + ". Expected config version: " + milvus_config_version_map.at(MILVUS_VERSION); SERVER_LOG_ERROR << msg; - // return Status(SERVER_INVALID_ARGUMENT, msg); + return Status(SERVER_INVALID_ARGUMENT, msg); } } return Status::OK(); diff --git a/core/unittest/server/test_config.cpp b/core/unittest/server/test_config.cpp index 412c917935..2c26e25140 100644 --- a/core/unittest/server/test_config.cpp +++ b/core/unittest/server/test_config.cpp @@ -650,10 +650,10 @@ TEST_F(ConfigTest, SERVER_CONFIG_VALID_FAIL_TEST) { milvus::Status s = config.LoadConfigFile(config_path + VALID_CONFIG_FILE); ASSERT_TRUE(s.ok()); - // fiu_enable("check_config_version_fail", 1, NULL, 0); - // s = config.ValidateConfig(); - // ASSERT_FALSE(s.ok()); - // fiu_disable("check_config_version_fail"); + fiu_enable("check_config_version_fail", 1, NULL, 0); + s = config.ValidateConfig(); + ASSERT_FALSE(s.ok()); + fiu_disable("check_config_version_fail"); /* server config */ fiu_enable("check_config_address_fail", 1, NULL, 0); From a1457910605b2d9923dbbd445d2c4035c1738747 Mon Sep 17 00:00:00 2001 From: groot Date: Sun, 8 Mar 2020 10:27:36 +0800 Subject: [PATCH 02/58] fix sqlite bug Signed-off-by: groot --- core/src/db/meta/SqliteMetaImpl.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/db/meta/SqliteMetaImpl.cpp b/core/src/db/meta/SqliteMetaImpl.cpp index 09706ac896..031a473a9e 100644 --- a/core/src/db/meta/SqliteMetaImpl.cpp +++ b/core/src/db/meta/SqliteMetaImpl.cpp @@ -69,7 +69,7 @@ StoragePrototype(const std::string& path) { make_column("flag", &TableSchema::flag_, default_value(0)), make_column("index_file_size", &TableSchema::index_file_size_), make_column("engine_type", &TableSchema::engine_type_), - make_column("index_params", &TableSchema::index_params_, default_value("")), + make_column("index_params", &TableSchema::index_params_), make_column("metric_type", &TableSchema::metric_type_), make_column("owner_table", &TableSchema::owner_table_, default_value("")), make_column("partition_tag", &TableSchema::partition_tag_, default_value("")), From f8fa76ac96794c9d8354f0924868113bb05dc83b Mon Sep 17 00:00:00 2001 From: groot Date: Sun, 8 Mar 2020 10:44:19 +0800 Subject: [PATCH 03/58] remove search by id interface from c++sdk Signed-off-by: groot --- sdk/examples/simple/src/ClientTest.cpp | 8 ---- sdk/examples/simple/src/ClientTest.h | 17 ++++++- sdk/examples/utils/Utils.cpp | 62 -------------------------- sdk/examples/utils/Utils.h | 6 --- sdk/grpc/ClientProxy.cpp | 43 ------------------ sdk/grpc/ClientProxy.h | 5 --- sdk/grpc/GrpcClient.cpp | 20 --------- sdk/grpc/GrpcClient.h | 3 -- sdk/include/MilvusApi.h | 18 -------- sdk/interface/ConnectionImpl.cpp | 10 ----- sdk/interface/ConnectionImpl.h | 4 -- 11 files changed, 15 insertions(+), 181 deletions(-) diff --git a/sdk/examples/simple/src/ClientTest.cpp b/sdk/examples/simple/src/ClientTest.cpp index 572f120dcc..635dabdc16 100644 --- a/sdk/examples/simple/src/ClientTest.cpp +++ b/sdk/examples/simple/src/ClientTest.cpp @@ -159,13 +159,6 @@ ClientTest::SearchVectors(const std::string& table_name, int64_t topk, int64_t n topk_query_result); } -void -ClientTest::SearchVectorsByIds(const std::string& table_name, int64_t topk, int64_t nprobe) { - std::vector partition_tags; - milvus::TopKQueryResult topk_query_result; - milvus_sdk::Utils::DoSearch(conn_, table_name, partition_tags, topk, nprobe, search_id_array_, topk_query_result); -} - void ClientTest::CreateIndex(const std::string& table_name, milvus::IndexType type, int64_t nlist) { milvus_sdk::TimeRecorder rc("Create index"); @@ -245,7 +238,6 @@ ClientTest::Test() { GetVectorById(table_name, search_id_array_[0]); SearchVectors(table_name, TOP_K, NPROBE); - SearchVectorsByIds(table_name, TOP_K, NPROBE); CreateIndex(table_name, INDEX_TYPE, NLIST); ShowTableInfo(table_name); diff --git a/sdk/examples/simple/src/ClientTest.h b/sdk/examples/simple/src/ClientTest.h index 1082878109..0c62b29729 100644 --- a/sdk/examples/simple/src/ClientTest.h +++ b/sdk/examples/simple/src/ClientTest.h @@ -29,36 +29,49 @@ class ClientTest { private: void ShowServerVersion(); + void ShowSdkVersion(); + void ShowTables(std::vector&); + void CreateTable(const std::string&, int64_t, milvus::MetricType); + void DescribeTable(const std::string&); + void InsertVectors(const std::string&, int64_t); + void BuildSearchVectors(int64_t, int64_t); + void Flush(const std::string&); + void ShowTableInfo(const std::string&); + void GetVectorById(const std::string&, int64_t); + void SearchVectors(const std::string&, int64_t, int64_t); - void - SearchVectorsByIds(const std::string&, int64_t, int64_t); + void CreateIndex(const std::string&, milvus::IndexType, int64_t); + void PreloadTable(const std::string&); + void DeleteByIds(const std::string&, const std::vector&); + void DropIndex(const std::string&); + void DropTable(const std::string&); diff --git a/sdk/examples/utils/Utils.cpp b/sdk/examples/utils/Utils.cpp index 068e5099b0..c9294cb3aa 100644 --- a/sdk/examples/utils/Utils.cpp +++ b/sdk/examples/utils/Utils.cpp @@ -220,68 +220,6 @@ Utils::DoSearch(std::shared_ptr conn, const std::string& tab CheckSearchResult(search_record_array, topk_query_result); } -void -Utils::DoSearch(std::shared_ptr conn, const std::string& table_name, - const std::vector& partition_tags, int64_t top_k, int64_t nprobe, - const std::vector& search_id_array, milvus::TopKQueryResult& topk_query_result) { - topk_query_result.clear(); - - { - BLOCK_SPLITER - JSON json_params = {{"nprobe", nprobe}}; - for (auto& search_id : search_id_array) { - milvus_sdk::TimeRecorder rc("search by id " + std::to_string(search_id)); - milvus::TopKQueryResult result; - milvus::Status - stat = conn->SearchByID(table_name, partition_tags, search_id, top_k, json_params.dump(), result); - topk_query_result.insert(topk_query_result.end(), std::make_move_iterator(result.begin()), - std::make_move_iterator(result.end())); - std::cout << "SearchByID function call status: " << stat.message() << std::endl; - } - BLOCK_SPLITER - } - - if (topk_query_result.size() != search_id_array.size()) { - std::cout << "ERROR: Returned result count does not equal nq" << std::endl; - return; - } - - BLOCK_SPLITER - for (size_t i = 0; i < topk_query_result.size(); i++) { - const milvus::QueryResult& one_result = topk_query_result[i]; - size_t topk = one_result.ids.size(); - auto search_id = search_id_array[i]; - std::cout << "No." << i << " vector " << search_id << " top " << topk << " search result:" << std::endl; - for (size_t j = 0; j < topk; j++) { - std::cout << "\t" << one_result.ids[j] << "\t" << one_result.distances[j] << std::endl; - } - } - BLOCK_SPLITER - - BLOCK_SPLITER - size_t nq = topk_query_result.size(); - for (size_t i = 0; i < nq; i++) { - const milvus::QueryResult& one_result = topk_query_result[i]; - auto search_id = search_id_array[i]; - - uint64_t match_index = one_result.ids.size(); - for (uint64_t index = 0; index < one_result.ids.size(); index++) { - if (search_id == one_result.ids[index]) { - match_index = index; - break; - } - } - - if (match_index >= one_result.ids.size()) { - std::cout << "The topk result is wrong: not return search target in result set" << std::endl; - } else { - std::cout << "No." << i << " Check result successfully for target: " << search_id << " at top " - << match_index << std::endl; - } - } - BLOCK_SPLITER -} - void PrintPartitionStat(const milvus::PartitionStat& partition_stat) { std::cout << "\tPartition " << partition_stat.tag << " row count: " << partition_stat.row_count << std::endl; diff --git a/sdk/examples/utils/Utils.h b/sdk/examples/utils/Utils.h index c89f5151a9..4f33dafc33 100644 --- a/sdk/examples/utils/Utils.h +++ b/sdk/examples/utils/Utils.h @@ -70,12 +70,6 @@ class Utils { const std::vector>& search_record_array, milvus::TopKQueryResult& topk_query_result); - static void - DoSearch(std::shared_ptr conn, const std::string& table_name, - const std::vector& partition_tags, int64_t top_k, int64_t nprobe, - const std::vector& search_id_array, - milvus::TopKQueryResult& topk_query_result); - static void PrintTableInfo(const milvus::TableInfo& info); }; diff --git a/sdk/grpc/ClientProxy.cpp b/sdk/grpc/ClientProxy.cpp index 7dae7b3b16..ad23f9b954 100644 --- a/sdk/grpc/ClientProxy.cpp +++ b/sdk/grpc/ClientProxy.cpp @@ -314,49 +314,6 @@ ClientProxy::Search(const std::string& table_name, const std::vector& partition_tag_array, - int64_t query_id, - int64_t topk, - const std::string& extra_params, - TopKQueryResult& topk_query_result) { - try { - // step 1: convert vector id array - ::milvus::grpc::SearchByIDParam search_param; - ConstructSearchParam(table_name, - partition_tag_array, - topk, - extra_params, - search_param); - search_param.set_id(query_id); - - // step 2: search vectors - ::milvus::grpc::TopKQueryResult result; - Status status = client_ptr_->SearchByID(search_param, result); - if (result.row_num() == 0) { - return status; - } - - // step 4: convert result array - topk_query_result.reserve(result.row_num()); - int64_t nq = result.row_num(); - int64_t topk = result.ids().size() / nq; - for (int64_t i = 0; i < result.row_num(); i++) { - milvus::QueryResult one_result; - one_result.ids.resize(topk); - one_result.distances.resize(topk); - memcpy(one_result.ids.data(), result.ids().data() + topk * i, topk * sizeof(int64_t)); - memcpy(one_result.distances.data(), result.distances().data() + topk * i, topk * sizeof(float)); - topk_query_result.emplace_back(one_result); - } - - return status; - } catch (std::exception& ex) { - return Status(StatusCode::UnknownError, "Failed to search vectors: " + std::string(ex.what())); - } -} - Status ClientProxy::DescribeTable(const std::string& table_name, TableSchema& table_schema) { try { diff --git a/sdk/grpc/ClientProxy.h b/sdk/grpc/ClientProxy.h index 30bffedf0f..0fd493dcd2 100644 --- a/sdk/grpc/ClientProxy.h +++ b/sdk/grpc/ClientProxy.h @@ -63,11 +63,6 @@ class ClientProxy : public Connection { const std::vector& query_record_array, int64_t topk, const std::string& extra_params, TopKQueryResult& topk_query_result) override; - Status - SearchByID(const std::string& table_name, const std::vector& partition_tag_array, - int64_t query_id, int64_t topk, - const std::string& extra_params, TopKQueryResult& topk_query_result) override; - Status DescribeTable(const std::string& table_name, TableSchema& table_schema) override; diff --git a/sdk/grpc/GrpcClient.cpp b/sdk/grpc/GrpcClient.cpp index 47c6c9e8ab..96773f9c75 100644 --- a/sdk/grpc/GrpcClient.cpp +++ b/sdk/grpc/GrpcClient.cpp @@ -178,26 +178,6 @@ GrpcClient::Search( return Status::OK(); } -Status -GrpcClient::SearchByID(const ::milvus::grpc::SearchByIDParam& search_param, - ::milvus::grpc::TopKQueryResult& topk_query_result) { - ::milvus::grpc::TopKQueryResult query_result; - ClientContext context; - ::grpc::Status grpc_status = stub_->SearchByID(&context, search_param, &topk_query_result); - - if (!grpc_status.ok()) { - std::cerr << "SearchByID rpc failed!" << std::endl; - std::cerr << grpc_status.error_message() << std::endl; - return Status(StatusCode::RPCFailed, grpc_status.error_message()); - } - if (topk_query_result.status().error_code() != grpc::SUCCESS) { - std::cerr << topk_query_result.status().reason() << std::endl; - return Status(StatusCode::ServerFailed, topk_query_result.status().reason()); - } - - return Status::OK(); -} - Status GrpcClient::DescribeTable(const std::string& table_name, ::milvus::grpc::TableSchema& grpc_schema) { ClientContext context; diff --git a/sdk/grpc/GrpcClient.h b/sdk/grpc/GrpcClient.h index 51af08a2c7..d7b7ae42c5 100644 --- a/sdk/grpc/GrpcClient.h +++ b/sdk/grpc/GrpcClient.h @@ -59,9 +59,6 @@ class GrpcClient { Status Search(const grpc::SearchParam& search_param, ::milvus::grpc::TopKQueryResult& topk_query_result); - Status - SearchByID(const grpc::SearchByIDParam& search_param, ::milvus::grpc::TopKQueryResult& topk_query_result); - Status DescribeTable(const std::string& table_name, grpc::TableSchema& grpc_schema); diff --git a/sdk/include/MilvusApi.h b/sdk/include/MilvusApi.h index 8196bc676f..111cc79dc3 100644 --- a/sdk/include/MilvusApi.h +++ b/sdk/include/MilvusApi.h @@ -334,24 +334,6 @@ class Connection { const std::vector& query_record_array, int64_t topk, const std::string& extra_params, TopKQueryResult& topk_query_result) = 0; - /** - * @brief Search vector by ID - * - * This method is used to query vector in table. - * - * @param table_name, target table's name. - * @param partition_tag_array, target partitions, keep empty if no partition. - * @param query_id, vector id to be queried. - * @param topk, how many similarity vectors will be returned. - * @param extra_params, extra search parameters according to different index type, must be json format. - * @param topk_query_result, result array. - * - * @return Indicate if query is successful. - */ - virtual Status - SearchByID(const std::string& table_name, const PartitionTagList& partition_tag_array, int64_t query_id, - int64_t topk, const std::string& extra_params, TopKQueryResult& topk_query_result) = 0; - /** * @brief Show table description * diff --git a/sdk/interface/ConnectionImpl.cpp b/sdk/interface/ConnectionImpl.cpp index 54c3aded92..4acbcec7b8 100644 --- a/sdk/interface/ConnectionImpl.cpp +++ b/sdk/interface/ConnectionImpl.cpp @@ -100,16 +100,6 @@ ConnectionImpl::Search(const std::string& table_name, const std::vectorSearch(table_name, partition_tags, query_record_array, topk, extra_params, topk_query_result); } -Status -ConnectionImpl::SearchByID(const std::string& table_name, - const std::vector& partition_tags, - int64_t query_id, - int64_t topk, - const std::string& extra_params, - TopKQueryResult& topk_query_result) { - return client_proxy_->SearchByID(table_name, partition_tags, query_id, topk, extra_params, topk_query_result); -} - Status ConnectionImpl::DescribeTable(const std::string& table_name, TableSchema& table_schema) { return client_proxy_->DescribeTable(table_name, table_schema); diff --git a/sdk/interface/ConnectionImpl.h b/sdk/interface/ConnectionImpl.h index 0a89a7e9a7..ec0f423857 100644 --- a/sdk/interface/ConnectionImpl.h +++ b/sdk/interface/ConnectionImpl.h @@ -65,10 +65,6 @@ class ConnectionImpl : public Connection { const std::vector& query_record_array, int64_t topk, const std::string& extra_params, TopKQueryResult& topk_query_result) override; - Status - SearchByID(const std::string& table_name, const std::vector& partition_tag_array, int64_t query_id, - int64_t topk, const std::string& extra_params, TopKQueryResult& topk_query_result) override; - Status DescribeTable(const std::string& table_name, TableSchema& table_schema) override; From 1829624a43cc015a29f02572096527ef3bb0bdfc Mon Sep 17 00:00:00 2001 From: groot Date: Sun, 8 Mar 2020 11:59:25 +0800 Subject: [PATCH 04/58] #1556 Index file not create after table and index create Signed-off-by: groot --- CHANGELOG.md | 1 + core/src/db/meta/MySQLMetaImpl.cpp | 11 +---------- core/src/db/meta/SqliteMetaImpl.cpp | 11 +---------- 3 files changed, 3 insertions(+), 20 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6711f85820..2ddd57d815 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -43,6 +43,7 @@ Please mark all change in change log and use the issue from GitHub - \#1530 Set table file with correct engine type in meta - \#1532 Search with ivf_flat failed with open-dataset: sift-256-hamming - \#1535 Degradation searching performance with metric_type: binary_idmap +- \#1556 Index file not create after table and index create ## Feature - \#216 Add CLI to get server info diff --git a/core/src/db/meta/MySQLMetaImpl.cpp b/core/src/db/meta/MySQLMetaImpl.cpp index 6f6e8cdcd3..b1d28c2d34 100644 --- a/core/src/db/meta/MySQLMetaImpl.cpp +++ b/core/src/db/meta/MySQLMetaImpl.cpp @@ -674,16 +674,7 @@ MySQLMetaImpl::CreateTableFile(TableFileSchema& file_schema) { file_schema.updated_time_ = file_schema.created_on_; file_schema.index_file_size_ = table_schema.index_file_size_; file_schema.index_params_ = table_schema.index_params_; - - if (file_schema.file_type_ == TableFileSchema::FILE_TYPE::NEW || - file_schema.file_type_ == TableFileSchema::FILE_TYPE::NEW_MERGE) { - file_schema.engine_type_ = server::ValidationUtil::IsBinaryMetricType(table_schema.metric_type_) - ? (int32_t)EngineType::FAISS_BIN_IDMAP - : (int32_t)EngineType::FAISS_IDMAP; - } else { - file_schema.engine_type_ = table_schema.engine_type_; - } - + file_schema.engine_type_ = table_schema.engine_type_; file_schema.metric_type_ = table_schema.metric_type_; std::string id = "NULL"; // auto-increment diff --git a/core/src/db/meta/SqliteMetaImpl.cpp b/core/src/db/meta/SqliteMetaImpl.cpp index 09706ac896..49a58aafe7 100644 --- a/core/src/db/meta/SqliteMetaImpl.cpp +++ b/core/src/db/meta/SqliteMetaImpl.cpp @@ -375,16 +375,7 @@ SqliteMetaImpl::CreateTableFile(TableFileSchema& file_schema) { file_schema.updated_time_ = file_schema.created_on_; file_schema.index_file_size_ = table_schema.index_file_size_; file_schema.index_params_ = table_schema.index_params_; - - if (file_schema.file_type_ == TableFileSchema::FILE_TYPE::NEW || - file_schema.file_type_ == TableFileSchema::FILE_TYPE::NEW_MERGE) { - file_schema.engine_type_ = server::ValidationUtil::IsBinaryMetricType(table_schema.metric_type_) - ? (int32_t)EngineType::FAISS_BIN_IDMAP - : (int32_t)EngineType::FAISS_IDMAP; - } else { - file_schema.engine_type_ = table_schema.engine_type_; - } - + file_schema.engine_type_ = table_schema.engine_type_; file_schema.metric_type_ = table_schema.metric_type_; // multi-threads call sqlite update may get exception('bad logic', etc), so we add a lock here From fcb5656819b74069d26c0138f08e47813d271c8e Mon Sep 17 00:00:00 2001 From: groot Date: Sun, 8 Mar 2020 12:13:06 +0800 Subject: [PATCH 05/58] refine c++sdk Signed-off-by: groot --- sdk/grpc/ClientProxy.cpp | 15 -- sdk/grpc/ClientProxy.h | 3 - sdk/include/MilvusApi.h | 396 +++++++++++++++---------------- sdk/interface/ConnectionImpl.cpp | 5 - sdk/interface/ConnectionImpl.h | 3 - 5 files changed, 193 insertions(+), 229 deletions(-) diff --git a/sdk/grpc/ClientProxy.cpp b/sdk/grpc/ClientProxy.cpp index ad23f9b954..6986355b4e 100644 --- a/sdk/grpc/ClientProxy.cpp +++ b/sdk/grpc/ClientProxy.cpp @@ -415,21 +415,6 @@ ClientProxy::ServerStatus() const { } } -std::string -ClientProxy::DumpTaskTables() const { - if (channel_ == nullptr) { - return "not connected to server"; - } - - try { - std::string dummy; - Status status = client_ptr_->Cmd("tasktable", dummy); - return dummy; - } catch (std::exception& ex) { - return "connection lost"; - } -} - Status ClientProxy::DeleteByID(const std::string& table_name, const std::vector& id_array) { try { diff --git a/sdk/grpc/ClientProxy.h b/sdk/grpc/ClientProxy.h index 0fd493dcd2..2c4a195072 100644 --- a/sdk/grpc/ClientProxy.h +++ b/sdk/grpc/ClientProxy.h @@ -84,9 +84,6 @@ class ClientProxy : public Connection { std::string ServerStatus() const override; - std::string - DumpTaskTables() const override; - Status DeleteByID(const std::string& table_name, const std::vector& id_array) override; diff --git a/sdk/include/MilvusApi.h b/sdk/include/MilvusApi.h index 111cc79dc3..e899210c91 100644 --- a/sdk/include/MilvusApi.h +++ b/sdk/include/MilvusApi.h @@ -54,20 +54,19 @@ struct ConnectParam { }; /** - * @brief Table Schema + * @brief Collection parameters */ -struct TableSchema { - std::string table_name; ///< Table name +struct CollectionParam { + std::string collection_name; ///< Collection_name name int64_t dimension = 0; ///< Vector dimension, must be a positive value int64_t index_file_size = 1024; ///< Index file size, must be a positive value, unit: MB MetricType metric_type = MetricType::L2; ///< Index metric type }; - /** - * @brief Record inserted + * @brief Entity inserted, currently each entity represent a vector */ -struct RowRecord { +struct Entity { std::vector float_data; ///< Vector raw float data std::vector binary_data; ///< Vector raw binary data }; @@ -76,7 +75,7 @@ struct RowRecord { * @brief TopK query result */ struct QueryResult { - std::vector ids; ///< Query ids result + std::vector ids; ///< Query entity ids result std::vector distances; ///< Query distances result }; using TopKQueryResult = std::vector; ///< Topk query result @@ -85,13 +84,14 @@ using TopKQueryResult = std::vector; ///< Topk query result * @brief Index parameters * Note: extra_params is extra parameters list, it must be json format * For different index type, parameter list is different accordingly, for example: - * FLAT/IVFLAT/SQ8: "{nlist: '16384'}" - * IVFPQ: "{nlist: '16384', nbits: "16"}" - * NSG: "{search_length: '100', out_degree:'40', pool_size:'66'}" - * HNSW "{M: '16', ef_construct:'500'}" + * FLAT/IVFLAT/SQ8: "{nlist: '16384'}" nlist range:[1, 999999] + * IVFPQ: "{nlist: '16384', m: "12"}" nlist range:[1, 999999] m is decided by dim and have a couple of results. + * NSG: "{search_length: '45', out_degree:'50', candidate_pool_size:'300', "knng":'100'}" + * search_length range:[10, 300] out_degree range:[5, 300] candidate_pool_size range:[50, 1000] knng range:[5, 300] + * HNSW "{M: '16', efConstruction:'500'}" M range:[5, 48] efConstruction range:[topk, 4096] */ struct IndexParam { - std::string table_name; ///< Table name for create index + std::string collection_name; ///< Collection name for create index IndexType index_type; ///< Index type std::string extra_params; ///< Extra parameters according to different index type, must be json format }; @@ -100,7 +100,7 @@ struct IndexParam { * @brief partition parameters */ struct PartitionParam { - std::string table_name; + std::string collection_name; std::string partition_tag; }; @@ -126,11 +126,11 @@ struct PartitionStat { }; /** - * @brief table info + * @brief collection info */ -struct TableInfo { - int64_t total_row_count; ///< Table total row count - std::vector partitions_stat; ///< Table's partitions statistics +struct CollectionInfo { + int64_t total_row_count; ///< Collection total entity count + std::vector partitions_stat; ///< Collection's partitions statistics }; /** @@ -209,182 +209,6 @@ class Connection { virtual Status Disconnect() = 0; - /** - * @brief Create table method - * - * This method is used to create table. - * - * @param param, use to provide table information to be created. - * - * @return Indicate if table is created successfully - */ - virtual Status - CreateTable(const TableSchema& param) = 0; - - /** - * @brief Test table existence method - * - * This method is used to create table. - * - * @param table_name, target table's name. - * - * @return Indicate if table is cexist - */ - virtual bool - HasTable(const std::string& table_name) = 0; - - /** - * @brief Drop table method - * - * This method is used to drop table(and its partitions). - * - * @param table_name, target table's name. - * - * @return Indicate if table is drop successfully. - */ - virtual Status - DropTable(const std::string& table_name) = 0; - - /** - * @brief Create index method - * - * This method is used to create index for whole table(and its partitions). - * - * @param IndexParam - * table_name, table name is going to be create index. - * index type, - * nlist, - * index file size - * - * @return Indicate if build index successfully. - */ - virtual Status - CreateIndex(const IndexParam& index_param) = 0; - - /** - * @brief Insert vector to table - * - * This method is used to insert vector array to table. - * - * @param table_name, target table's name. - * @param partition_tag, target partition's tag, keep empty if no partition specified. - * @param record_array, vector array is inserted. - * @param id_array, - * specify id for each vector, - * if this array is empty, milvus will generate unique id for each vector, - * and return all ids by this parameter. - * - * @return Indicate if vector array are inserted successfully - */ - virtual Status - Insert(const std::string& table_name, const std::string& partition_tag, const std::vector& record_array, - std::vector& id_array) = 0; - - /** - * @brief Get vector data by id - * - * This method is used to get vector data by id from a table. - * Return the first found vector if there are vectors with duplicated id - * - * @param table_name, target table's name. - * @param vector_id, target vector id. - * @param vector_data, returned vector data. - * - * @return Indicate if the operation is succeed. - */ - virtual Status - GetVectorByID(const std::string& table_name, int64_t vector_id, RowRecord& vector_data) = 0; - - /** - * @brief Get vector ids from a segment - * - * This method is used to get vector ids from a segment - * Return all vector(not deleted) ids - * - * @param table_name, target table's name. - * @param segment_name, target segment name. - * @param id_array, returned vector id array. - * - * @return Indicate if the operation is succeed. - */ - virtual Status - GetIDsInSegment(const std::string& table_name, const std::string& segment_name, std::vector& id_array) = 0; - - /** - * @brief Search vector - * - * This method is used to query vector in table. - * - * @param table_name, target table's name. - * @param partition_tag_array, target partitions, keep empty if no partition specified. - * @param query_record_array, vectors to be queried. - * @param topk, how many similarity vectors will be returned. - * @param extra_params, extra search parameters according to different index type, must be json format. - * Note: extra_params is extra parameters list, it must be json format, for example: - * For different index type, parameter list is different accordingly - * FLAT/IVFLAT/SQ8/IVFPQ: "{nprobe: '32'}" - * NSG: "{search_length:'100'} - * HNSW "{ef: '64'} - * @param topk_query_result, result array. - * - * @return Indicate if query is successful. - */ - virtual Status - Search(const std::string& table_name, const PartitionTagList& partition_tag_array, - const std::vector& query_record_array, int64_t topk, - const std::string& extra_params, TopKQueryResult& topk_query_result) = 0; - - /** - * @brief Show table description - * - * This method is used to show table information. - * - * @param table_name, target table's name. - * @param table_schema, table_schema is given when operation is successful. - * - * @return Indicate if this operation is successful. - */ - virtual Status - DescribeTable(const std::string& table_name, TableSchema& table_schema) = 0; - - /** - * @brief Get table row count - * - * This method is used to get table row count. - * - * @param table_name, target table's name. - * @param row_count, table total row count(including partitions). - * - * @return Indicate if this operation is successful. - */ - virtual Status - CountTable(const std::string& table_name, int64_t& row_count) = 0; - - /** - * @brief Show all tables in database - * - * This method is used to list all tables. - * - * @param table_array, all tables in database. - * - * @return Indicate if this operation is successful. - */ - virtual Status - ShowTables(std::vector& table_array) = 0; - - /** - * @brief Show table information - * - * This method is used to get detail information of a table. - * - * @param table_name, target table's name. - * @param table_info, target table's information - * - * @return Indicate if this operation is successful. - */ - virtual Status - ShowTableInfo(const std::string& table_name, TableInfo& table_info) = 0; - /** * @brief Give the client version * @@ -405,6 +229,182 @@ class Connection { virtual std::string ServerVersion() const = 0; + /** + * @brief Create collection method + * + * This method is used to create collection. + * + * @param param, use to provide collection information to be created. + * + * @return Indicate if collection is created successfully + */ + virtual Status + CreateCollection(const CollectionParam& param) = 0; + + /** + * @brief Test collection existence method + * + * This method is used to test collection existence. + * + * @param collection_name, target collection's name. + * + * @return Indicate if collection is cexist + */ + virtual bool + HasCollection(const std::string& collection_name) = 0; + + /** + * @brief Drop collection method + * + * This method is used to drop collection(and its partitions). + * + * @param collection_name, target collection's name. + * + * @return Indicate if collection is drop successfully. + */ + virtual Status + DropCollection(const std::string& collection_name) = 0; + + /** + * @brief Create index method + * + * This method is used to create index for whole collection(and its partitions). + * + * @param index_param, use to provide index information to be created. + * + * @return Indicate if create index successfully. + */ + virtual Status + CreateIndex(const IndexParam& index_param) = 0; + + /** + * @brief Insert entity to table + * + * This method is used to insert vector array to table. + * + * @param collection_name, target collection's name. + * @param partition_tag, target partition's tag, keep empty if no partition specified. + * @param entity_array, entity array is inserted, each entitu represent a vector. + * @param id_array, + * specify id for each entity, + * if this array is empty, milvus will generate unique id for each entity, + * and return all ids by this parameter. + * + * @return Indicate if entity array are inserted successfully + */ + virtual Status + Insert(const std::string& collection_name, + const std::string& partition_tag, + const std::vector& entity_array, + std::vector& id_array) = 0; + + /** + * @brief Get entity data by id + * + * This method is used to get entity data by id from a collection. + * Return the first found entity if there are entities with duplicated id + * + * @param collection_name, target collection's name. + * @param entity_id, target entity id. + * @param entity_data, returned entity data. + * + * @return Indicate if the operation is succeed. + */ + virtual Status + GetEntityByID(const std::string& collection_name, int64_t entity_id, Entity& entity_data) = 0; + + /** + * @brief Get entity ids from a segment + * + * This method is used to get entity ids from a segment + * Return all entity(not deleted) ids + * + * @param collection_name, target collection's name. + * @param segment_name, target segment name. + * @param id_array, returned entity id array. + * + * @return Indicate if the operation is succeed. + */ + virtual Status + GetIDsInSegment(const std::string& collection_name, + const std::string& segment_name, + std::vector& id_array) = 0; + + /** + * @brief Search entities in a collection + * + * This method is used to query entity in collection. + * + * @param collection_name, target collection's name. + * @param partition_tag_array, target partitions, keep empty if no partition specified. + * @param query_record_array, vectors to be queried. + * @param topk, how many similarity vectors will be returned. + * @param extra_params, extra search parameters according to different index type, must be json format. + * Note: extra_params is extra parameters list, it must be json format, for example: + * For different index type, parameter list is different accordingly + * FLAT/IVFLAT/SQ8/IVFPQ: "{nprobe: '32'}" + * NSG: "{search_length:'100'} + * HNSW "{ef: '64'} + * @param topk_query_result, result array. + * + * @return Indicate if query is successful. + */ + virtual Status + Search(const std::string& table_name, const PartitionTagList& partition_tag_array, + const std::vector& query_record_array, int64_t topk, + const std::string& extra_params, TopKQueryResult& topk_query_result) = 0; + + /** + * @brief Show collection description + * + * This method is used to show collection information. + * + * @param collection_name, target collection's name. + * @param collection_schema, collection_schema is given when operation is successful. + * + * @return Indicate if this operation is successful. + */ + virtual Status + DescribeCollection(const std::string& collection_name, TableSchema& collection_schema) = 0; + + /** + * @brief Get collection row count + * + * This method is used to get collection row count. + * + * @param collection_name, target collection's name. + * @param entity_count, collection total entity count(including partitions). + * + * @return Indicate if this operation is successful. + */ + virtual Status + CountCollection(const std::string& collection_name, int64_t& entity_count) = 0; + + /** + * @brief Show all collections in database + * + * This method is used to list all collections. + * + * @param collection_array, all collections in database. + * + * @return Indicate if this operation is successful. + */ + virtual Status + ShowCollections(std::vector& collection_array) = 0; + + /** + * @brief Show collections information + * + * This method is used to get detail information of a collections. + * + * @param collections_name, target collections's name. + * @param collections_info, target collections's information + * + * @return Indicate if this operation is successful. + */ + virtual Status + ShowCollectionInfo(const std::string& collections_name, TableInfo& collections_info) = 0; + /** * @brief Give the server status * @@ -415,16 +415,6 @@ class Connection { virtual std::string ServerStatus() const = 0; - /** - * @brief dump server tasks information - * - * This method is internal used. - * - * @return Task information in tasktables. - */ - virtual std::string - DumpTaskTables() const = 0; - /** * [deprecated] * @brief delete tables by vector id diff --git a/sdk/interface/ConnectionImpl.cpp b/sdk/interface/ConnectionImpl.cpp index 4acbcec7b8..3a3fa52516 100644 --- a/sdk/interface/ConnectionImpl.cpp +++ b/sdk/interface/ConnectionImpl.cpp @@ -130,11 +130,6 @@ ConnectionImpl::ServerStatus() const { return client_proxy_->ServerStatus(); } -std::string -ConnectionImpl::DumpTaskTables() const { - return client_proxy_->DumpTaskTables(); -} - Status ConnectionImpl::DeleteByID(const std::string& table_name, const std::vector& id_array) { return client_proxy_->DeleteByID(table_name, id_array); diff --git a/sdk/interface/ConnectionImpl.h b/sdk/interface/ConnectionImpl.h index ec0f423857..469adf1a27 100644 --- a/sdk/interface/ConnectionImpl.h +++ b/sdk/interface/ConnectionImpl.h @@ -86,9 +86,6 @@ class ConnectionImpl : public Connection { std::string ServerStatus() const override; - std::string - DumpTaskTables() const override; - Status DeleteByID(const std::string& table_name, const std::vector& id_array) override; From 9f7bf032da8f74c06ee47f17dbd93d2d1069e061 Mon Sep 17 00:00:00 2001 From: groot Date: Sun, 8 Mar 2020 12:14:05 +0800 Subject: [PATCH 06/58] typo Signed-off-by: groot --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2ddd57d815..74bd843649 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -43,7 +43,7 @@ Please mark all change in change log and use the issue from GitHub - \#1530 Set table file with correct engine type in meta - \#1532 Search with ivf_flat failed with open-dataset: sift-256-hamming - \#1535 Degradation searching performance with metric_type: binary_idmap -- \#1556 Index file not create after table and index create +- \#1556 Index file not created after table and index created ## Feature - \#216 Add CLI to get server info From a96a837186e3ec56d2b0aa0f7ab520cbb01eb765 Mon Sep 17 00:00:00 2001 From: JinHai-CN Date: Sun, 8 Mar 2020 12:39:00 +0800 Subject: [PATCH 07/58] Update CHANGELOG.md Signed-off-by: JinHai-CN --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 74bd843649..951494dddc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,7 +13,7 @@ Please mark all change in change log and use the issue from GitHub - \#977 Server crash when create tables concurrently - \#990 Check gpu resources setting when assign repeated value - \#995 table count set to 0 if no tables found -- \#1010 improve error message when offset or page_size is equal 0 +- \#1010 Improve error message when offset or page_size is equal 0 - \#1022 check if partition name is legal - \#1028 check if table exists when show partitions - \#1029 check if table exists when try to delete partition From c7bc4a8a224929ee749c98d2cf91ffc28f3f1abc Mon Sep 17 00:00:00 2001 From: "xiaojun.lin" Date: Sun, 8 Mar 2020 14:21:59 +0800 Subject: [PATCH 08/58] update max_support_dim Signed-off-by: xiaojun.lin --- CHANGELOG.md | 1 + core/src/wrapper/ConfAdapter.cpp | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 951494dddc..221a6891dd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -44,6 +44,7 @@ Please mark all change in change log and use the issue from GitHub - \#1532 Search with ivf_flat failed with open-dataset: sift-256-hamming - \#1535 Degradation searching performance with metric_type: binary_idmap - \#1556 Index file not created after table and index created +- \#1560 Search crashed with Super-high dimensional binary vector ## Feature - \#216 Add CLI to get server info diff --git a/core/src/wrapper/ConfAdapter.cpp b/core/src/wrapper/ConfAdapter.cpp index 7742f3bab7..db06504def 100644 --- a/core/src/wrapper/ConfAdapter.cpp +++ b/core/src/wrapper/ConfAdapter.cpp @@ -31,7 +31,7 @@ namespace engine { #define GPU_MAX_NRPOBE 1024 #endif -#define DEFAULT_MAX_DIM 16384 +#define DEFAULT_MAX_DIM 32768 #define DEFAULT_MIN_DIM 1 #define DEFAULT_MAX_K 16384 #define DEFAULT_MIN_K 1 From 9203bdf84db282223c765dcb27e605cde4c82494 Mon Sep 17 00:00:00 2001 From: groot Date: Sun, 8 Mar 2020 14:27:16 +0800 Subject: [PATCH 09/58] rename c++sdk Signed-off-by: groot --- sdk/examples/binary_vector/src/ClientTest.cpp | 107 +-- sdk/examples/partition/src/ClientTest.cpp | 153 ++-- sdk/examples/simple/src/ClientTest.cpp | 199 ++--- sdk/examples/simple/src/ClientTest.h | 24 +- sdk/examples/utils/Utils.cpp | 79 +- sdk/examples/utils/Utils.h | 18 +- sdk/grpc/ClientProxy.cpp | 751 +++++++++--------- sdk/grpc/ClientProxy.h | 130 +-- sdk/include/MilvusApi.h | 212 ++--- sdk/interface/ConnectionImpl.cpp | 206 ++--- sdk/interface/ConnectionImpl.h | 130 +-- 11 files changed, 1015 insertions(+), 994 deletions(-) diff --git a/sdk/examples/binary_vector/src/ClientTest.cpp b/sdk/examples/binary_vector/src/ClientTest.cpp index f6dbd4b8d2..db16575f61 100644 --- a/sdk/examples/binary_vector/src/ClientTest.cpp +++ b/sdk/examples/binary_vector/src/ClientTest.cpp @@ -22,52 +22,53 @@ namespace { -const char* TABLE_NAME = milvus_sdk::Utils::GenTableName().c_str(); +const char* COLLECTION_NAME = milvus_sdk::Utils::GenCollectionName().c_str(); -constexpr int64_t TABLE_DIMENSION = 512; -constexpr int64_t TABLE_INDEX_FILE_SIZE = 128; -constexpr milvus::MetricType TABLE_METRIC_TYPE = milvus::MetricType::TANIMOTO; -constexpr int64_t BATCH_ROW_COUNT = 100000; +constexpr int64_t COLLECTION_DIMENSION = 512; +constexpr int64_t COLLECTION_INDEX_FILE_SIZE = 128; +constexpr milvus::MetricType COLLECTION_METRIC_TYPE = milvus::MetricType::TANIMOTO; +constexpr int64_t BATCH_ENTITY_COUNT = 100000; constexpr int64_t NQ = 5; constexpr int64_t TOP_K = 10; constexpr int64_t NPROBE = 32; -constexpr int64_t SEARCH_TARGET = 5000; // change this value, result is different, ensure less than BATCH_ROW_COUNT -constexpr int64_t ADD_VECTOR_LOOP = 20; +constexpr int64_t SEARCH_TARGET = 5000; // change this value, result is different, ensure less than BATCH_ENTITY_COUNT +constexpr int64_t ADD_ENTITY_LOOP = 20; constexpr milvus::IndexType INDEX_TYPE = milvus::IndexType::IVFFLAT; -milvus::TableSchema -BuildTableSchema() { - milvus::TableSchema tb_schema = {TABLE_NAME, TABLE_DIMENSION, TABLE_INDEX_FILE_SIZE, TABLE_METRIC_TYPE}; - return tb_schema; +milvus::CollectionParam +BuildCollectionParam() { + milvus::CollectionParam + collection_param = {COLLECTION_NAME, COLLECTION_DIMENSION, COLLECTION_INDEX_FILE_SIZE, COLLECTION_METRIC_TYPE}; + return collection_param; } milvus::IndexParam BuildIndexParam() { JSON json_params = {{"nlist", 1024}}; - milvus::IndexParam index_param = {TABLE_NAME, INDEX_TYPE, json_params.dump()}; + milvus::IndexParam index_param = {COLLECTION_NAME, INDEX_TYPE, json_params.dump()}; return index_param; } void -BuildBinaryVectors(int64_t from, int64_t to, std::vector& vector_record_array, - std::vector& record_ids, int64_t dimension) { +BuildBinaryVectors(int64_t from, int64_t to, std::vector& entity_array, + std::vector& entity_ids, int64_t dimension) { if (to <= from) { return; } - vector_record_array.clear(); - record_ids.clear(); + entity_array.clear(); + entity_ids.clear(); int64_t dim_byte = dimension/8; for (int64_t k = from; k < to; k++) { - milvus::RowRecord record; - record.binary_data.resize(dim_byte); + milvus::Entity entity; + entity.binary_data.resize(dim_byte); for (int64_t i = 0; i < dim_byte; i++) { - record.binary_data[i] = (uint8_t)lrand48(); + entity.binary_data[i] = (uint8_t)lrand48(); } - vector_record_array.emplace_back(record); - record_ids.push_back(k); + entity_array.emplace_back(entity); + entity_ids.push_back(k); } } @@ -84,54 +85,54 @@ ClientTest::Test(const std::string& address, const std::string& port) { std::cout << "Connect function call status: " << stat.message() << std::endl; } - { // create table - milvus::TableSchema tb_schema = BuildTableSchema(); - stat = conn->CreateTable(tb_schema); - std::cout << "CreateTable function call status: " << stat.message() << std::endl; - milvus_sdk::Utils::PrintTableSchema(tb_schema); + { // create collection + milvus::CollectionParam collection_param = BuildCollectionParam(); + stat = conn->CreateCollection(collection_param); + std::cout << "CreateCollection function call status: " << stat.message() << std::endl; + milvus_sdk::Utils::PrintCollectionParam(collection_param); - bool has_table = conn->HasTable(tb_schema.table_name); - if (has_table) { - std::cout << "Table is created" << std::endl; + bool has_collection = conn->HasCollection(collection_param.collection_name); + if (has_collection) { + std::cout << "Collection is created" << std::endl; } } - std::vector> search_record_array; + std::vector> search_entity_array; { // insert vectors - for (int i = 0; i < ADD_VECTOR_LOOP; i++) { - std::vector record_array; - std::vector record_ids; - int64_t begin_index = i * BATCH_ROW_COUNT; + for (int i = 0; i < ADD_ENTITY_LOOP; i++) { + std::vector entity_array; + std::vector entity_ids; + int64_t begin_index = i * BATCH_ENTITY_COUNT; { // generate vectors - milvus_sdk::TimeRecorder rc("Build vectors No." + std::to_string(i)); + milvus_sdk::TimeRecorder rc("Build entities No." + std::to_string(i)); BuildBinaryVectors(begin_index, - begin_index + BATCH_ROW_COUNT, - record_array, - record_ids, - TABLE_DIMENSION); + begin_index + BATCH_ENTITY_COUNT, + entity_array, + entity_ids, + COLLECTION_DIMENSION); } - if (search_record_array.size() < NQ) { - search_record_array.push_back(std::make_pair(record_ids[SEARCH_TARGET], record_array[SEARCH_TARGET])); + if (search_entity_array.size() < NQ) { + search_entity_array.push_back(std::make_pair(entity_ids[SEARCH_TARGET], entity_array[SEARCH_TARGET])); } - std::string title = "Insert " + std::to_string(record_array.size()) + " vectors No." + std::to_string(i); + std::string title = "Insert " + std::to_string(entity_array.size()) + " entities No." + std::to_string(i); milvus_sdk::TimeRecorder rc(title); - stat = conn->Insert(TABLE_NAME, "", record_array, record_ids); - std::cout << "InsertVector function call status: " << stat.message() << std::endl; - std::cout << "Returned id array count: " << record_ids.size() << std::endl; + stat = conn->Insert(COLLECTION_NAME, "", entity_array, entity_ids); + std::cout << "Insert function call status: " << stat.message() << std::endl; + std::cout << "Returned id array count: " << entity_ids.size() << std::endl; } } { // flush buffer - stat = conn->FlushTable(TABLE_NAME); - std::cout << "FlushTable function call status: " << stat.message() << std::endl; + stat = conn->FlushCollection(COLLECTION_NAME); + std::cout << "FlushCollection function call status: " << stat.message() << std::endl; } { // search vectors std::vector partition_tags; milvus::TopKQueryResult topk_query_result; - milvus_sdk::Utils::DoSearch(conn, TABLE_NAME, partition_tags, TOP_K, NPROBE, search_record_array, + milvus_sdk::Utils::DoSearch(conn, COLLECTION_NAME, partition_tags, TOP_K, NPROBE, search_entity_array, topk_query_result); } @@ -144,7 +145,7 @@ ClientTest::Test(const std::string& address, const std::string& port) { std::cout << "CreateIndex function call status: " << stat.message() << std::endl; milvus::IndexParam index2; - stat = conn->DescribeIndex(TABLE_NAME, index2); + stat = conn->DescribeIndex(COLLECTION_NAME, index2); std::cout << "DescribeIndex function call status: " << stat.message() << std::endl; milvus_sdk::Utils::PrintIndexParam(index2); } @@ -152,13 +153,13 @@ ClientTest::Test(const std::string& address, const std::string& port) { { // search vectors std::vector partition_tags; milvus::TopKQueryResult topk_query_result; - milvus_sdk::Utils::DoSearch(conn, TABLE_NAME, partition_tags, TOP_K, NPROBE, search_record_array, + milvus_sdk::Utils::DoSearch(conn, COLLECTION_NAME, partition_tags, TOP_K, NPROBE, search_entity_array, topk_query_result); } - { // drop table - stat = conn->DropTable(TABLE_NAME); - std::cout << "DropTable function call status: " << stat.message() << std::endl; + { // drop collection + stat = conn->DropCollection(COLLECTION_NAME); + std::cout << "DropCollection function call status: " << stat.message() << std::endl; } milvus::Connection::Destroy(conn); diff --git a/sdk/examples/partition/src/ClientTest.cpp b/sdk/examples/partition/src/ClientTest.cpp index aca815c6ed..e5841fed70 100644 --- a/sdk/examples/partition/src/ClientTest.cpp +++ b/sdk/examples/partition/src/ClientTest.cpp @@ -22,12 +22,12 @@ namespace { -const char* TABLE_NAME = milvus_sdk::Utils::GenTableName().c_str(); +const char* COLLECTION_NAME = milvus_sdk::Utils::GenCollectionName().c_str(); -constexpr int64_t TABLE_DIMENSION = 512; -constexpr int64_t TABLE_INDEX_FILE_SIZE = 1024; -constexpr milvus::MetricType TABLE_METRIC_TYPE = milvus::MetricType::L2; -constexpr int64_t BATCH_ROW_COUNT = 10000; +constexpr int64_t COLLECTION_DIMENSION = 512; +constexpr int64_t COLLECTION_INDEX_FILE_SIZE = 1024; +constexpr milvus::MetricType COLLECTION_METRIC_TYPE = milvus::MetricType::L2; +constexpr int64_t BATCH_ENTITY_COUNT = 10000; constexpr int64_t NQ = 5; constexpr int64_t TOP_K = 10; constexpr int64_t NPROBE = 32; @@ -36,27 +36,45 @@ constexpr milvus::IndexType INDEX_TYPE = milvus::IndexType::IVFSQ8; constexpr int32_t PARTITION_COUNT = 5; constexpr int32_t TARGET_PARTITION = 3; -milvus::TableSchema -BuildTableSchema() { - milvus::TableSchema tb_schema = {TABLE_NAME, TABLE_DIMENSION, TABLE_INDEX_FILE_SIZE, TABLE_METRIC_TYPE}; - return tb_schema; +milvus::CollectionParam +BuildCollectionParam() { + milvus::CollectionParam + collection_param = {COLLECTION_NAME, COLLECTION_DIMENSION, COLLECTION_INDEX_FILE_SIZE, COLLECTION_METRIC_TYPE}; + return collection_param; } milvus::PartitionParam BuildPartitionParam(int32_t index) { std::string tag = std::to_string(index); - std::string partition_name = std::string(TABLE_NAME) + "_" + tag; - milvus::PartitionParam partition_param = {TABLE_NAME, tag}; + std::string partition_name = std::string(COLLECTION_NAME) + "_" + tag; + milvus::PartitionParam partition_param = {COLLECTION_NAME, tag}; return partition_param; } milvus::IndexParam BuildIndexParam() { JSON json_params = {{"nlist", 16384}}; - milvus::IndexParam index_param = {TABLE_NAME, INDEX_TYPE, json_params.dump()}; + milvus::IndexParam index_param = {COLLECTION_NAME, INDEX_TYPE, json_params.dump()}; return index_param; } +void +CountCollection(std::shared_ptr& conn) { + int64_t entity_count = 0; + auto stat = conn->CountCollection(COLLECTION_NAME, entity_count); + std::cout << COLLECTION_NAME << "(" << entity_count << " entities)" << std::endl; +} + +void +ShowCollectionInfo(std::shared_ptr& conn) { + CountCollection(conn); + + milvus::CollectionInfo collection_info; + auto stat = conn->ShowCollectionInfo(COLLECTION_NAME, collection_info); + milvus_sdk::Utils::PrintCollectionInfo(collection_info); + std::cout << "ShowCollectionInfo function call status: " << stat.message() << std::endl; +} + } // namespace void @@ -70,11 +88,11 @@ ClientTest::Test(const std::string& address, const std::string& port) { std::cout << "Connect function call status: " << stat.message() << std::endl; } - { // create table - milvus::TableSchema tb_schema = BuildTableSchema(); - stat = conn->CreateTable(tb_schema); - std::cout << "CreateTable function call status: " << stat.message() << std::endl; - milvus_sdk::Utils::PrintTableSchema(tb_schema); + { // create collection + milvus::CollectionParam tb_schema = BuildCollectionParam(); + stat = conn->CreateCollection(tb_schema); + std::cout << "CreateCollection function call status: " << stat.message() << std::endl; + milvus_sdk::Utils::PrintCollectionParam(tb_schema); } { // create partition @@ -87,7 +105,7 @@ ClientTest::Test(const std::string& address, const std::string& port) { // show partitions milvus::PartitionTagList partition_array; - stat = conn->ShowPartitions(TABLE_NAME, partition_array); + stat = conn->ShowPartitions(COLLECTION_NAME, partition_array); std::cout << partition_array.size() << " partitions created:" << std::endl; for (auto& partition_tag : partition_array) { @@ -96,64 +114,56 @@ ClientTest::Test(const std::string& address, const std::string& port) { } { // insert vectors - milvus_sdk::TimeRecorder rc("All vectors"); + milvus_sdk::TimeRecorder rc("All entities"); for (int i = 0; i < PARTITION_COUNT * 5; i++) { - std::vector record_array; - std::vector record_ids; - int64_t begin_index = i * BATCH_ROW_COUNT; + std::vector entity_array; + std::vector entity_ids; + int64_t begin_index = i * BATCH_ENTITY_COUNT; { // generate vectors - milvus_sdk::TimeRecorder rc("Build vectors No." + std::to_string(i)); - milvus_sdk::Utils::BuildVectors(begin_index, begin_index + BATCH_ROW_COUNT, record_array, record_ids, - TABLE_DIMENSION); + milvus_sdk::TimeRecorder rc("Build entities No." + std::to_string(i)); + milvus_sdk::Utils::BuildEntities(begin_index, + begin_index + BATCH_ENTITY_COUNT, + entity_array, + entity_ids, + COLLECTION_DIMENSION); } - std::string title = "Insert " + std::to_string(record_array.size()) + " vectors No." + std::to_string(i); + std::string title = "Insert " + std::to_string(entity_array.size()) + " entities No." + std::to_string(i); milvus_sdk::TimeRecorder rc(title); - stat = conn->Insert(TABLE_NAME, std::to_string(i % PARTITION_COUNT), record_array, record_ids); + stat = conn->Insert(COLLECTION_NAME, std::to_string(i % PARTITION_COUNT), entity_array, entity_ids); } } { // flush buffer - stat = conn->FlushTable(TABLE_NAME); - std::cout << "FlushTable function call status: " << stat.message() << std::endl; + stat = conn->FlushCollection(COLLECTION_NAME); + std::cout << "FlushCollection function call status: " << stat.message() << std::endl; } - { // table row count - int64_t row_count = 0; - stat = conn->CountTable(TABLE_NAME, row_count); - std::cout << TABLE_NAME << "(" << row_count << " rows)" << std::endl; - } + ShowCollectionInfo(conn); - { // get table information - milvus::TableInfo table_info; - stat = conn->ShowTableInfo(TABLE_NAME, table_info); - milvus_sdk::Utils::PrintTableInfo(table_info); - std::cout << "ShowTableInfo function call status: " << stat.message() << std::endl; - } - - std::vector> search_record_array; + std::vector> search_entity_array; { // build search vectors - std::vector record_array; - std::vector record_ids; - int64_t index = TARGET_PARTITION * BATCH_ROW_COUNT + SEARCH_TARGET; - milvus_sdk::Utils::BuildVectors(index, index + 1, record_array, record_ids, TABLE_DIMENSION); - search_record_array.push_back(std::make_pair(record_ids[0], record_array[0])); + std::vector entity_array; + std::vector entity_ids; + int64_t index = TARGET_PARTITION * BATCH_ENTITY_COUNT + SEARCH_TARGET; + milvus_sdk::Utils::BuildEntities(index, index + 1, entity_array, entity_ids, COLLECTION_DIMENSION); + search_entity_array.push_back(std::make_pair(entity_ids[0], entity_array[0])); } { // search vectors std::cout << "Search in correct partition" << std::endl; std::vector partition_tags = {std::to_string(TARGET_PARTITION)}; milvus::TopKQueryResult topk_query_result; - milvus_sdk::Utils::DoSearch(conn, TABLE_NAME, partition_tags, TOP_K, NPROBE, search_record_array, + milvus_sdk::Utils::DoSearch(conn, COLLECTION_NAME, partition_tags, TOP_K, NPROBE, search_entity_array, topk_query_result); std::cout << "Search in wrong partition" << std::endl; partition_tags = {"0"}; - milvus_sdk::Utils::DoSearch(conn, TABLE_NAME, partition_tags, TOP_K, NPROBE, search_record_array, + milvus_sdk::Utils::DoSearch(conn, COLLECTION_NAME, partition_tags, TOP_K, NPROBE, search_entity_array, topk_query_result); std::cout << "Search by regex matched partition tag" << std::endl; partition_tags = {"\\d"}; - milvus_sdk::Utils::DoSearch(conn, TABLE_NAME, partition_tags, TOP_K, NPROBE, search_record_array, + milvus_sdk::Utils::DoSearch(conn, COLLECTION_NAME, partition_tags, TOP_K, NPROBE, search_entity_array, topk_query_result); } @@ -166,57 +176,42 @@ ClientTest::Test(const std::string& address, const std::string& port) { std::cout << "CreateIndex function call status: " << stat.message() << std::endl; milvus::IndexParam index2; - stat = conn->DescribeIndex(TABLE_NAME, index2); + stat = conn->DescribeIndex(COLLECTION_NAME, index2); std::cout << "DescribeIndex function call status: " << stat.message() << std::endl; milvus_sdk::Utils::PrintIndexParam(index2); } - { // table row count - int64_t row_count = 0; - stat = conn->CountTable(TABLE_NAME, row_count); - std::cout << TABLE_NAME << "(" << row_count << " rows)" << std::endl; - } - - { // get table information - milvus::TableInfo table_info; - stat = conn->ShowTableInfo(TABLE_NAME, table_info); - milvus_sdk::Utils::PrintTableInfo(table_info); - std::cout << "ShowTableInfo function call status: " << stat.message() << std::endl; - } + ShowCollectionInfo(conn); { // drop partition - milvus::PartitionParam param1 = {TABLE_NAME, std::to_string(TARGET_PARTITION)}; + milvus::PartitionParam param1 = {COLLECTION_NAME, std::to_string(TARGET_PARTITION)}; milvus_sdk::Utils::PrintPartitionParam(param1); stat = conn->DropPartition(param1); std::cout << "DropPartition function call status: " << stat.message() << std::endl; } - { // table row count - int64_t row_count = 0; - stat = conn->CountTable(TABLE_NAME, row_count); - std::cout << TABLE_NAME << "(" << row_count << " rows)" << std::endl; - } + CountCollection(conn); - { // search vectors - std::cout << "Search in whole table" << std::endl; + { // search vectors, will get search error since we delete a partition + std::cout << "Search in whole collection after delete one partition" << std::endl; std::vector partition_tags; milvus::TopKQueryResult topk_query_result; - milvus_sdk::Utils::DoSearch(conn, TABLE_NAME, partition_tags, TOP_K, NPROBE, search_record_array, + milvus_sdk::Utils::DoSearch(conn, COLLECTION_NAME, partition_tags, TOP_K, NPROBE, search_entity_array, topk_query_result); } { // drop index - stat = conn->DropIndex(TABLE_NAME); + stat = conn->DropIndex(COLLECTION_NAME); std::cout << "DropIndex function call status: " << stat.message() << std::endl; - int64_t row_count = 0; - stat = conn->CountTable(TABLE_NAME, row_count); - std::cout << TABLE_NAME << "(" << row_count << " rows)" << std::endl; + int64_t entity_count = 0; + stat = conn->CountCollection(COLLECTION_NAME, entity_count); + std::cout << COLLECTION_NAME << "(" << entity_count << " entities)" << std::endl; } - { // drop table - stat = conn->DropTable(TABLE_NAME); - std::cout << "DropTable function call status: " << stat.message() << std::endl; + { // drop collection + stat = conn->DropCollection(COLLECTION_NAME); + std::cout << "DropCollection function call status: " << stat.message() << std::endl; } milvus::Connection::Destroy(conn); diff --git a/sdk/examples/simple/src/ClientTest.cpp b/sdk/examples/simple/src/ClientTest.cpp index 635dabdc16..fb2e2b138d 100644 --- a/sdk/examples/simple/src/ClientTest.cpp +++ b/sdk/examples/simple/src/ClientTest.cpp @@ -19,20 +19,19 @@ #include #include - namespace { -const char* TABLE_NAME = milvus_sdk::Utils::GenTableName().c_str(); +const char* COLLECTION_NAME = milvus_sdk::Utils::GenCollectionName().c_str(); -constexpr int64_t TABLE_DIMENSION = 512; -constexpr int64_t TABLE_INDEX_FILE_SIZE = 1024; -constexpr milvus::MetricType TABLE_METRIC_TYPE = milvus::MetricType::L2; -constexpr int64_t BATCH_ROW_COUNT = 100000; +constexpr int64_t COLLECTION_DIMENSION = 512; +constexpr int64_t COLLECTION_INDEX_FILE_SIZE = 1024; +constexpr milvus::MetricType COLLECTION_METRIC_TYPE = milvus::MetricType::L2; +constexpr int64_t BATCH_ENTITY_COUNT = 100000; constexpr int64_t NQ = 5; constexpr int64_t TOP_K = 10; constexpr int64_t NPROBE = 32; constexpr int64_t SEARCH_TARGET = 5000; // change this value, result is different -constexpr int64_t ADD_VECTOR_LOOP = 5; +constexpr int64_t ADD_ENTITY_LOOP = 5; constexpr milvus::IndexType INDEX_TYPE = milvus::IndexType::IVFSQ8; constexpr int32_t NLIST = 16384; @@ -63,191 +62,201 @@ ClientTest::ShowSdkVersion() { } void -ClientTest::ShowTables(std::vector& tables) { - milvus::Status stat = conn_->ShowTables(tables); - std::cout << "ShowTables function call status: " << stat.message() << std::endl; - std::cout << "All tables: " << std::endl; - for (auto& table : tables) { - int64_t row_count = 0; - stat = conn_->CountTable(table, row_count); - std::cout << "\t" << table << "(" << row_count << " rows)" << std::endl; +ClientTest::ShowCollections(std::vector& collection_array) { + milvus::Status stat = conn_->ShowCollections(collection_array); + std::cout << "ShowCollections function call status: " << stat.message() << std::endl; + std::cout << "All collections: " << std::endl; + for (auto& collection : collection_array) { + int64_t entity_count = 0; + stat = conn_->CountCollection(collection, entity_count); + std::cout << "\t" << collection << "(" << entity_count << " entities)" << std::endl; } } void -ClientTest::CreateTable(const std::string& table_name, int64_t dim, milvus::MetricType type) { - milvus::TableSchema tb_schema = {table_name, dim, TABLE_INDEX_FILE_SIZE, type}; - milvus::Status stat = conn_->CreateTable(tb_schema); - std::cout << "CreateTable function call status: " << stat.message() << std::endl; - milvus_sdk::Utils::PrintTableSchema(tb_schema); +ClientTest::CreateCollection(const std::string& collection_name, int64_t dim, milvus::MetricType type) { + milvus::CollectionParam collection_param = {collection_name, dim, COLLECTION_INDEX_FILE_SIZE, type}; + milvus::Status stat = conn_->CreateCollection(collection_param); + std::cout << "CreateCollection function call status: " << stat.message() << std::endl; + milvus_sdk::Utils::PrintCollectionParam(collection_param); - bool has_table = conn_->HasTable(tb_schema.table_name); + bool has_table = conn_->HasCollection(collection_param.collection_name); if (has_table) { - std::cout << "Table is created" << std::endl; + std::cout << "Collection is created" << std::endl; } } void -ClientTest::DescribeTable(const std::string& table_name) { - milvus::TableSchema tb_schema; - milvus::Status stat = conn_->DescribeTable(table_name, tb_schema); - std::cout << "DescribeTable function call status: " << stat.message() << std::endl; - milvus_sdk::Utils::PrintTableSchema(tb_schema); +ClientTest::DescribeCollection(const std::string& collection_name) { + milvus::CollectionParam collection_param; + milvus::Status stat = conn_->DescribeCollection(collection_name, collection_param); + std::cout << "DescribeCollection function call status: " << stat.message() << std::endl; + milvus_sdk::Utils::PrintCollectionParam(collection_param); } void -ClientTest::InsertVectors(const std::string& table_name, int64_t dim) { - for (int i = 0; i < ADD_VECTOR_LOOP; i++) { - std::vector record_array; +ClientTest::InsertEntities(const std::string& collection_name, int64_t dim) { + for (int i = 0; i < ADD_ENTITY_LOOP; i++) { + std::vector entity_array; std::vector record_ids; - int64_t begin_index = i * BATCH_ROW_COUNT; + int64_t begin_index = i * BATCH_ENTITY_COUNT; { // generate vectors - milvus_sdk::TimeRecorder rc("Build vectors No." + std::to_string(i)); - milvus_sdk::Utils::BuildVectors(begin_index, begin_index + BATCH_ROW_COUNT, record_array, record_ids, dim); + milvus_sdk::TimeRecorder rc("Build entities No." + std::to_string(i)); + milvus_sdk::Utils::BuildEntities(begin_index, + begin_index + BATCH_ENTITY_COUNT, + entity_array, + record_ids, + dim); } - std::string title = "Insert " + std::to_string(record_array.size()) + " vectors No." + std::to_string(i); + std::string title = "Insert " + std::to_string(entity_array.size()) + " entities No." + std::to_string(i); milvus_sdk::TimeRecorder rc(title); - milvus::Status stat = conn_->Insert(table_name, "", record_array, record_ids); - std::cout << "InsertVector function call status: " << stat.message() << std::endl; + milvus::Status stat = conn_->Insert(collection_name, "", entity_array, record_ids); + std::cout << "InsertEntities function call status: " << stat.message() << std::endl; std::cout << "Returned id array count: " << record_ids.size() << std::endl; } } void -ClientTest::BuildSearchVectors(int64_t nq, int64_t dim) { - search_record_array_.clear(); +ClientTest::BuildSearchEntities(int64_t nq, int64_t dim) { + search_entity_array_.clear(); search_id_array_.clear(); for (int64_t i = 0; i < nq; i++) { - std::vector record_array; + std::vector entity_array; std::vector record_ids; - int64_t index = i * BATCH_ROW_COUNT + SEARCH_TARGET; - milvus_sdk::Utils::BuildVectors(index, index + 1, record_array, record_ids, dim); - search_record_array_.push_back(std::make_pair(record_ids[0], record_array[0])); + int64_t index = i * BATCH_ENTITY_COUNT + SEARCH_TARGET; + milvus_sdk::Utils::BuildEntities(index, index + 1, entity_array, record_ids, dim); + search_entity_array_.push_back(std::make_pair(record_ids[0], entity_array[0])); search_id_array_.push_back(record_ids[0]); } } void -ClientTest::Flush(const std::string& table_name) { +ClientTest::Flush(const std::string& collection_name) { milvus_sdk::TimeRecorder rc("Flush"); - milvus::Status stat = conn_->FlushTable(table_name); - std::cout << "FlushTable function call status: " << stat.message() << std::endl; + milvus::Status stat = conn_->FlushCollection(collection_name); + std::cout << "FlushCollection function call status: " << stat.message() << std::endl; } void -ClientTest::ShowTableInfo(const std::string& table_name) { - milvus::TableInfo table_info; - milvus::Status stat = conn_->ShowTableInfo(table_name, table_info); - milvus_sdk::Utils::PrintTableInfo(table_info); - std::cout << "ShowTableInfo function call status: " << stat.message() << std::endl; +ClientTest::ShowCollectionInfo(const std::string& collection_name) { + milvus::CollectionInfo collection_info; + milvus::Status stat = conn_->ShowCollectionInfo(collection_name, collection_info); + milvus_sdk::Utils::PrintCollectionInfo(collection_info); + std::cout << "ShowCollectionInfo function call status: " << stat.message() << std::endl; } void -ClientTest::GetVectorById(const std::string& table_name, int64_t id) { - milvus::RowRecord vector_data; - milvus::Status stat = conn_->GetVectorByID(table_name, id, vector_data); - std::cout << "The vector " << id << " has " << vector_data.float_data.size() << " float elements" << std::endl; - std::cout << "GetVectorByID function call status: " << stat.message() << std::endl; +ClientTest::GetEntityById(const std::string& collection_name, int64_t id) { + milvus::Entity entity; + milvus::Status stat = conn_->GetEntityByID(collection_name, id, entity); + std::cout << "The entity " << id << " has " << entity.float_data.size() << " float elements" << std::endl; + std::cout << "GetEntityById function call status: " << stat.message() << std::endl; } void -ClientTest::SearchVectors(const std::string& table_name, int64_t topk, int64_t nprobe) { +ClientTest::SearchEntities(const std::string& collection_name, int64_t topk, int64_t nprobe) { std::vector partition_tags; milvus::TopKQueryResult topk_query_result; - milvus_sdk::Utils::DoSearch(conn_, table_name, partition_tags, topk, nprobe, search_record_array_, + milvus_sdk::Utils::DoSearch(conn_, collection_name, partition_tags, topk, nprobe, search_entity_array_, topk_query_result); } void -ClientTest::CreateIndex(const std::string& table_name, milvus::IndexType type, int64_t nlist) { +ClientTest::CreateIndex(const std::string& collection_name, milvus::IndexType type, int64_t nlist) { milvus_sdk::TimeRecorder rc("Create index"); std::cout << "Wait until create all index done" << std::endl; JSON json_params = {{"nlist", nlist}}; - milvus::IndexParam index1 = {table_name, type, json_params.dump()}; + milvus::IndexParam index1 = {collection_name, type, json_params.dump()}; milvus_sdk::Utils::PrintIndexParam(index1); milvus::Status stat = conn_->CreateIndex(index1); std::cout << "CreateIndex function call status: " << stat.message() << std::endl; milvus::IndexParam index2; - stat = conn_->DescribeIndex(table_name, index2); + stat = conn_->DescribeIndex(collection_name, index2); std::cout << "DescribeIndex function call status: " << stat.message() << std::endl; milvus_sdk::Utils::PrintIndexParam(index2); } void -ClientTest::PreloadTable(const std::string& table_name) { - milvus::Status stat = conn_->PreloadTable(table_name); - std::cout << "PreloadTable function call status: " << stat.message() << std::endl; +ClientTest::PreloadCollection(const std::string& collection_name) { + milvus::Status stat = conn_->PreloadCollection(collection_name); + std::cout << "PreloadCollection function call status: " << stat.message() << std::endl; } void -ClientTest::DeleteByIds(const std::string& table_name, const std::vector& id_array) { - milvus::Status stat = conn_->DeleteByID(table_name, id_array); +ClientTest::DeleteByIds(const std::string& collection_name, const std::vector& id_array) { + std::cout << "Delete entity: "; + for (auto id : id_array) { + std::cout << "\t" << id; + } + std::cout << std::endl; + + milvus::Status stat = conn_->DeleteByID(collection_name, id_array); std::cout << "DeleteByID function call status: " << stat.message() << std::endl; { milvus_sdk::TimeRecorder rc("Flush"); - stat = conn_->FlushTable(table_name); - std::cout << "FlushTable function call status: " << stat.message() << std::endl; + stat = conn_->FlushCollection(collection_name); + std::cout << "FlushCollection function call status: " << stat.message() << std::endl; } { // compact table milvus_sdk::TimeRecorder rc1("Compact"); - stat = conn_->CompactTable(table_name); - std::cout << "CompactTable function call status: " << stat.message() << std::endl; + stat = conn_->CompactCollection(collection_name); + std::cout << "CompactCollection function call status: " << stat.message() << std::endl; } } void -ClientTest::DropIndex(const std::string& table_name) { - milvus::Status stat = conn_->DropIndex(table_name); +ClientTest::DropIndex(const std::string& collection_name) { + milvus::Status stat = conn_->DropIndex(collection_name); std::cout << "DropIndex function call status: " << stat.message() << std::endl; int64_t row_count = 0; - stat = conn_->CountTable(table_name, row_count); - std::cout << table_name << "(" << row_count << " rows)" << std::endl; + stat = conn_->CountCollection(collection_name, row_count); + std::cout << collection_name << "(" << row_count << " rows)" << std::endl; } void -ClientTest::DropTable(const std::string& table_name) { - milvus::Status stat = conn_->DropTable(table_name); - std::cout << "DropTable function call status: " << stat.message() << std::endl; +ClientTest::DropCollection(const std::string& collection_name) { + milvus::Status stat = conn_->DropCollection(collection_name); + std::cout << "DropCollection function call status: " << stat.message() << std::endl; } void ClientTest::Test() { - std::string table_name = TABLE_NAME; - int64_t dim = TABLE_DIMENSION; - milvus::MetricType metric_type = TABLE_METRIC_TYPE; + std::string collection_name = COLLECTION_NAME; + int64_t dim = COLLECTION_DIMENSION; + milvus::MetricType metric_type = COLLECTION_METRIC_TYPE; ShowServerVersion(); ShowSdkVersion(); std::vector table_array; - ShowTables(table_array); + ShowCollections(table_array); - CreateTable(table_name, dim, metric_type); - DescribeTable(table_name); + CreateCollection(collection_name, dim, metric_type); + DescribeCollection(collection_name); - InsertVectors(table_name, dim); - BuildSearchVectors(NQ, dim); - Flush(table_name); - ShowTableInfo(table_name); + InsertEntities(collection_name, dim); + BuildSearchEntities(NQ, dim); + Flush(collection_name); + ShowCollectionInfo(collection_name); - GetVectorById(table_name, search_id_array_[0]); - SearchVectors(table_name, TOP_K, NPROBE); + GetEntityById(collection_name, search_id_array_[0]); + SearchEntities(collection_name, TOP_K, NPROBE); - CreateIndex(table_name, INDEX_TYPE, NLIST); - ShowTableInfo(table_name); + CreateIndex(collection_name, INDEX_TYPE, NLIST); + ShowCollectionInfo(collection_name); - PreloadTable(table_name); + PreloadCollection(collection_name); std::vector delete_ids = {search_id_array_[0], search_id_array_[1]}; - DeleteByIds(table_name, delete_ids); - SearchVectors(table_name, TOP_K, NPROBE); + DeleteByIds(collection_name, delete_ids); + SearchEntities(collection_name, TOP_K, NPROBE); // this line get two search error since we delete two entities - DropIndex(table_name); - DropTable(table_name); + DropIndex(collection_name); + DropCollection(collection_name); } diff --git a/sdk/examples/simple/src/ClientTest.h b/sdk/examples/simple/src/ClientTest.h index 0c62b29729..2e676353eb 100644 --- a/sdk/examples/simple/src/ClientTest.h +++ b/sdk/examples/simple/src/ClientTest.h @@ -34,49 +34,49 @@ class ClientTest { ShowSdkVersion(); void - ShowTables(std::vector&); + ShowCollections(std::vector&); void - CreateTable(const std::string&, int64_t, milvus::MetricType); + CreateCollection(const std::string&, int64_t, milvus::MetricType); void - DescribeTable(const std::string&); + DescribeCollection(const std::string&); void - InsertVectors(const std::string&, int64_t); + InsertEntities(const std::string&, int64_t); void - BuildSearchVectors(int64_t, int64_t); + BuildSearchEntities(int64_t, int64_t); void Flush(const std::string&); void - ShowTableInfo(const std::string&); + ShowCollectionInfo(const std::string&); void - GetVectorById(const std::string&, int64_t); + GetEntityById(const std::string&, int64_t); void - SearchVectors(const std::string&, int64_t, int64_t); + SearchEntities(const std::string&, int64_t, int64_t); void CreateIndex(const std::string&, milvus::IndexType, int64_t); void - PreloadTable(const std::string&); + PreloadCollection(const std::string&); void - DeleteByIds(const std::string&, const std::vector&); + DeleteByIds(const std::string&, const std::vector& id_array); void DropIndex(const std::string&); void - DropTable(const std::string&); + DropCollection(const std::string&); private: std::shared_ptr conn_; - std::vector> search_record_array_; + std::vector> search_entity_array_; std::vector search_id_array_; }; diff --git a/sdk/examples/utils/Utils.cpp b/sdk/examples/utils/Utils.cpp index c9294cb3aa..ed709f4ffe 100644 --- a/sdk/examples/utils/Utils.cpp +++ b/sdk/examples/utils/Utils.cpp @@ -64,8 +64,8 @@ Utils::Sleep(int seconds) { } const std::string& -Utils::GenTableName() { - static std::string s_id("tbl_" + CurrentTime()); +Utils::GenCollectionName() { + static std::string s_id("C_" + CurrentTime()); return s_id; } @@ -97,19 +97,19 @@ Utils::IndexTypeName(const milvus::IndexType& index_type) { } void -Utils::PrintTableSchema(const milvus::TableSchema& tb_schema) { +Utils::PrintCollectionParam(const milvus::CollectionParam& collection_param) { BLOCK_SPLITER - std::cout << "Table name: " << tb_schema.table_name << std::endl; - std::cout << "Table dimension: " << tb_schema.dimension << std::endl; - std::cout << "Table index file size: " << tb_schema.index_file_size << std::endl; - std::cout << "Table metric type: " << MetricTypeName(tb_schema.metric_type) << std::endl; + std::cout << "Collection name: " << collection_param.collection_name << std::endl; + std::cout << "Collection dimension: " << collection_param.dimension << std::endl; + std::cout << "Collection index file size: " << collection_param.index_file_size << std::endl; + std::cout << "Collection metric type: " << MetricTypeName(collection_param.metric_type) << std::endl; BLOCK_SPLITER } void Utils::PrintPartitionParam(const milvus::PartitionParam& partition_param) { BLOCK_SPLITER - std::cout << "Table name: " << partition_param.table_name << std::endl; + std::cout << "Collection name: " << partition_param.collection_name << std::endl; std::cout << "Partition tag: " << partition_param.partition_tag << std::endl; BLOCK_SPLITER } @@ -117,40 +117,40 @@ Utils::PrintPartitionParam(const milvus::PartitionParam& partition_param) { void Utils::PrintIndexParam(const milvus::IndexParam& index_param) { BLOCK_SPLITER - std::cout << "Index table name: " << index_param.table_name << std::endl; + std::cout << "Index collection name: " << index_param.collection_name << std::endl; std::cout << "Index type: " << IndexTypeName(index_param.index_type) << std::endl; std::cout << "Index extra_params: " << index_param.extra_params << std::endl; BLOCK_SPLITER } void -Utils::BuildVectors(int64_t from, int64_t to, std::vector& vector_record_array, - std::vector& record_ids, int64_t dimension) { +Utils::BuildEntities(int64_t from, int64_t to, std::vector& entity_array, + std::vector& entity_ids, int64_t dimension) { if (to <= from) { return; } - vector_record_array.clear(); - record_ids.clear(); + entity_array.clear(); + entity_ids.clear(); for (int64_t k = from; k < to; k++) { - milvus::RowRecord record; - record.float_data.resize(dimension); + milvus::Entity entity; + entity.float_data.resize(dimension); for (int64_t i = 0; i < dimension; i++) { - record.float_data[i] = (float)(k % (i + 1)); + entity.float_data[i] = (float)(k % (i + 1)); } - vector_record_array.emplace_back(record); - record_ids.push_back(k); + entity_array.emplace_back(entity); + entity_ids.push_back(k); } } void -Utils::PrintSearchResult(const std::vector>& search_record_array, +Utils::PrintSearchResult(const std::vector>& entity_array, const milvus::TopKQueryResult& topk_query_result) { BLOCK_SPLITER std::cout << "Returned result count: " << topk_query_result.size() << std::endl; - if (topk_query_result.size() != search_record_array.size()) { + if (topk_query_result.size() != entity_array.size()) { std::cout << "ERROR: Returned result count not equal nq" << std::endl; return; } @@ -158,8 +158,8 @@ Utils::PrintSearchResult(const std::vector for (size_t i = 0; i < topk_query_result.size(); i++) { const milvus::QueryResult& one_result = topk_query_result[i]; size_t topk = one_result.ids.size(); - auto search_id = search_record_array[i].first; - std::cout << "No." << i << " vector " << search_id << " top " << topk << " search result:" << std::endl; + auto search_id = entity_array[i].first; + std::cout << "No." << i << " entity " << search_id << " top " << topk << " search result:" << std::endl; for (size_t j = 0; j < topk; j++) { std::cout << "\t" << one_result.ids[j] << "\t" << one_result.distances[j] << std::endl; } @@ -168,13 +168,13 @@ Utils::PrintSearchResult(const std::vector } void -Utils::CheckSearchResult(const std::vector>& search_record_array, +Utils::CheckSearchResult(const std::vector>& entity_array, const milvus::TopKQueryResult& topk_query_result) { BLOCK_SPLITER size_t nq = topk_query_result.size(); for (size_t i = 0; i < nq; i++) { const milvus::QueryResult& one_result = topk_query_result[i]; - auto search_id = search_record_array[i].first; + auto search_id = entity_array[i].first; uint64_t match_index = one_result.ids.size(); for (uint64_t index = 0; index < one_result.ids.size(); index++) { @@ -195,15 +195,15 @@ Utils::CheckSearchResult(const std::vector } void -Utils::DoSearch(std::shared_ptr conn, const std::string& table_name, +Utils::DoSearch(std::shared_ptr conn, const std::string& collection_name, const std::vector& partition_tags, int64_t top_k, int64_t nprobe, - const std::vector>& search_record_array, + const std::vector>& entity_array, milvus::TopKQueryResult& topk_query_result) { topk_query_result.clear(); - std::vector record_array; - for (auto& pair : search_record_array) { - record_array.push_back(pair.second); + std::vector temp_entity_array; + for (auto& pair : entity_array) { + temp_entity_array.push_back(pair.second); } { @@ -211,28 +211,33 @@ Utils::DoSearch(std::shared_ptr conn, const std::string& tab JSON json_params = {{"nprobe", nprobe}}; milvus_sdk::TimeRecorder rc("search"); milvus::Status stat = - conn->Search(table_name, partition_tags, record_array, top_k, json_params.dump(), topk_query_result); - std::cout << "SearchVector function call status: " << stat.message() << std::endl; + conn->Search(collection_name, + partition_tags, + temp_entity_array, + top_k, + json_params.dump(), + topk_query_result); + std::cout << "Search function call status: " << stat.message() << std::endl; BLOCK_SPLITER } - PrintSearchResult(search_record_array, topk_query_result); - CheckSearchResult(search_record_array, topk_query_result); + PrintSearchResult(entity_array, topk_query_result); + CheckSearchResult(entity_array, topk_query_result); } void PrintPartitionStat(const milvus::PartitionStat& partition_stat) { - std::cout << "\tPartition " << partition_stat.tag << " row count: " << partition_stat.row_count << std::endl; + std::cout << "\tPartition " << partition_stat.tag << " entity count: " << partition_stat.row_count << std::endl; for (auto& seg_stat : partition_stat.segments_stat) { - std::cout << "\t\tsegment " << seg_stat.segment_name << " row count: " << seg_stat.row_count + std::cout << "\t\tsegment " << seg_stat.segment_name << " entity count: " << seg_stat.row_count << " index: " << seg_stat.index_name << " data size: " << seg_stat.data_size << std::endl; } } void -Utils::PrintTableInfo(const milvus::TableInfo& info) { +Utils::PrintCollectionInfo(const milvus::CollectionInfo& info) { BLOCK_SPLITER - std::cout << "Table " << " total row count: " << info.total_row_count << std::endl; + std::cout << "Collection " << " total entity count: " << info.total_row_count << std::endl; for (const milvus::PartitionStat& partition_stat : info.partitions_stat) { PrintPartitionStat(partition_stat); } diff --git a/sdk/examples/utils/Utils.h b/sdk/examples/utils/Utils.h index 4f33dafc33..f83ee069b7 100644 --- a/sdk/examples/utils/Utils.h +++ b/sdk/examples/utils/Utils.h @@ -32,7 +32,7 @@ class Utils { CurrentTmDate(int64_t offset_day = 0); static const std::string& - GenTableName(); + GenCollectionName(); static void Sleep(int seconds); @@ -44,7 +44,7 @@ class Utils { IndexTypeName(const milvus::IndexType& index_type); static void - PrintTableSchema(const milvus::TableSchema& tb_schema); + PrintCollectionParam(const milvus::CollectionParam& collection_param); static void PrintPartitionParam(const milvus::PartitionParam& partition_param); @@ -53,25 +53,25 @@ class Utils { PrintIndexParam(const milvus::IndexParam& index_param); static void - BuildVectors(int64_t from, int64_t to, std::vector& vector_record_array, - std::vector& record_ids, int64_t dimension); + BuildEntities(int64_t from, int64_t to, std::vector& entity_array, + std::vector& entity_ids, int64_t dimension); static void - PrintSearchResult(const std::vector>& search_record_array, + PrintSearchResult(const std::vector>& entity_array, const milvus::TopKQueryResult& topk_query_result); static void - CheckSearchResult(const std::vector>& search_record_array, + CheckSearchResult(const std::vector>& entity_array, const milvus::TopKQueryResult& topk_query_result); static void - DoSearch(std::shared_ptr conn, const std::string& table_name, + DoSearch(std::shared_ptr conn, const std::string& collection_name, const std::vector& partition_tags, int64_t top_k, int64_t nprobe, - const std::vector>& search_record_array, + const std::vector>& entity_array, milvus::TopKQueryResult& topk_query_result); static void - PrintTableInfo(const milvus::TableInfo& info); + PrintCollectionInfo(const milvus::CollectionInfo& collection_info); }; } // namespace milvus_sdk diff --git a/sdk/grpc/ClientProxy.cpp b/sdk/grpc/ClientProxy.cpp index 6986355b4e..219f78e4ad 100644 --- a/sdk/grpc/ClientProxy.cpp +++ b/sdk/grpc/ClientProxy.cpp @@ -31,12 +31,12 @@ UriCheck(const std::string& uri) { template void -ConstructSearchParam(const std::string& table_name, +ConstructSearchParam(const std::string& collection_name, const std::vector& partition_tag_array, int64_t topk, const std::string& extra_params, T& search_param) { - search_param.set_table_name(table_name); + search_param.set_table_name(collection_name); search_param.set_topk(topk); milvus::grpc::KeyValuePair* kv = search_param.add_extra_params(); kv->set_key(EXTRA_PARAM_KEY); @@ -48,7 +48,7 @@ ConstructSearchParam(const std::string& table_name, } void -CopyRowRecord(::milvus::grpc::RowRecord* target, const RowRecord& src) { +CopyRowRecord(::milvus::grpc::RowRecord* target, const Entity& src) { if (!src.float_data.empty()) { auto vector_data = target->mutable_float_data(); vector_data->Resize(static_cast(src.float_data.size()), 0.0); @@ -86,7 +86,7 @@ ClientProxy::Connect(const ConnectParam& param) { return Status::OK(); } - std::string reason = "connect Failed!"; + std::string reason = "Connect failed!"; connected_ = false; return Status(StatusCode::NotConnected, reason); } @@ -111,7 +111,7 @@ ClientProxy::Connected() const { std::string info; return client_ptr_->Cmd("", info); } catch (std::exception& ex) { - return Status(StatusCode::NotConnected, "connection lost: " + std::string(ex.what())); + return Status(StatusCode::NotConnected, "Connection lost: " + std::string(ex.what())); } } @@ -132,262 +132,6 @@ ClientProxy::ClientVersion() const { return MILVUS_SDK_VERSION; } -Status -ClientProxy::CreateTable(const TableSchema& param) { - try { - ::milvus::grpc::TableSchema schema; - schema.set_table_name(param.table_name); - schema.set_dimension(param.dimension); - schema.set_index_file_size(param.index_file_size); - schema.set_metric_type(static_cast(param.metric_type)); - - return client_ptr_->CreateTable(schema); - } catch (std::exception& ex) { - return Status(StatusCode::UnknownError, "Failed to create table: " + std::string(ex.what())); - } -} - -bool -ClientProxy::HasTable(const std::string& table_name) { - Status status = Status::OK(); - ::milvus::grpc::TableName grpc_table_name; - grpc_table_name.set_table_name(table_name); - bool result = client_ptr_->HasTable(grpc_table_name, status); - return result; -} - -Status -ClientProxy::DropTable(const std::string& table_name) { - try { - ::milvus::grpc::TableName grpc_table_name; - grpc_table_name.set_table_name(table_name); - return client_ptr_->DropTable(grpc_table_name); - } catch (std::exception& ex) { - return Status(StatusCode::UnknownError, "Failed to drop table: " + std::string(ex.what())); - } -} - -Status -ClientProxy::CreateIndex(const IndexParam& index_param) { - try { - ::milvus::grpc::IndexParam grpc_index_param; - grpc_index_param.set_table_name(index_param.table_name); - grpc_index_param.set_index_type(static_cast(index_param.index_type)); - milvus::grpc::KeyValuePair* kv = grpc_index_param.add_extra_params(); - kv->set_key(EXTRA_PARAM_KEY); - kv->set_value(index_param.extra_params); - return client_ptr_->CreateIndex(grpc_index_param); - } catch (std::exception& ex) { - return Status(StatusCode::UnknownError, "Failed to build index: " + std::string(ex.what())); - } -} - -Status -ClientProxy::Insert(const std::string& table_name, const std::string& partition_tag, - const std::vector& record_array, std::vector& id_array) { - Status status = Status::OK(); - try { - ::milvus::grpc::InsertParam insert_param; - insert_param.set_table_name(table_name); - insert_param.set_partition_tag(partition_tag); - - for (auto& record : record_array) { - ::milvus::grpc::RowRecord* grpc_record = insert_param.add_row_record_array(); - CopyRowRecord(grpc_record, record); - } - - // Single thread - ::milvus::grpc::VectorIds vector_ids; - if (!id_array.empty()) { - /* set user's ids */ - auto row_ids = insert_param.mutable_row_id_array(); - row_ids->Resize(static_cast(id_array.size()), -1); - memcpy(row_ids->mutable_data(), id_array.data(), id_array.size() * sizeof(int64_t)); - status = client_ptr_->Insert(insert_param, vector_ids); - } else { - status = client_ptr_->Insert(insert_param, vector_ids); - /* return Milvus generated ids back to user */ - id_array.insert(id_array.end(), vector_ids.vector_id_array().begin(), vector_ids.vector_id_array().end()); - } - } catch (std::exception& ex) { - return Status(StatusCode::UnknownError, "Failed to add vector: " + std::string(ex.what())); - } - - return status; -} - -Status -ClientProxy::GetVectorByID(const std::string& table_name, int64_t vector_id, RowRecord& vector_data) { - try { - ::milvus::grpc::VectorIdentity vector_identity; - vector_identity.set_table_name(table_name); - vector_identity.set_id(vector_id); - - ::milvus::grpc::VectorData grpc_data; - Status status = client_ptr_->GetVectorByID(vector_identity, grpc_data); - if (!status.ok()) { - return status; - } - - int float_size = grpc_data.vector_data().float_data_size(); - if (float_size > 0) { - vector_data.float_data.resize(float_size); - memcpy(vector_data.float_data.data(), grpc_data.vector_data().float_data().data(), - float_size * sizeof(float)); - } - - auto byte_size = grpc_data.vector_data().binary_data().length(); - if (byte_size > 0) { - vector_data.binary_data.resize(byte_size); - memcpy(vector_data.binary_data.data(), grpc_data.vector_data().binary_data().data(), byte_size); - } - - return status; - } catch (std::exception& ex) { - return Status(StatusCode::UnknownError, "Failed to get vector by id: " + std::string(ex.what())); - } -} - -Status -ClientProxy::GetIDsInSegment(const std::string& table_name, const std::string& segment_name, - std::vector& id_array) { - try { - ::milvus::grpc::GetVectorIDsParam param; - param.set_table_name(table_name); - param.set_segment_name(segment_name); - - ::milvus::grpc::VectorIds vector_ids; - Status status = client_ptr_->GetIDsInSegment(param, vector_ids); - if (!status.ok()) { - return status; - } - - id_array.insert(id_array.end(), vector_ids.vector_id_array().begin(), vector_ids.vector_id_array().end()); - - return status; - } catch (std::exception& ex) { - return Status(StatusCode::UnknownError, "Failed to get vector by id: " + std::string(ex.what())); - } -} - -Status -ClientProxy::Search(const std::string& table_name, const std::vector& partition_tag_array, - const std::vector& query_record_array, int64_t topk, const std::string& extra_params, - TopKQueryResult& topk_query_result) { - try { - // step 1: convert vectors data - ::milvus::grpc::SearchParam search_param; - ConstructSearchParam(table_name, - partition_tag_array, - topk, - extra_params, - search_param); - - for (auto& record : query_record_array) { - ::milvus::grpc::RowRecord* row_record = search_param.add_query_record_array(); - CopyRowRecord(row_record, record); - } - - // step 2: search vectors - ::milvus::grpc::TopKQueryResult result; - Status status = client_ptr_->Search(search_param, result); - if (result.row_num() == 0) { - return status; - } - - // step 3: convert result array - topk_query_result.reserve(result.row_num()); - int64_t nq = result.row_num(); - int64_t topk = result.ids().size() / nq; - for (int64_t i = 0; i < result.row_num(); i++) { - milvus::QueryResult one_result; - one_result.ids.resize(topk); - one_result.distances.resize(topk); - memcpy(one_result.ids.data(), result.ids().data() + topk * i, topk * sizeof(int64_t)); - memcpy(one_result.distances.data(), result.distances().data() + topk * i, topk * sizeof(float)); - topk_query_result.emplace_back(one_result); - } - - return status; - } catch (std::exception& ex) { - return Status(StatusCode::UnknownError, "Failed to search vectors: " + std::string(ex.what())); - } -} - -Status -ClientProxy::DescribeTable(const std::string& table_name, TableSchema& table_schema) { - try { - ::milvus::grpc::TableSchema grpc_schema; - - Status status = client_ptr_->DescribeTable(table_name, grpc_schema); - - table_schema.table_name = grpc_schema.table_name(); - table_schema.dimension = grpc_schema.dimension(); - table_schema.index_file_size = grpc_schema.index_file_size(); - table_schema.metric_type = static_cast(grpc_schema.metric_type()); - - return status; - } catch (std::exception& ex) { - return Status(StatusCode::UnknownError, "Failed to describe table: " + std::string(ex.what())); - } -} - -Status -ClientProxy::CountTable(const std::string& table_name, int64_t& row_count) { - try { - Status status; - ::milvus::grpc::TableName grpc_table_name; - grpc_table_name.set_table_name(table_name); - row_count = client_ptr_->CountTable(grpc_table_name, status); - return status; - } catch (std::exception& ex) { - return Status(StatusCode::UnknownError, "Failed to show tables: " + std::string(ex.what())); - } -} - -Status -ClientProxy::ShowTables(std::vector& table_array) { - try { - Status status; - milvus::grpc::TableNameList table_name_list; - status = client_ptr_->ShowTables(table_name_list); - - table_array.resize(table_name_list.table_names_size()); - for (uint64_t i = 0; i < table_name_list.table_names_size(); ++i) { - table_array[i] = table_name_list.table_names(i); - } - return status; - } catch (std::exception& ex) { - return Status(StatusCode::UnknownError, "Failed to show tables: " + std::string(ex.what())); - } -} - -Status -ClientProxy::ShowTableInfo(const std::string& table_name, TableInfo& table_info) { - try { - Status status; - ::milvus::grpc::TableName grpc_table_name; - grpc_table_name.set_table_name(table_name); - milvus::grpc::TableInfo grpc_table_info; - status = client_ptr_->ShowTableInfo(grpc_table_name, grpc_table_info); - - // get native info - table_info.total_row_count = grpc_table_info.total_row_count(); - - // get partitions info - for (int i = 0; i < grpc_table_info.partitions_stat_size(); i++) { - auto& grpc_partition_stat = grpc_table_info.partitions_stat(i); - PartitionStat partition_stat; - ConstructPartitionStat(grpc_partition_stat, partition_stat); - table_info.partitions_stat.emplace_back(partition_stat); - } - - return status; - } catch (std::exception& ex) { - return Status(StatusCode::UnknownError, "Failed to show table info: " + std::string(ex.what())); - } -} - std::string ClientProxy::ServerVersion() const { Status status = Status::OK(); @@ -415,112 +159,6 @@ ClientProxy::ServerStatus() const { } } -Status -ClientProxy::DeleteByID(const std::string& table_name, const std::vector& id_array) { - try { - ::milvus::grpc::DeleteByIDParam delete_by_id_param; - delete_by_id_param.set_table_name(table_name); - - for (auto id : id_array) { - delete_by_id_param.add_id_array(id); - } - - return client_ptr_->DeleteByID(delete_by_id_param); - } catch (std::exception& ex) { - return Status(StatusCode::UnknownError, "Failed to delete by range: " + std::string(ex.what())); - } -} - -Status -ClientProxy::PreloadTable(const std::string& table_name) const { - try { - ::milvus::grpc::TableName grpc_table_name; - grpc_table_name.set_table_name(table_name); - Status status = client_ptr_->PreloadTable(grpc_table_name); - return status; - } catch (std::exception& ex) { - return Status(StatusCode::UnknownError, "Failed to preload tables: " + std::string(ex.what())); - } -} - -Status -ClientProxy::DescribeIndex(const std::string& table_name, IndexParam& index_param) const { - try { - ::milvus::grpc::TableName grpc_table_name; - grpc_table_name.set_table_name(table_name); - - ::milvus::grpc::IndexParam grpc_index_param; - Status status = client_ptr_->DescribeIndex(grpc_table_name, grpc_index_param); - index_param.index_type = static_cast(grpc_index_param.index_type()); - - for (int i = 0; i < grpc_index_param.extra_params_size(); i++) { - const milvus::grpc::KeyValuePair& kv = grpc_index_param.extra_params(i); - if (kv.key() == EXTRA_PARAM_KEY) { - index_param.extra_params = kv.value(); - } - } - - return status; - } catch (std::exception& ex) { - return Status(StatusCode::UnknownError, "Failed to describe index: " + std::string(ex.what())); - } -} - -Status -ClientProxy::DropIndex(const std::string& table_name) const { - try { - ::milvus::grpc::TableName grpc_table_name; - grpc_table_name.set_table_name(table_name); - Status status = client_ptr_->DropIndex(grpc_table_name); - return status; - } catch (std::exception& ex) { - return Status(StatusCode::UnknownError, "Failed to drop index: " + std::string(ex.what())); - } -} - -Status -ClientProxy::CreatePartition(const PartitionParam& partition_param) { - try { - ::milvus::grpc::PartitionParam grpc_partition_param; - grpc_partition_param.set_table_name(partition_param.table_name); - grpc_partition_param.set_tag(partition_param.partition_tag); - Status status = client_ptr_->CreatePartition(grpc_partition_param); - return status; - } catch (std::exception& ex) { - return Status(StatusCode::UnknownError, "Failed to create partition: " + std::string(ex.what())); - } -} - -Status -ClientProxy::ShowPartitions(const std::string& table_name, PartitionTagList& partition_tag_array) const { - try { - ::milvus::grpc::TableName grpc_table_name; - grpc_table_name.set_table_name(table_name); - ::milvus::grpc::PartitionList grpc_partition_list; - Status status = client_ptr_->ShowPartitions(grpc_table_name, grpc_partition_list); - partition_tag_array.resize(grpc_partition_list.partition_tag_array_size()); - for (uint64_t i = 0; i < grpc_partition_list.partition_tag_array_size(); ++i) { - partition_tag_array[i] = grpc_partition_list.partition_tag_array(i); - } - return status; - } catch (std::exception& ex) { - return Status(StatusCode::UnknownError, "Failed to show partitions: " + std::string(ex.what())); - } -} - -Status -ClientProxy::DropPartition(const PartitionParam& partition_param) { - try { - ::milvus::grpc::PartitionParam grpc_partition_param; - grpc_partition_param.set_table_name(partition_param.table_name); - grpc_partition_param.set_tag(partition_param.partition_tag); - Status status = client_ptr_->DropPartition(grpc_partition_param); - return status; - } catch (std::exception& ex) { - return Status(StatusCode::UnknownError, "Failed to drop partition: " + std::string(ex.what())); - } -} - Status ClientProxy::GetConfig(const std::string& node_name, std::string& value) const { try { @@ -541,12 +179,373 @@ ClientProxy::SetConfig(const std::string& node_name, const std::string& value) c } Status -ClientProxy::FlushTable(const std::string& table_name) { +ClientProxy::CreateCollection(const CollectionParam& param) { + try { + ::milvus::grpc::TableSchema schema; + schema.set_table_name(param.collection_name); + schema.set_dimension(param.dimension); + schema.set_index_file_size(param.index_file_size); + schema.set_metric_type(static_cast(param.metric_type)); + + return client_ptr_->CreateTable(schema); + } catch (std::exception& ex) { + return Status(StatusCode::UnknownError, "Failed to create collection: " + std::string(ex.what())); + } +} + +bool +ClientProxy::HasCollection(const std::string& collection_name) { + Status status = Status::OK(); + ::milvus::grpc::TableName grpc_collection_name; + grpc_collection_name.set_table_name(collection_name); + bool result = client_ptr_->HasTable(grpc_collection_name, status); + return result; +} + +Status +ClientProxy::DropCollection(const std::string& collection_name) { + try { + ::milvus::grpc::TableName grpc_collection_name; + grpc_collection_name.set_table_name(collection_name); + return client_ptr_->DropTable(grpc_collection_name); + } catch (std::exception& ex) { + return Status(StatusCode::UnknownError, "Failed to drop collection: " + std::string(ex.what())); + } +} + +Status +ClientProxy::CreateIndex(const IndexParam& index_param) { + try { + ::milvus::grpc::IndexParam grpc_index_param; + grpc_index_param.set_table_name(index_param.collection_name); + grpc_index_param.set_index_type(static_cast(index_param.index_type)); + milvus::grpc::KeyValuePair* kv = grpc_index_param.add_extra_params(); + kv->set_key(EXTRA_PARAM_KEY); + kv->set_value(index_param.extra_params); + return client_ptr_->CreateIndex(grpc_index_param); + } catch (std::exception& ex) { + return Status(StatusCode::UnknownError, "Failed to build index: " + std::string(ex.what())); + } +} + +Status +ClientProxy::Insert(const std::string& collection_name, const std::string& partition_tag, + const std::vector& entity_array, std::vector& id_array) { + Status status = Status::OK(); + try { + ::milvus::grpc::InsertParam insert_param; + insert_param.set_table_name(collection_name); + insert_param.set_partition_tag(partition_tag); + + for (auto& entity : entity_array) { + ::milvus::grpc::RowRecord* grpc_record = insert_param.add_row_record_array(); + CopyRowRecord(grpc_record, entity); + } + + // Single thread + ::milvus::grpc::VectorIds vector_ids; + if (!id_array.empty()) { + /* set user's ids */ + auto row_ids = insert_param.mutable_row_id_array(); + row_ids->Resize(static_cast(id_array.size()), -1); + memcpy(row_ids->mutable_data(), id_array.data(), id_array.size() * sizeof(int64_t)); + status = client_ptr_->Insert(insert_param, vector_ids); + } else { + status = client_ptr_->Insert(insert_param, vector_ids); + /* return Milvus generated ids back to user */ + id_array.insert(id_array.end(), vector_ids.vector_id_array().begin(), vector_ids.vector_id_array().end()); + } + } catch (std::exception& ex) { + return Status(StatusCode::UnknownError, "Failed to add entities: " + std::string(ex.what())); + } + + return status; +} + +Status +ClientProxy::GetEntityByID(const std::string& collection_name, int64_t entity_id, Entity& entity_data) { + try { + ::milvus::grpc::VectorIdentity vector_identity; + vector_identity.set_table_name(collection_name); + vector_identity.set_id(entity_id); + + ::milvus::grpc::VectorData grpc_data; + Status status = client_ptr_->GetVectorByID(vector_identity, grpc_data); + if (!status.ok()) { + return status; + } + + int float_size = grpc_data.vector_data().float_data_size(); + if (float_size > 0) { + entity_data.float_data.resize(float_size); + memcpy(entity_data.float_data.data(), grpc_data.vector_data().float_data().data(), + float_size * sizeof(float)); + } + + auto byte_size = grpc_data.vector_data().binary_data().length(); + if (byte_size > 0) { + entity_data.binary_data.resize(byte_size); + memcpy(entity_data.binary_data.data(), grpc_data.vector_data().binary_data().data(), byte_size); + } + + return status; + } catch (std::exception& ex) { + return Status(StatusCode::UnknownError, "Failed to get entity by id: " + std::string(ex.what())); + } +} + +Status +ClientProxy::GetIDsInSegment(const std::string& collection_name, const std::string& segment_name, + std::vector& id_array) { + try { + ::milvus::grpc::GetVectorIDsParam param; + param.set_table_name(collection_name); + param.set_segment_name(segment_name); + + ::milvus::grpc::VectorIds vector_ids; + Status status = client_ptr_->GetIDsInSegment(param, vector_ids); + if (!status.ok()) { + return status; + } + + id_array.insert(id_array.end(), vector_ids.vector_id_array().begin(), vector_ids.vector_id_array().end()); + + return status; + } catch (std::exception& ex) { + return Status(StatusCode::UnknownError, "Failed to get ids from segment: " + std::string(ex.what())); + } +} + +Status +ClientProxy::Search(const std::string& collection_name, const std::vector& partition_tag_array, + const std::vector& entity_array, int64_t topk, const std::string& extra_params, + TopKQueryResult& topk_query_result) { + try { + // step 1: convert vectors data + ::milvus::grpc::SearchParam search_param; + ConstructSearchParam(collection_name, + partition_tag_array, + topk, + extra_params, + search_param); + + for (auto& entity : entity_array) { + ::milvus::grpc::RowRecord* row_record = search_param.add_query_record_array(); + CopyRowRecord(row_record, entity); + } + + // step 2: search vectors + ::milvus::grpc::TopKQueryResult result; + Status status = client_ptr_->Search(search_param, result); + if (result.row_num() == 0) { + return status; + } + + // step 3: convert result array + topk_query_result.reserve(result.row_num()); + int64_t nq = result.row_num(); + int64_t topk = result.ids().size() / nq; + for (int64_t i = 0; i < result.row_num(); i++) { + milvus::QueryResult one_result; + one_result.ids.resize(topk); + one_result.distances.resize(topk); + memcpy(one_result.ids.data(), result.ids().data() + topk * i, topk * sizeof(int64_t)); + memcpy(one_result.distances.data(), result.distances().data() + topk * i, topk * sizeof(float)); + topk_query_result.emplace_back(one_result); + } + + return status; + } catch (std::exception& ex) { + return Status(StatusCode::UnknownError, "Failed to search entities: " + std::string(ex.what())); + } +} + +Status +ClientProxy::DescribeCollection(const std::string& collection_name, CollectionParam& collection_param) { + try { + ::milvus::grpc::TableSchema grpc_schema; + + Status status = client_ptr_->DescribeTable(collection_name, grpc_schema); + + collection_param.collection_name = grpc_schema.table_name(); + collection_param.dimension = grpc_schema.dimension(); + collection_param.index_file_size = grpc_schema.index_file_size(); + collection_param.metric_type = static_cast(grpc_schema.metric_type()); + + return status; + } catch (std::exception& ex) { + return Status(StatusCode::UnknownError, "Failed to describe collection: " + std::string(ex.what())); + } +} + +Status +ClientProxy::CountCollection(const std::string& collection_name, int64_t& row_count) { + try { + Status status; + ::milvus::grpc::TableName grpc_collection_name; + grpc_collection_name.set_table_name(collection_name); + row_count = client_ptr_->CountTable(grpc_collection_name, status); + return status; + } catch (std::exception& ex) { + return Status(StatusCode::UnknownError, "Failed to count collection: " + std::string(ex.what())); + } +} + +Status +ClientProxy::ShowCollections(std::vector& collection_array) { + try { + Status status; + milvus::grpc::TableNameList collection_name_list; + status = client_ptr_->ShowTables(collection_name_list); + + collection_array.resize(collection_name_list.table_names_size()); + for (uint64_t i = 0; i < collection_name_list.table_names_size(); ++i) { + collection_array[i] = collection_name_list.table_names(i); + } + return status; + } catch (std::exception& ex) { + return Status(StatusCode::UnknownError, "Failed to show collections: " + std::string(ex.what())); + } +} + +Status +ClientProxy::ShowCollectionInfo(const std::string& collection_name, CollectionInfo& collection_info) { + try { + Status status; + ::milvus::grpc::TableName grpc_collection_name; + grpc_collection_name.set_table_name(collection_name); + milvus::grpc::TableInfo grpc_collection_info; + status = client_ptr_->ShowTableInfo(grpc_collection_name, grpc_collection_info); + + // get native info + collection_info.total_row_count = grpc_collection_info.total_row_count(); + + // get partitions info + for (int i = 0; i < grpc_collection_info.partitions_stat_size(); i++) { + auto& grpc_partition_stat = grpc_collection_info.partitions_stat(i); + PartitionStat partition_stat; + ConstructPartitionStat(grpc_partition_stat, partition_stat); + collection_info.partitions_stat.emplace_back(partition_stat); + } + + return status; + } catch (std::exception& ex) { + return Status(StatusCode::UnknownError, "Failed to show collection info: " + std::string(ex.what())); + } +} + +Status +ClientProxy::DeleteByID(const std::string& collection_name, const std::vector& id_array) { + try { + ::milvus::grpc::DeleteByIDParam delete_by_id_param; + delete_by_id_param.set_table_name(collection_name); + for (auto id : id_array) { + delete_by_id_param.add_id_array(id); + } + + return client_ptr_->DeleteByID(delete_by_id_param); + } catch (std::exception& ex) { + return Status(StatusCode::UnknownError, "Failed to delete entity id: " + std::string(ex.what())); + } +} + +Status +ClientProxy::PreloadCollection(const std::string& collection_name) const { + try { + ::milvus::grpc::TableName grpc_collection_name; + grpc_collection_name.set_table_name(collection_name); + Status status = client_ptr_->PreloadTable(grpc_collection_name); + return status; + } catch (std::exception& ex) { + return Status(StatusCode::UnknownError, "Failed to preload collection: " + std::string(ex.what())); + } +} + +Status +ClientProxy::DescribeIndex(const std::string& collection_name, IndexParam& index_param) const { + try { + ::milvus::grpc::TableName grpc_collection_name; + grpc_collection_name.set_table_name(collection_name); + + ::milvus::grpc::IndexParam grpc_index_param; + Status status = client_ptr_->DescribeIndex(grpc_collection_name, grpc_index_param); + index_param.index_type = static_cast(grpc_index_param.index_type()); + + for (int i = 0; i < grpc_index_param.extra_params_size(); i++) { + const milvus::grpc::KeyValuePair& kv = grpc_index_param.extra_params(i); + if (kv.key() == EXTRA_PARAM_KEY) { + index_param.extra_params = kv.value(); + } + } + + return status; + } catch (std::exception& ex) { + return Status(StatusCode::UnknownError, "Failed to describe index: " + std::string(ex.what())); + } +} + +Status +ClientProxy::DropIndex(const std::string& collection_name) const { + try { + ::milvus::grpc::TableName grpc_collection_name; + grpc_collection_name.set_table_name(collection_name); + Status status = client_ptr_->DropIndex(grpc_collection_name); + return status; + } catch (std::exception& ex) { + return Status(StatusCode::UnknownError, "Failed to drop index: " + std::string(ex.what())); + } +} + +Status +ClientProxy::CreatePartition(const PartitionParam& partition_param) { + try { + ::milvus::grpc::PartitionParam grpc_partition_param; + grpc_partition_param.set_table_name(partition_param.collection_name); + grpc_partition_param.set_tag(partition_param.partition_tag); + Status status = client_ptr_->CreatePartition(grpc_partition_param); + return status; + } catch (std::exception& ex) { + return Status(StatusCode::UnknownError, "Failed to create partition: " + std::string(ex.what())); + } +} + +Status +ClientProxy::ShowPartitions(const std::string& collection_name, PartitionTagList& partition_tag_array) const { + try { + ::milvus::grpc::TableName grpc_collection_name; + grpc_collection_name.set_table_name(collection_name); + ::milvus::grpc::PartitionList grpc_partition_list; + Status status = client_ptr_->ShowPartitions(grpc_collection_name, grpc_partition_list); + partition_tag_array.resize(grpc_partition_list.partition_tag_array_size()); + for (uint64_t i = 0; i < grpc_partition_list.partition_tag_array_size(); ++i) { + partition_tag_array[i] = grpc_partition_list.partition_tag_array(i); + } + return status; + } catch (std::exception& ex) { + return Status(StatusCode::UnknownError, "Failed to show partitions: " + std::string(ex.what())); + } +} + +Status +ClientProxy::DropPartition(const PartitionParam& partition_param) { + try { + ::milvus::grpc::PartitionParam grpc_partition_param; + grpc_partition_param.set_table_name(partition_param.collection_name); + grpc_partition_param.set_tag(partition_param.partition_tag); + Status status = client_ptr_->DropPartition(grpc_partition_param); + return status; + } catch (std::exception& ex) { + return Status(StatusCode::UnknownError, "Failed to drop partition: " + std::string(ex.what())); + } +} + +Status +ClientProxy::FlushCollection(const std::string& collection_name) { try { std::string dummy; - return client_ptr_->Flush(table_name); + return client_ptr_->Flush(collection_name); } catch (std::exception& ex) { - return Status(StatusCode::UnknownError, "Failed to flush"); + return Status(StatusCode::UnknownError, "Failed to flush collection"); } } @@ -556,19 +555,19 @@ ClientProxy::Flush() { std::string dummy; return client_ptr_->Flush(""); } catch (std::exception& ex) { - return Status(StatusCode::UnknownError, "Failed to flush"); + return Status(StatusCode::UnknownError, "Failed to flush collections"); } } Status -ClientProxy::CompactTable(const std::string& table_name) { +ClientProxy::CompactCollection(const std::string& collection_name) { try { - ::milvus::grpc::TableName grpc_table_name; - grpc_table_name.set_table_name(table_name); - Status status = client_ptr_->Compact(grpc_table_name); + ::milvus::grpc::TableName grpc_collection_name; + grpc_collection_name.set_table_name(collection_name); + Status status = client_ptr_->Compact(grpc_collection_name); return status; } catch (std::exception& ex) { - return Status(StatusCode::UnknownError, "Failed to compact table: " + std::string(ex.what())); + return Status(StatusCode::UnknownError, "Failed to compact collection: " + std::string(ex.what())); } } diff --git a/sdk/grpc/ClientProxy.h b/sdk/grpc/ClientProxy.h index 2c4a195072..f395ce32ca 100644 --- a/sdk/grpc/ClientProxy.h +++ b/sdk/grpc/ClientProxy.h @@ -24,7 +24,7 @@ class ClientProxy : public Connection { public: // Implementations of the Connection interface Status - Connect(const ConnectParam& param) override; + Connect(const ConnectParam& connect_param) override; Status Connect(const std::string& uri) override; @@ -35,46 +35,6 @@ class ClientProxy : public Connection { Status Disconnect() override; - Status - CreateTable(const TableSchema& param) override; - - bool - HasTable(const std::string& table_name) override; - - Status - DropTable(const std::string& table_name) override; - - Status - CreateIndex(const IndexParam& index_param) override; - - Status - Insert(const std::string& table_name, const std::string& partition_tag, const std::vector& record_array, - std::vector& id_array) override; - - Status - GetVectorByID(const std::string& table_name, int64_t vector_id, RowRecord& vector_data) override; - - Status - GetIDsInSegment(const std::string& table_name, const std::string& segment_name, - std::vector& id_array) override; - - Status - Search(const std::string& table_name, const std::vector& partition_tag_array, - const std::vector& query_record_array, int64_t topk, const std::string& extra_params, - TopKQueryResult& topk_query_result) override; - - Status - DescribeTable(const std::string& table_name, TableSchema& table_schema) override; - - Status - CountTable(const std::string& table_name, int64_t& row_count) override; - - Status - ShowTables(std::vector& table_array) override; - - Status - ShowTableInfo(const std::string& table_name, TableInfo& table_info) override; - std::string ClientVersion() const override; @@ -84,27 +44,6 @@ class ClientProxy : public Connection { std::string ServerStatus() const override; - Status - DeleteByID(const std::string& table_name, const std::vector& id_array) override; - - Status - PreloadTable(const std::string& table_name) const override; - - Status - DescribeIndex(const std::string& table_name, IndexParam& index_param) const override; - - Status - DropIndex(const std::string& table_name) const override; - - Status - CreatePartition(const PartitionParam& partition_param) override; - - Status - ShowPartitions(const std::string& table_name, PartitionTagList& partition_tag_array) const override; - - Status - DropPartition(const PartitionParam& partition_param) override; - Status GetConfig(const std::string& node_name, std::string& value) const override; @@ -112,13 +51,76 @@ class ClientProxy : public Connection { SetConfig(const std::string& node_name, const std::string& value) const override; Status - FlushTable(const std::string& table_name) override; + CreateCollection(const CollectionParam& param) override; + + bool + HasCollection(const std::string& collection_name) override; + + Status + DropCollection(const std::string& collection_name) override; + + Status + CreateIndex(const IndexParam& index_param) override; + + Status + Insert(const std::string& collection_name, + const std::string& partition_tag, + const std::vector& entity_array, + std::vector& id_array) override; + + Status + GetEntityByID(const std::string& collection_name, int64_t entity_id, Entity& entity_data) override; + + Status + GetIDsInSegment(const std::string& collection_name, const std::string& segment_name, + std::vector& id_array) override; + + Status + Search(const std::string& collection_name, const std::vector& partition_tag_array, + const std::vector& entity_array, int64_t topk, + const std::string& extra_params, TopKQueryResult& topk_query_result) override; + + Status + DescribeCollection(const std::string& collection_name, CollectionParam& collection_schema) override; + + Status + CountCollection(const std::string& collection_name, int64_t& entity_count) override; + + Status + ShowCollections(std::vector& collection_array) override; + + Status + ShowCollectionInfo(const std::string& collection_name, CollectionInfo& collection_info) override; + + Status + DeleteByID(const std::string& collection_name, const std::vector& id_array) override; + + Status + PreloadCollection(const std::string& collection_name) const override; + + Status + DescribeIndex(const std::string& collection_name, IndexParam& index_param) const override; + + Status + DropIndex(const std::string& collection_name) const override; + + Status + CreatePartition(const PartitionParam& partition_param) override; + + Status + ShowPartitions(const std::string& collection_name, PartitionTagList& partition_tag_array) const override; + + Status + DropPartition(const PartitionParam& partition_param) override; + + Status + FlushCollection(const std::string& collection_name) override; Status Flush() override; Status - CompactTable(const std::string& table_name) override; + CompactCollection(const std::string& collection_name) override; private: std::shared_ptr<::grpc::Channel> channel_; diff --git a/sdk/include/MilvusApi.h b/sdk/include/MilvusApi.h index e899210c91..743248744c 100644 --- a/sdk/include/MilvusApi.h +++ b/sdk/include/MilvusApi.h @@ -84,11 +84,19 @@ using TopKQueryResult = std::vector; ///< Topk query result * @brief Index parameters * Note: extra_params is extra parameters list, it must be json format * For different index type, parameter list is different accordingly, for example: - * FLAT/IVFLAT/SQ8: "{nlist: '16384'}" nlist range:[1, 999999] - * IVFPQ: "{nlist: '16384', m: "12"}" nlist range:[1, 999999] m is decided by dim and have a couple of results. + * FLAT/IVFLAT/SQ8: "{nlist: '16384'}" + * ///< nlist range:[1, 999999] + * IVFPQ: "{nlist: '16384', m: "12"}" + * ///< nlist range:[1, 999999] + * ///< m is decided by dim and have a couple of results. * NSG: "{search_length: '45', out_degree:'50', candidate_pool_size:'300', "knng":'100'}" - * search_length range:[10, 300] out_degree range:[5, 300] candidate_pool_size range:[50, 1000] knng range:[5, 300] - * HNSW "{M: '16', efConstruction:'500'}" M range:[5, 48] efConstruction range:[topk, 4096] + * ///< search_length range:[10, 300] + * ///< out_degree range:[5, 300] + * ///< candidate_pool_size range:[50, 1000] + * ///< knng range:[5, 300] + * HNSW "{M: '16', efConstruction:'500'}" + * ///< M range:[5, 48] + * ///< efConstruction range:[topk, 4096] */ struct IndexParam { std::string collection_name; ///< Collection name for create index @@ -139,18 +147,18 @@ struct CollectionInfo { class Connection { public: /** - * @brief CreateConnection + * @brief Create connection * * Create a connection instance and return it's shared pointer * - * @return Connection instance pointer + * @return connection instance pointer */ static std::shared_ptr Create(); /** - * @brief DestroyConnection + * @brief Destroy connection * * Destroy the connection instance * @@ -174,7 +182,7 @@ class Connection { */ virtual Status - Connect(const ConnectParam& param) = 0; + Connect(const ConnectParam& connect_param) = 0; /** * @brief Connect @@ -190,7 +198,7 @@ class Connection { Connect(const std::string& uri) = 0; /** - * @brief connected + * @brief Connected * * This method is used to test whether server is connected. * @@ -210,7 +218,7 @@ class Connection { Disconnect() = 0; /** - * @brief Give the client version + * @brief Get the client version * * This method is used to give the client version. * @@ -220,7 +228,7 @@ class Connection { ClientVersion() const = 0; /** - * @brief Give the server version + * @brief Get the server version * * This method is used to give the server version. * @@ -229,6 +237,42 @@ class Connection { virtual std::string ServerVersion() const = 0; + /** + * @brief Get the server status + * + * This method is used to give the server status. + * + * @return Server status. + */ + virtual std::string + ServerStatus() const = 0; + + /** + * @brief Get config method + * + * This method is used to set config. + * + * @param node_name, config node name. + * @param value, config value. + * + * @return Indicate if this operation is successful. + */ + virtual Status + GetConfig(const std::string& node_name, std::string& value) const = 0; + + /** + * @brief Set config method + * + * This method is used to set config. + * + * @param node_name, config node name. + * @param value, config value. + * + * @return Indicate if this operation is successful. + */ + virtual Status + SetConfig(const std::string& node_name, const std::string& value) const = 0; + /** * @brief Create collection method * @@ -278,9 +322,9 @@ class Connection { CreateIndex(const IndexParam& index_param) = 0; /** - * @brief Insert entity to table + * @brief Insert entity to collection * - * This method is used to insert vector array to table. + * This method is used to insert vector array to collection. * * @param collection_name, target collection's name. * @param partition_tag, target partition's tag, keep empty if no partition specified. @@ -337,21 +381,24 @@ class Connection { * * @param collection_name, target collection's name. * @param partition_tag_array, target partitions, keep empty if no partition specified. - * @param query_record_array, vectors to be queried. - * @param topk, how many similarity vectors will be returned. + * @param query_entity_array, vectors to be queried. + * @param topk, how many similarity entities will be returned. * @param extra_params, extra search parameters according to different index type, must be json format. * Note: extra_params is extra parameters list, it must be json format, for example: * For different index type, parameter list is different accordingly * FLAT/IVFLAT/SQ8/IVFPQ: "{nprobe: '32'}" + * ///< nprobe range:[1,999999] * NSG: "{search_length:'100'} + * ///< search_length range:[10, 300] * HNSW "{ef: '64'} + * ///< ef range:[k, 4096] * @param topk_query_result, result array. * * @return Indicate if query is successful. */ virtual Status - Search(const std::string& table_name, const PartitionTagList& partition_tag_array, - const std::vector& query_record_array, int64_t topk, + Search(const std::string& collection_name, const PartitionTagList& partition_tag_array, + const std::vector& entity_array, int64_t topk, const std::string& extra_params, TopKQueryResult& topk_query_result) = 0; /** @@ -360,17 +407,17 @@ class Connection { * This method is used to show collection information. * * @param collection_name, target collection's name. - * @param collection_schema, collection_schema is given when operation is successful. + * @param collection_param, collection_param is given when operation is successful. * * @return Indicate if this operation is successful. */ virtual Status - DescribeCollection(const std::string& collection_name, TableSchema& collection_schema) = 0; + DescribeCollection(const std::string& collection_name, CollectionParam& collection_param) = 0; /** - * @brief Get collection row count + * @brief Get collection entity count * - * This method is used to get collection row count. + * This method is used to get collection entity count. * * @param collection_name, target collection's name. * @param entity_count, collection total entity count(including partitions). @@ -393,160 +440,121 @@ class Connection { ShowCollections(std::vector& collection_array) = 0; /** - * @brief Show collections information + * @brief Show collection information * - * This method is used to get detail information of a collections. + * This method is used to get detail information of a collection. * - * @param collections_name, target collections's name. - * @param collections_info, target collections's information + * @param collection_name, target collection's name. + * @param collection_info, target collection's information * * @return Indicate if this operation is successful. */ virtual Status - ShowCollectionInfo(const std::string& collections_name, TableInfo& collections_info) = 0; + ShowCollectionInfo(const std::string& collection_name, CollectionInfo& collection_info) = 0; /** - * @brief Give the server status + * @brief Delete entity by id * - * This method is used to give the server status. + * This method is used to delete entity by id. * - * @return Server status. - */ - virtual std::string - ServerStatus() const = 0; - - /** - * [deprecated] - * @brief delete tables by vector id - * - * This method is used to delete table data by date range. - * - * @param table_name, target table's name. - * @param id_array, vector ids to deleted. + * @param collection_name, target collection's name. + * @param id_array, entity id array to be deleted. * * @return Indicate if this operation is successful. */ virtual Status - DeleteByID(const std::string& table_name, const std::vector& id_array) = 0; + DeleteByID(const std::string& collection_name, const std::vector& id_array) = 0; /** - * @brief preload table + * @brief Preload collection * - * This method is used to preload table + * This method is used to preload collection data into memory * - * @param table_name + * @param collection_name, target collection's name. * * @return Indicate if this operation is successful. */ virtual Status - PreloadTable(const std::string& table_name) const = 0; + PreloadCollection(const std::string& collection_name) const = 0; /** - * @brief describe index + * @brief Describe index * * This method is used to describe index * - * @param table_name, target table's name. + * @param collection_name, target collection's name. * @param index_param, returned index information. * * @return Indicate if this operation is successful. */ virtual Status - DescribeIndex(const std::string& table_name, IndexParam& index_param) const = 0; + DescribeIndex(const std::string& collection_name, IndexParam& index_param) const = 0; /** - * @brief drop index + * @brief Drop index * - * This method is used to drop index of table(and its partitions) + * This method is used to drop index of collection(and its partitions) * - * @param table_name, target table's name. + * @param collection_name, target collection's name. * * @return Indicate if this operation is successful. */ virtual Status - DropIndex(const std::string& table_name) const = 0; + DropIndex(const std::string& collection_name) const = 0; /** * @brief Create partition method * - * This method is used to create table partition + * This method is used to create collection's partition * - * @param param, use to provide partition information to be created. + * @param partition_param, use to provide partition information to be created. * * @return Indicate if partition is created successfully */ virtual Status - CreatePartition(const PartitionParam& param) = 0; + CreatePartition(const PartitionParam& partition_param) = 0; /** - * @brief Test table existence method + * @brief Show all partitions method * - * This method is used to create table + * This method is used to show all partitions(return their tags) * - * @param table_name, table name is going to be tested. - * @param partition_tag_array, partition tag array of the table. + * @param collection_name, target collection's name. + * @param partition_tag_array, partition tag array of the collection. * * @return Indicate if this operation is successful */ virtual Status - ShowPartitions(const std::string& table_name, PartitionTagList& partition_tag_array) const = 0; + ShowPartitions(const std::string& collection_name, PartitionTagList& partition_tag_array) const = 0; /** * @brief Delete partition method * - * This method is used to delete table partition. + * This method is used to delete collection's partition. * - * @param param, target partition to be deleted. - * NOTE: if param.table_name is empty, you must specify param.partition_name, - * else you can specify param.table_name and param.tag and let the param.partition_name be empty + * @param partition_param, target partition to be deleted. * * @return Indicate if partition is delete successfully. */ virtual Status - DropPartition(const PartitionParam& param) = 0; + DropPartition(const PartitionParam& partition_param) = 0; /** - * @brief Get config method + * @brief Flush collection buffer into storage * - * This method is used to set config. + * This method is used to flush collection buffer into storage * - * @param node_name, config node name. - * @param value, config value. + * @param collection_name, target collection's name. * * @return Indicate if this operation is successful. */ virtual Status - GetConfig(const std::string& node_name, std::string& value) const = 0; + FlushCollection(const std::string& collection_name) = 0; /** - * @brief Set config method + * @brief Flush all buffer into storage * - * This method is used to set config. - * - * @param node_name, config node name. - * @param value, config value. - * - * @return Indicate if this operation is successful. - */ - virtual Status - SetConfig(const std::string& node_name, const std::string& value) const = 0; - - /** - * @brief flush table buffer into storage - * - * This method is used to flush table buffer into storage - * - * @param table_name, target table's name. - * - * @return Indicate if this operation is successful. - */ - virtual Status - FlushTable(const std::string& table_name) = 0; - - /** - * @brief flush all buffer into storage - * - * This method is used to all table buffer into storage + * This method is used to all collection buffer into storage * * @return Indicate if this operation is successful. */ @@ -554,16 +562,16 @@ class Connection { Flush() = 0; /** - * @brief compact table, remove deleted vectors + * @brief Compact collection, permanently remove deleted vectors * - * This method is used to compact table + * This method is used to compact collection * - * @param table_name, target table's name. + * @param collection_name, target collection's name. * * @return Indicate if this operation is successful. */ virtual Status - CompactTable(const std::string& table_name) = 0; + CompactCollection(const std::string& collection_name) = 0; }; } // namespace milvus diff --git a/sdk/interface/ConnectionImpl.cpp b/sdk/interface/ConnectionImpl.cpp index 3a3fa52516..2a51a11d50 100644 --- a/sdk/interface/ConnectionImpl.cpp +++ b/sdk/interface/ConnectionImpl.cpp @@ -56,70 +56,6 @@ ConnectionImpl::ClientVersion() const { return client_proxy_->ClientVersion(); } -Status -ConnectionImpl::CreateTable(const TableSchema& param) { - return client_proxy_->CreateTable(param); -} - -bool -ConnectionImpl::HasTable(const std::string& table_name) { - return client_proxy_->HasTable(table_name); -} - -Status -ConnectionImpl::DropTable(const std::string& table_name) { - return client_proxy_->DropTable(table_name); -} - -Status -ConnectionImpl::CreateIndex(const IndexParam& index_param) { - return client_proxy_->CreateIndex(index_param); -} - -Status -ConnectionImpl::Insert(const std::string& table_name, const std::string& partition_tag, - const std::vector& record_array, std::vector& id_array) { - return client_proxy_->Insert(table_name, partition_tag, record_array, id_array); -} - -Status -ConnectionImpl::GetVectorByID(const std::string& table_name, int64_t vector_id, RowRecord& vector_data) { - return client_proxy_->GetVectorByID(table_name, vector_id, vector_data); -} - -Status -ConnectionImpl::GetIDsInSegment(const std::string& table_name, const std::string& segment_name, - std::vector& id_array) { - return client_proxy_->GetIDsInSegment(table_name, segment_name, id_array); -} - -Status -ConnectionImpl::Search(const std::string& table_name, const std::vector& partition_tags, - const std::vector& query_record_array, int64_t topk, const std::string& extra_params, - TopKQueryResult& topk_query_result) { - return client_proxy_->Search(table_name, partition_tags, query_record_array, topk, extra_params, topk_query_result); -} - -Status -ConnectionImpl::DescribeTable(const std::string& table_name, TableSchema& table_schema) { - return client_proxy_->DescribeTable(table_name, table_schema); -} - -Status -ConnectionImpl::CountTable(const std::string& table_name, int64_t& row_count) { - return client_proxy_->CountTable(table_name, row_count); -} - -Status -ConnectionImpl::ShowTables(std::vector& table_array) { - return client_proxy_->ShowTables(table_array); -} - -Status -ConnectionImpl::ShowTableInfo(const std::string& table_name, TableInfo& table_info) { - return client_proxy_->ShowTableInfo(table_name, table_info); -} - std::string ConnectionImpl::ServerVersion() const { return client_proxy_->ServerVersion(); @@ -130,41 +66,6 @@ ConnectionImpl::ServerStatus() const { return client_proxy_->ServerStatus(); } -Status -ConnectionImpl::DeleteByID(const std::string& table_name, const std::vector& id_array) { - return client_proxy_->DeleteByID(table_name, id_array); -} - -Status -ConnectionImpl::PreloadTable(const std::string& table_name) const { - return client_proxy_->PreloadTable(table_name); -} - -Status -ConnectionImpl::DescribeIndex(const std::string& table_name, IndexParam& index_param) const { - return client_proxy_->DescribeIndex(table_name, index_param); -} - -Status -ConnectionImpl::DropIndex(const std::string& table_name) const { - return client_proxy_->DropIndex(table_name); -} - -Status -ConnectionImpl::CreatePartition(const PartitionParam& param) { - return client_proxy_->CreatePartition(param); -} - -Status -ConnectionImpl::ShowPartitions(const std::string& table_name, PartitionTagList& partition_array) const { - return client_proxy_->ShowPartitions(table_name, partition_array); -} - -Status -ConnectionImpl::DropPartition(const PartitionParam& param) { - return client_proxy_->DropPartition(param); -} - Status ConnectionImpl::GetConfig(const std::string& node_name, std::string& value) const { return client_proxy_->GetConfig(node_name, value); @@ -176,8 +77,107 @@ ConnectionImpl::SetConfig(const std::string& node_name, const std::string& value } Status -ConnectionImpl::FlushTable(const std::string& Status) { - return client_proxy_->FlushTable(Status); +ConnectionImpl::CreateCollection(const CollectionParam& param) { + return client_proxy_->CreateCollection(param); +} + +bool +ConnectionImpl::HasCollection(const std::string& collection_name) { + return client_proxy_->HasCollection(collection_name); +} + +Status +ConnectionImpl::DropCollection(const std::string& collection_name) { + return client_proxy_->DropCollection(collection_name); +} + +Status +ConnectionImpl::CreateIndex(const IndexParam& index_param) { + return client_proxy_->CreateIndex(index_param); +} + +Status +ConnectionImpl::Insert(const std::string& collection_name, const std::string& partition_tag, + const std::vector& entity_array, std::vector& id_array) { + return client_proxy_->Insert(collection_name, partition_tag, entity_array, id_array); +} + +Status +ConnectionImpl::GetEntityByID(const std::string& collection_name, int64_t entity_id, Entity& entity_data) { + return client_proxy_->GetEntityByID(collection_name, entity_id, entity_data); +} + +Status +ConnectionImpl::GetIDsInSegment(const std::string& collection_name, const std::string& segment_name, + std::vector& id_array) { + return client_proxy_->GetIDsInSegment(collection_name, segment_name, id_array); +} + +Status +ConnectionImpl::Search(const std::string& collection_name, const std::vector& partition_tags, + const std::vector& entity_array, int64_t topk, const std::string& extra_params, + TopKQueryResult& topk_query_result) { + return client_proxy_->Search(collection_name, partition_tags, entity_array, topk, extra_params, topk_query_result); +} + +Status +ConnectionImpl::DescribeCollection(const std::string& collection_name, CollectionParam& collection_schema) { + return client_proxy_->DescribeCollection(collection_name, collection_schema); +} + +Status +ConnectionImpl::CountCollection(const std::string& collection_name, int64_t& row_count) { + return client_proxy_->CountCollection(collection_name, row_count); +} + +Status +ConnectionImpl::ShowCollections(std::vector& collection_array) { + return client_proxy_->ShowCollections(collection_array); +} + +Status +ConnectionImpl::ShowCollectionInfo(const std::string& collection_name, CollectionInfo& collection_info) { + return client_proxy_->ShowCollectionInfo(collection_name, collection_info); +} + +Status +ConnectionImpl::DeleteByID(const std::string& collection_name, const std::vector& id_array) { + return client_proxy_->DeleteByID(collection_name, id_array); +} + +Status +ConnectionImpl::PreloadCollection(const std::string& collection_name) const { + return client_proxy_->PreloadCollection(collection_name); +} + +Status +ConnectionImpl::DescribeIndex(const std::string& collection_name, IndexParam& index_param) const { + return client_proxy_->DescribeIndex(collection_name, index_param); +} + +Status +ConnectionImpl::DropIndex(const std::string& collection_name) const { + return client_proxy_->DropIndex(collection_name); +} + +Status +ConnectionImpl::CreatePartition(const PartitionParam& partition_param) { + return client_proxy_->CreatePartition(partition_param); +} + +Status +ConnectionImpl::ShowPartitions(const std::string& collection_name, PartitionTagList& partition_array) const { + return client_proxy_->ShowPartitions(collection_name, partition_array); +} + +Status +ConnectionImpl::DropPartition(const PartitionParam& partition_param) { + return client_proxy_->DropPartition(partition_param); +} + +Status +ConnectionImpl::FlushCollection(const std::string& Status) { + return client_proxy_->FlushCollection(Status); } Status @@ -186,8 +186,8 @@ ConnectionImpl::Flush() { } Status -ConnectionImpl::CompactTable(const std::string& table_name) { - return client_proxy_->CompactTable(table_name); +ConnectionImpl::CompactCollection(const std::string& collection_name) { + return client_proxy_->CompactCollection(collection_name); } } // namespace milvus diff --git a/sdk/interface/ConnectionImpl.h b/sdk/interface/ConnectionImpl.h index 469adf1a27..15533b9de6 100644 --- a/sdk/interface/ConnectionImpl.h +++ b/sdk/interface/ConnectionImpl.h @@ -26,7 +26,7 @@ class ConnectionImpl : public Connection { // Implementations of the Connection interface Status - Connect(const ConnectParam& param) override; + Connect(const ConnectParam& connect_param) override; Status Connect(const std::string& uri) override; @@ -37,46 +37,6 @@ class ConnectionImpl : public Connection { Status Disconnect() override; - Status - CreateTable(const TableSchema& param) override; - - bool - HasTable(const std::string& table_name) override; - - Status - DropTable(const std::string& table_name) override; - - Status - CreateIndex(const IndexParam& index_param) override; - - Status - Insert(const std::string& table_name, const std::string& partition_tag, const std::vector& record_array, - std::vector& id_array) override; - - Status - GetVectorByID(const std::string& table_name, int64_t vector_id, RowRecord& vector_data) override; - - Status - GetIDsInSegment(const std::string& table_name, const std::string& segment_name, - std::vector& id_array) override; - - Status - Search(const std::string& table_name, const std::vector& partition_tag_array, - const std::vector& query_record_array, int64_t topk, - const std::string& extra_params, TopKQueryResult& topk_query_result) override; - - Status - DescribeTable(const std::string& table_name, TableSchema& table_schema) override; - - Status - CountTable(const std::string& table_name, int64_t& row_count) override; - - Status - ShowTables(std::vector& table_array) override; - - Status - ShowTableInfo(const std::string& table_name, TableInfo& table_info) override; - std::string ClientVersion() const override; @@ -86,27 +46,6 @@ class ConnectionImpl : public Connection { std::string ServerStatus() const override; - Status - DeleteByID(const std::string& table_name, const std::vector& id_array) override; - - Status - PreloadTable(const std::string& table_name) const override; - - Status - DescribeIndex(const std::string& table_name, IndexParam& index_param) const override; - - Status - DropIndex(const std::string& table_name) const override; - - Status - CreatePartition(const PartitionParam& param) override; - - Status - ShowPartitions(const std::string& table_name, PartitionTagList& partition_tag_array) const override; - - Status - DropPartition(const PartitionParam& param) override; - Status GetConfig(const std::string& node_name, std::string& value) const override; @@ -114,13 +53,76 @@ class ConnectionImpl : public Connection { SetConfig(const std::string& node_name, const std::string& value) const override; Status - FlushTable(const std::string& table_name) override; + CreateCollection(const CollectionParam& param) override; + + bool + HasCollection(const std::string& collection_name) override; + + Status + DropCollection(const std::string& collection_name) override; + + Status + CreateIndex(const IndexParam& index_param) override; + + Status + Insert(const std::string& collection_name, + const std::string& partition_tag, + const std::vector& entity_array, + std::vector& id_array) override; + + Status + GetEntityByID(const std::string& collection_name, int64_t entity_id, Entity& entity_data) override; + + Status + GetIDsInSegment(const std::string& collection_name, const std::string& segment_name, + std::vector& id_array) override; + + Status + Search(const std::string& collection_name, const std::vector& partition_tag_array, + const std::vector& entity_array, int64_t topk, + const std::string& extra_params, TopKQueryResult& topk_query_result) override; + + Status + DescribeCollection(const std::string& collection_name, CollectionParam& collection_schema) override; + + Status + CountCollection(const std::string& collection_name, int64_t& entity_count) override; + + Status + ShowCollections(std::vector& collection_array) override; + + Status + ShowCollectionInfo(const std::string& collection_name, CollectionInfo& collection_info) override; + + Status + DeleteByID(const std::string& collection_name, const std::vector& id_array) override; + + Status + PreloadCollection(const std::string& collection_name) const override; + + Status + DescribeIndex(const std::string& collection_name, IndexParam& index_param) const override; + + Status + DropIndex(const std::string& collection_name) const override; + + Status + CreatePartition(const PartitionParam& partition_param) override; + + Status + ShowPartitions(const std::string& collection_name, PartitionTagList& partition_tag_array) const override; + + Status + DropPartition(const PartitionParam& partition_param) override; + + Status + FlushCollection(const std::string& collection_name) override; Status Flush() override; Status - CompactTable(const std::string& table_name) override; + CompactCollection(const std::string& collection_name) override; private: std::shared_ptr client_proxy_; From 24daade18e4d99b85e58eb4cb56de100a4cc7e7f Mon Sep 17 00:00:00 2001 From: Yhz Date: Sun, 8 Mar 2020 14:30:02 +0800 Subject: [PATCH 10/58] fix config wal valid_recovery_error_ignore bug Signed-off-by: Yhz --- CHANGELOG.md | 1 + core/src/server/Config.cpp | 2 +- core/src/server/Config.h | 3 +++ 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index abb94a552c..1fa431015c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -42,6 +42,7 @@ Please mark all change in change log and use the issue from GitHub - \#1529 Fix server crash when cache_insert_data enabled - \#1530 Set table file with correct engine type in meta - \#1535 Degradation searching performance with metric_type: binary_idmap +- \#1549 Fix server/wal config setting bug ## Feature - \#216 Add CLI to get server info diff --git a/core/src/server/Config.cpp b/core/src/server/Config.cpp index f0b88fa98c..8506a28f0f 100644 --- a/core/src/server/Config.cpp +++ b/core/src/server/Config.cpp @@ -2013,7 +2013,7 @@ Config::SetWalConfigEnable(const std::string& value) { Status Config::SetWalConfigRecoveryErrorIgnore(const std::string& value) { CONFIG_CHECK(CheckWalConfigRecoveryErrorIgnore(value)); - return SetConfigValueInMem(CONFIG_WAL, CONFIG_WAL_RECOVERY_ERROR_IGNORE_DEFAULT, value); + return SetConfigValueInMem(CONFIG_WAL, CONFIG_WAL_RECOVERY_ERROR_IGNORE, value); } Status diff --git a/core/src/server/Config.h b/core/src/server/Config.h index 1bd99a6b9d..77bdd5def6 100644 --- a/core/src/server/Config.h +++ b/core/src/server/Config.h @@ -191,6 +191,9 @@ class Config { Status SetConfigCli(const std::string& parent_key, const std::string& child_key, const std::string& value); + Status + CheckDuplicatePort(); + Status UpdateFileConfigFromMem(const std::string& parent_key, const std::string& child_key); From 0ce5a3f5eb34ebf59a9326ba5a1d65cc787da0e0 Mon Sep 17 00:00:00 2001 From: "xiaojun.lin" Date: Sun, 8 Mar 2020 16:15:59 +0800 Subject: [PATCH 11/58] update. Signed-off-by: xiaojun.lin --- core/unittest/wrapper/test_wrapper.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/unittest/wrapper/test_wrapper.cpp b/core/unittest/wrapper/test_wrapper.cpp index ba3ed0f1a1..65e672e8c6 100644 --- a/core/unittest/wrapper/test_wrapper.cpp +++ b/core/unittest/wrapper/test_wrapper.cpp @@ -85,7 +85,7 @@ INSTANTIATE_TEST_CASE_P( // std::make_tuple(milvus::engine::IndexType::NSG_MIX, "Default", 128, 250000, 10, 10), #endif // std::make_tuple(milvus::engine::IndexType::SPTAG_KDT_RNT_CPU, "Default", 128, 100, 10, 10), - // std::make_tuple(milvus::engine::IndexType::SPTAG_BKT_RNT_CPU, "Default", 128, 100, 10, 10), + // std::make_tuple(milvus::engine::IndexType::SPTAG_BKT_RNT_CPU, "Default", 126, 100, 10, 10), std::make_tuple(milvus::engine::IndexType::HNSW, "Default", 64, 10000, 5, 10), std::make_tuple(milvus::engine::IndexType::FAISS_IDMAP, "Default", 64, 1000, 10, 10), std::make_tuple(milvus::engine::IndexType::FAISS_IVFFLAT_CPU, "Default", 64, 1000, 10, 10), From 5892d4b9ba3e1255204188f92760e3ad289e1854 Mon Sep 17 00:00:00 2001 From: Yhz Date: Sun, 8 Mar 2020 16:22:13 +0800 Subject: [PATCH 12/58] remove port check declaration Signed-off-by: Yhz --- core/src/server/Config.h | 3 --- 1 file changed, 3 deletions(-) diff --git a/core/src/server/Config.h b/core/src/server/Config.h index 77bdd5def6..1bd99a6b9d 100644 --- a/core/src/server/Config.h +++ b/core/src/server/Config.h @@ -191,9 +191,6 @@ class Config { Status SetConfigCli(const std::string& parent_key, const std::string& child_key, const std::string& value); - Status - CheckDuplicatePort(); - Status UpdateFileConfigFromMem(const std::string& parent_key, const std::string& child_key); From 8cd5d64f06bec7bf84f3adfbe0d0e0f0204c8e86 Mon Sep 17 00:00:00 2001 From: Yhz Date: Sun, 8 Mar 2020 18:06:25 +0800 Subject: [PATCH 13/58] use bool conversation Signed-off-by: Yhz --- core/src/server/Config.cpp | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/core/src/server/Config.cpp b/core/src/server/Config.cpp index 8506a28f0f..f1a3a421f7 100644 --- a/core/src/server/Config.cpp +++ b/core/src/server/Config.cpp @@ -1529,8 +1529,7 @@ Status Config::GetStorageConfigS3Enable(bool& value) { std::string str = GetConfigStr(CONFIG_STORAGE, CONFIG_STORAGE_S3_ENABLE, CONFIG_STORAGE_S3_ENABLE_DEFAULT); CONFIG_CHECK(CheckStorageConfigS3Enable(str)); - std::transform(str.begin(), str.end(), str.begin(), ::tolower); - value = (str == "true" || str == "on" || str == "yes" || str == "1"); + CONFIG_CHECK(StringHelpFunctions::ConvertToBoolean(str, value)); return Status::OK(); } @@ -1569,8 +1568,7 @@ Status Config::GetMetricConfigEnableMonitor(bool& value) { std::string str = GetConfigStr(CONFIG_METRIC, CONFIG_METRIC_ENABLE_MONITOR, CONFIG_METRIC_ENABLE_MONITOR_DEFAULT); CONFIG_CHECK(CheckMetricConfigEnableMonitor(str)); - std::transform(str.begin(), str.end(), str.begin(), ::tolower); - value = (str == "true" || str == "on" || str == "yes" || str == "1"); + CONFIG_CHECK(StringHelpFunctions::ConvertToBoolean(str, value)); return Status::OK(); } @@ -1671,8 +1669,7 @@ Status Config::GetGpuResourceConfigEnable(bool& value) { std::string str = GetConfigStr(CONFIG_GPU_RESOURCE, CONFIG_GPU_RESOURCE_ENABLE, CONFIG_GPU_RESOURCE_ENABLE_DEFAULT); CONFIG_CHECK(CheckGpuResourceConfigEnable(str)); - std::transform(str.begin(), str.end(), str.begin(), ::tolower); - value = (str == "true" || str == "on" || str == "yes" || str == "1"); + CONFIG_CHECK(StringHelpFunctions::ConvertToBoolean(str, value)); return Status::OK(); } @@ -1774,8 +1771,7 @@ Status Config::GetWalConfigEnable(bool& wal_enable) { std::string str = GetConfigStr(CONFIG_WAL, CONFIG_WAL_ENABLE, CONFIG_WAL_ENABLE_DEFAULT); CONFIG_CHECK(CheckWalConfigEnable(str)); - std::transform(str.begin(), str.end(), str.begin(), ::tolower); - wal_enable = (str == "true" || str == "on" || str == "yes" || str == "1"); + CONFIG_CHECK(StringHelpFunctions::ConvertToBoolean(str, wal_enable)); return Status::OK(); } @@ -1784,8 +1780,7 @@ Config::GetWalConfigRecoveryErrorIgnore(bool& recovery_error_ignore) { std::string str = GetConfigStr(CONFIG_WAL, CONFIG_WAL_RECOVERY_ERROR_IGNORE, CONFIG_WAL_RECOVERY_ERROR_IGNORE_DEFAULT); CONFIG_CHECK(CheckWalConfigRecoveryErrorIgnore(str)); - std::transform(str.begin(), str.end(), str.begin(), ::tolower); - recovery_error_ignore = (str == "true" || str == "on" || str == "yes" || str == "1"); + CONFIG_CHECK(StringHelpFunctions::ConvertToBoolean(str, recovery_error_ignore)); return Status::OK(); } From aedde79f264f618d54c20c0dc0a13e9742f00912 Mon Sep 17 00:00:00 2001 From: "yudong.cai" Date: Sun, 8 Mar 2020 19:14:40 +0800 Subject: [PATCH 14/58] #1567 update yaml config description Signed-off-by: yudong.cai --- CHANGELOG.md | 1 + core/conf/demo/server_config.yaml | 2 +- core/conf/server_cpu_config.template | 2 +- core/conf/server_gpu_config.template | 2 +- 4 files changed, 4 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 951494dddc..90c0754faa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -93,6 +93,7 @@ Please mark all change in change log and use the issue from GitHub - \#1480 Add return code for AVX512 selection - \#1524 Update config "preload_table" description - \#1544 Update resources name in HTTP module +- \#1567 Update yaml config description ## Task - \#1327 Exclude third-party code from codebeat diff --git a/core/conf/demo/server_config.yaml b/core/conf/demo/server_config.yaml index cb15b1487f..ef0e15c029 100644 --- a/core/conf/demo/server_config.yaml +++ b/core/conf/demo/server_config.yaml @@ -108,7 +108,7 @@ cache_config: # | compared with 'nq' to decide if OpenBLAS should be used. | | | # | If nq >= use_blas_threshold, OpenBLAS will be used, search | | | # | response times will be stable but the search speed will be | | | -# | slower; if nq < use_blas_threshold, SSE will be used, | | | +# | slower; if nq < use_blas_threshold, AVX will be used, | | | # | search speed will be faster but search response times will | | | # | fluctuate. | | | #----------------------+------------------------------------------------------------+------------+-----------------+ diff --git a/core/conf/server_cpu_config.template b/core/conf/server_cpu_config.template index e511bc22f7..083a47aa25 100644 --- a/core/conf/server_cpu_config.template +++ b/core/conf/server_cpu_config.template @@ -108,7 +108,7 @@ cache_config: # | compared with 'nq' to decide if OpenBLAS should be used. | | | # | If nq >= use_blas_threshold, OpenBLAS will be used, search | | | # | response times will be stable but the search speed will be | | | -# | slower; if nq < use_blas_threshold, SSE will be used, | | | +# | slower; if nq < use_blas_threshold, AVX will be used, | | | # | search speed will be faster but search response times will | | | # | fluctuate. | | | #----------------------+------------------------------------------------------------+------------+-----------------+ diff --git a/core/conf/server_gpu_config.template b/core/conf/server_gpu_config.template index 2a926f3f6a..f0455b1f31 100644 --- a/core/conf/server_gpu_config.template +++ b/core/conf/server_gpu_config.template @@ -108,7 +108,7 @@ cache_config: # | compared with 'nq' to decide if OpenBLAS should be used. | | | # | If nq >= use_blas_threshold, OpenBLAS will be used, search | | | # | response times will be stable but the search speed will be | | | -# | slower; if nq < use_blas_threshold, SSE will be used, | | | +# | slower; if nq < use_blas_threshold, AVX will be used, | | | # | search speed will be faster but search response times will | | | # | fluctuate. | | | #----------------------+------------------------------------------------------------+------------+-----------------+ From c1e917636f417193626019bea89391a632637870 Mon Sep 17 00:00:00 2001 From: "yudong.cai" Date: Sun, 8 Mar 2020 19:20:24 +0800 Subject: [PATCH 15/58] #1567 update yaml config description Signed-off-by: yudong.cai --- core/conf/demo/server_config.yaml | 8 ++++---- core/conf/server_cpu_config.template | 8 ++++---- core/conf/server_gpu_config.template | 8 ++++---- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/core/conf/demo/server_config.yaml b/core/conf/demo/server_config.yaml index ef0e15c029..52991431e2 100644 --- a/core/conf/demo/server_config.yaml +++ b/core/conf/demo/server_config.yaml @@ -107,10 +107,10 @@ cache_config: # use_blas_threshold | A Milvus performance tuning parameter. This value will be | Integer | 1100 | # | compared with 'nq' to decide if OpenBLAS should be used. | | | # | If nq >= use_blas_threshold, OpenBLAS will be used, search | | | -# | response times will be stable but the search speed will be | | | -# | slower; if nq < use_blas_threshold, AVX will be used, | | | -# | search speed will be faster but search response times will | | | -# | fluctuate. | | | +# | response time will be stable but the search speed will be | | | +# | slower; if nq < use_blas_threshold, AVX or SSE will be | | | +# | used, search speed will be faster but search response time | | | +# | will fluctuate. | | | #----------------------+------------------------------------------------------------+------------+-----------------+ # gpu_search_threshold | A Milvus performance tuning parameter. This value will be | Integer | 1000 | # | compared with 'nq' to decide if the search computation will| | | diff --git a/core/conf/server_cpu_config.template b/core/conf/server_cpu_config.template index 083a47aa25..2a3d52e119 100644 --- a/core/conf/server_cpu_config.template +++ b/core/conf/server_cpu_config.template @@ -107,10 +107,10 @@ cache_config: # use_blas_threshold | A Milvus performance tuning parameter. This value will be | Integer | 1100 | # | compared with 'nq' to decide if OpenBLAS should be used. | | | # | If nq >= use_blas_threshold, OpenBLAS will be used, search | | | -# | response times will be stable but the search speed will be | | | -# | slower; if nq < use_blas_threshold, AVX will be used, | | | -# | search speed will be faster but search response times will | | | -# | fluctuate. | | | +# | response time will be stable but the search speed will be | | | +# | slower; if nq < use_blas_threshold, AVX or SSE will be | | | +# | used, search speed will be faster but search response time | | | +# | will fluctuate. | | | #----------------------+------------------------------------------------------------+------------+-----------------+ # gpu_search_threshold | A Milvus performance tuning parameter. This value will be | Integer | 1000 | # | compared with 'nq' to decide if the search computation will| | | diff --git a/core/conf/server_gpu_config.template b/core/conf/server_gpu_config.template index f0455b1f31..593d830c38 100644 --- a/core/conf/server_gpu_config.template +++ b/core/conf/server_gpu_config.template @@ -107,10 +107,10 @@ cache_config: # use_blas_threshold | A Milvus performance tuning parameter. This value will be | Integer | 1100 | # | compared with 'nq' to decide if OpenBLAS should be used. | | | # | If nq >= use_blas_threshold, OpenBLAS will be used, search | | | -# | response times will be stable but the search speed will be | | | -# | slower; if nq < use_blas_threshold, AVX will be used, | | | -# | search speed will be faster but search response times will | | | -# | fluctuate. | | | +# | response time will be stable but the search speed will be | | | +# | slower; if nq < use_blas_threshold, AVX or SSE will be | | | +# | used, search speed will be faster but search response time | | | +# | will fluctuate. | | | #----------------------+------------------------------------------------------------+------------+-----------------+ # gpu_search_threshold | A Milvus performance tuning parameter. This value will be | Integer | 1000 | # | compared with 'nq' to decide if the search computation will| | | From c88fad09892694c2c6987a603e460f7b56eb1d58 Mon Sep 17 00:00:00 2001 From: Yhz Date: Sun, 8 Mar 2020 21:41:38 +0800 Subject: [PATCH 16/58] handler json error when create index Signed-off-by: Yhz --- core/src/server/web_impl/handler/WebRequestHandler.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/core/src/server/web_impl/handler/WebRequestHandler.cpp b/core/src/server/web_impl/handler/WebRequestHandler.cpp index 92d68e77ca..72f04aa748 100644 --- a/core/src/server/web_impl/handler/WebRequestHandler.cpp +++ b/core/src/server/web_impl/handler/WebRequestHandler.cpp @@ -1046,7 +1046,9 @@ WebRequestHandler::CreateIndex(const OString& table_name, const OString& body) { auto status = request_handler_.CreateIndex(context_ptr_, table_name->std_str(), index, request_json["params"]); ASSIGN_RETURN_STATUS_DTO(status); } catch (nlohmann::detail::parse_error& e) { + RETURN_STATUS_DTO(BODY_PARSE_FAIL, e.what()) } catch (nlohmann::detail::type_error& e) { + RETURN_STATUS_DTO(BODY_PARSE_FAIL, e.what()) } ASSIGN_RETURN_STATUS_DTO(Status::OK()) From b3b3111a7a3985443424040af7ac86ee8a018f32 Mon Sep 17 00:00:00 2001 From: Yhz Date: Sun, 8 Mar 2020 22:26:28 +0800 Subject: [PATCH 17/58] change create index url Signed-off-by: Yhz --- core/src/server/web_impl/controller/WebController.hpp | 2 +- core/unittest/server/test_web.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/core/src/server/web_impl/controller/WebController.hpp b/core/src/server/web_impl/controller/WebController.hpp index ab2a26c2ca..ea9574b026 100644 --- a/core/src/server/web_impl/controller/WebController.hpp +++ b/core/src/server/web_impl/controller/WebController.hpp @@ -336,7 +336,7 @@ class WebController : public oatpp::web::server::api::ApiController { ADD_CORS(CreateIndex) - ENDPOINT("POST", "/tables/{collection_name}/indexes", CreateIndex, + ENDPOINT("POST", "/collections/{collection_name}/indexes", CreateIndex, PATH(String, collection_name), BODY_STRING(String, body)) { TimeRecorder tr(std::string(WEB_LOG_PREFIX) + "POST \'/tables/" + collection_name->std_str() + "/indexes\'"); tr.RecordSection("Received request."); diff --git a/core/unittest/server/test_web.cpp b/core/unittest/server/test_web.cpp index 073e08ec91..00703482e1 100644 --- a/core/unittest/server/test_web.cpp +++ b/core/unittest/server/test_web.cpp @@ -646,7 +646,7 @@ class TestClient : public oatpp::web::client::ApiClient { API_CALL("OPTIONS", "/collections/{collection_name}/indexes", optionsIndexes, PATH(String, collection_name, "collection_name")) - API_CALL("POST", "/tables/{table_name}/indexes", createIndex, PATH(String, table_name, "table_name"), + API_CALL("POST", "/collections/{collection_name}/indexes", createIndex, PATH(String, collection_name, "collection_name"), BODY_STRING(OString, body)) API_CALL("GET", "/collections/{collection_name}/indexes", getIndex, PATH(String, collection_name, "collection_name")) From c5bfcaef74827ed9145b8403d9479af77beeb962 Mon Sep 17 00:00:00 2001 From: Yhz Date: Mon, 9 Mar 2020 09:22:39 +0800 Subject: [PATCH 18/58] lower variable in systemop Signed-off-by: Yhz --- core/src/server/web_impl/controller/WebController.hpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/core/src/server/web_impl/controller/WebController.hpp b/core/src/server/web_impl/controller/WebController.hpp index ea9574b026..55a51168a3 100644 --- a/core/src/server/web_impl/controller/WebController.hpp +++ b/core/src/server/web_impl/controller/WebController.hpp @@ -674,15 +674,15 @@ class WebController : public oatpp::web::server::api::ApiController { ADD_CORS(SystemOp) - ENDPOINT("PUT", "/system/{Op}", SystemOp, PATH(String, Op), BODY_STRING(String, body_str)) { - TimeRecorder tr(std::string(WEB_LOG_PREFIX) + "PUT \'/system/" + Op->std_str() + "\'"); + ENDPOINT("PUT", "/system/{op}", SystemOp, PATH(String, op), BODY_STRING(String, body_str)) { + TimeRecorder tr(std::string(WEB_LOG_PREFIX) + "PUT \'/system/" + op->std_str() + "\'"); tr.RecordSection("Received request."); WebRequestHandler handler = WebRequestHandler(); handler.RegisterRequestHandler(::milvus::server::RequestHandler()); String response_str; - auto status_dto = handler.SystemOp(Op, body_str, response_str); + auto status_dto = handler.SystemOp(op, body_str, response_str); std::shared_ptr response; switch (status_dto->code->getValue()) { From 5eaa76417d0333141466ee54962403c02a9ea989 Mon Sep 17 00:00:00 2001 From: Lutkin Wang Date: Mon, 9 Mar 2020 09:38:54 +0800 Subject: [PATCH 19/58] Update README.md Signed-off-by: Lutkin Wang --- core/src/server/web_impl/README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/core/src/server/web_impl/README.md b/core/src/server/web_impl/README.md index ac447a529c..67535ca2fa 100644 --- a/core/src/server/web_impl/README.md +++ b/core/src/server/web_impl/README.md @@ -471,6 +471,7 @@ Gets all information about a table by name. | Parameter | Description | Required? | |-----------------|---|------| | `table_name` | Name of the table. | Yes | +| `info` | Type of information to acquire. `info` must either be empty or `stat`. When `info` is empty, Milvus returns table name, dimension, index file size, metric type, offset, index type, and nlist of the table. When `info` is `stat`, Milvus returns the table offset, partition status, and segment status. | No | #### Response @@ -1089,7 +1090,7 @@ $ curl -X GET "http://192.168.1.65:19121/system/version" -H "accept: application ##### Response ```json -{"reply":"0.6.0"} +{"reply":"0.7.0"} ``` ## Error Codes From 37fd79631d21af2af32e92820eabfdfdd7b66f05 Mon Sep 17 00:00:00 2001 From: groot Date: Mon, 9 Mar 2020 11:14:12 +0800 Subject: [PATCH 20/58] #1571 Meta engine type become IDMAP after drop index for BINARY table Signed-off-by: groot --- CHANGELOG.md | 1 + core/src/db/DBImpl.cpp | 6 +- core/src/db/Utils.cpp | 13 ++++ core/src/db/Utils.h | 6 ++ core/src/db/engine/ExecutionEngineImpl.cpp | 5 +- core/src/db/meta/SqliteMetaImpl.cpp | 62 +++++++++---------- core/src/scheduler/task/BuildIndexTask.cpp | 5 +- core/src/scheduler/task/SearchTask.cpp | 4 +- .../delivery/request/CreateIndexRequest.cpp | 3 +- .../delivery/request/CreateTableRequest.cpp | 3 +- .../server/delivery/request/InsertRequest.cpp | 5 +- .../server/delivery/request/SearchRequest.cpp | 3 +- core/src/utils/ValidationUtil.cpp | 13 ---- core/src/utils/ValidationUtil.h | 6 -- 14 files changed, 68 insertions(+), 67 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 69e40ea6fb..6ffed248af 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -45,6 +45,7 @@ Please mark all change in change log and use the issue from GitHub - \#1535 Degradation searching performance with metric_type: binary_idmap - \#1556 Index file not created after table and index created - \#1560 Search crashed with Super-high dimensional binary vector +- \#1571 Meta engine type become IDMAP after drop index for BINARY table ## Feature - \#216 Add CLI to get server info diff --git a/core/src/db/DBImpl.cpp b/core/src/db/DBImpl.cpp index 29ed17308a..c32819a830 100644 --- a/core/src/db/DBImpl.cpp +++ b/core/src/db/DBImpl.cpp @@ -366,8 +366,8 @@ DBImpl::PreloadTable(const std::string& table_id) { if (file.file_type_ == meta::TableFileSchema::FILE_TYPE::RAW || file.file_type_ == meta::TableFileSchema::FILE_TYPE::TO_INDEX || file.file_type_ == meta::TableFileSchema::FILE_TYPE::BACKUP) { - engine_type = server::ValidationUtil::IsBinaryMetricType(file.metric_type_) ? EngineType::FAISS_BIN_IDMAP - : EngineType::FAISS_IDMAP; + engine_type = + utils::IsBinaryMetricType(file.metric_type_) ? EngineType::FAISS_BIN_IDMAP : EngineType::FAISS_IDMAP; } else { engine_type = (EngineType)file.engine_type_; } @@ -1019,7 +1019,7 @@ DBImpl::GetVectorByIdHelper(const std::string& table_id, IDNumber vector_id, Vec auto deleted = std::find(deleted_docs.begin(), deleted_docs.end(), offset); if (deleted == deleted_docs.end()) { // Load raw vector - bool is_binary = server::ValidationUtil::IsBinaryMetricType(file.metric_type_); + bool is_binary = utils::IsBinaryMetricType(file.metric_type_); size_t single_vector_bytes = is_binary ? file.dimension_ / 8 : file.dimension_ * sizeof(float); std::vector raw_vector; status = segment_reader.LoadVectors(offset * single_vector_bytes, single_vector_bytes, raw_vector); diff --git a/core/src/db/Utils.cpp b/core/src/db/Utils.cpp index f9ac5dfe40..d184cea085 100644 --- a/core/src/db/Utils.cpp +++ b/core/src/db/Utils.cpp @@ -220,6 +220,19 @@ IsRawIndexType(int32_t type) { return (type == (int32_t)EngineType::FAISS_IDMAP) || (type == (int32_t)EngineType::FAISS_BIN_IDMAP); } +bool +IsBinaryIndexType(int32_t index_type) { + return (index_type == (int32_t)engine::EngineType::FAISS_BIN_IDMAP) || + (index_type == (int32_t)engine::EngineType::FAISS_BIN_IVFFLAT); +} + +bool +IsBinaryMetricType(int32_t metric_type) { + return (metric_type == (int32_t)engine::MetricType::HAMMING) || + (metric_type == (int32_t)engine::MetricType::JACCARD) || + (metric_type == (int32_t)engine::MetricType::TANIMOTO); +} + meta::DateT GetDate(const std::time_t& t, int day_delta) { struct tm ltm; diff --git a/core/src/db/Utils.h b/core/src/db/Utils.h index 88197d7d5c..c78b4fd717 100644 --- a/core/src/db/Utils.h +++ b/core/src/db/Utils.h @@ -48,6 +48,12 @@ IsSameIndex(const TableIndex& index1, const TableIndex& index2); bool IsRawIndexType(int32_t type); +static bool +IsBinaryIndexType(int32_t index_type); + +bool +IsBinaryMetricType(int32_t metric_type); + meta::DateT GetDate(const std::time_t& t, int day_delta = 0); meta::DateT diff --git a/core/src/db/engine/ExecutionEngineImpl.cpp b/core/src/db/engine/ExecutionEngineImpl.cpp index 1fe820aa7a..9cb1dfb09f 100644 --- a/core/src/db/engine/ExecutionEngineImpl.cpp +++ b/core/src/db/engine/ExecutionEngineImpl.cpp @@ -100,9 +100,8 @@ ExecutionEngineImpl::ExecutionEngineImpl(uint16_t dimension, const std::string& index_type_(index_type), metric_type_(metric_type), index_params_(index_params) { - EngineType tmp_index_type = server::ValidationUtil::IsBinaryMetricType((int32_t)metric_type) - ? EngineType::FAISS_BIN_IDMAP - : EngineType::FAISS_IDMAP; + EngineType tmp_index_type = + utils::IsBinaryMetricType((int32_t)metric_type) ? EngineType::FAISS_BIN_IDMAP : EngineType::FAISS_IDMAP; index_ = CreatetVecIndex(tmp_index_type); if (!index_) { throw Exception(DB_ERROR, "Unsupported index type"); diff --git a/core/src/db/meta/SqliteMetaImpl.cpp b/core/src/db/meta/SqliteMetaImpl.cpp index ca37f2c258..ce5ff99742 100644 --- a/core/src/db/meta/SqliteMetaImpl.cpp +++ b/core/src/db/meta/SqliteMetaImpl.cpp @@ -804,8 +804,18 @@ SqliteMetaImpl::DropTableIndex(const std::string& table_id) { c(&TableFileSchema::file_type_) == (int)TableFileSchema::BACKUP)); // set table index type to raw + auto groups = ConnectorPtr->select(columns(&TableSchema::metric_type_), + where(c(&TableSchema::table_id_) == table_id)); + + int32_t raw_engine_type = DEFAULT_ENGINE_TYPE; + if (groups.size() == 1) { + int32_t metric_type_ = std::get<0>(groups[0]); + if (engine::utils::IsBinaryMetricType(metric_type_)) { + raw_engine_type = (int32_t)EngineType::FAISS_BIN_IDMAP; + } + } ConnectorPtr->update_all( - set(c(&TableSchema::engine_type_) = DEFAULT_ENGINE_TYPE, c(&TableSchema::index_params_) = "{}"), + set(c(&TableSchema::engine_type_) = raw_engine_type, c(&TableSchema::index_params_) = "{}"), where(c(&TableSchema::table_id_) == table_id)); ENGINE_LOG_DEBUG << "Successfully drop table index, table id = " << table_id; @@ -1189,29 +1199,21 @@ SqliteMetaImpl::FilesByType(const std::string& table_id, const std::vector& file_schema.metric_type_ = table_schema.metric_type_; switch (file_schema.file_type_) { - case (int)TableFileSchema::RAW: - ++raw_count; + case (int)TableFileSchema::RAW:++raw_count; break; - case (int)TableFileSchema::NEW: - ++new_count; + case (int)TableFileSchema::NEW:++new_count; break; - case (int)TableFileSchema::NEW_MERGE: - ++new_merge_count; + case (int)TableFileSchema::NEW_MERGE:++new_merge_count; break; - case (int)TableFileSchema::NEW_INDEX: - ++new_index_count; + case (int)TableFileSchema::NEW_INDEX:++new_index_count; break; - case (int)TableFileSchema::TO_INDEX: - ++to_index_count; + case (int)TableFileSchema::TO_INDEX:++to_index_count; break; - case (int)TableFileSchema::INDEX: - ++index_count; + case (int)TableFileSchema::INDEX:++index_count; break; - case (int)TableFileSchema::BACKUP: - ++backup_count; - break; - default: + case (int)TableFileSchema::BACKUP:++backup_count; break; + default:break; } auto status = utils::GetTableFilePath(options_, file_schema); @@ -1225,29 +1227,23 @@ SqliteMetaImpl::FilesByType(const std::string& table_id, const std::vector& std::string msg = "Get table files by type."; for (int file_type : file_types) { switch (file_type) { - case (int)TableFileSchema::RAW: - msg = msg + " raw files:" + std::to_string(raw_count); + case (int)TableFileSchema::RAW:msg = msg + " raw files:" + std::to_string(raw_count); break; - case (int)TableFileSchema::NEW: - msg = msg + " new files:" + std::to_string(new_count); + case (int)TableFileSchema::NEW:msg = msg + " new files:" + std::to_string(new_count); break; - case (int)TableFileSchema::NEW_MERGE: - msg = msg + " new_merge files:" + std::to_string(new_merge_count); + case (int)TableFileSchema::NEW_MERGE:msg = msg + " new_merge files:" + + std::to_string(new_merge_count); break; - case (int)TableFileSchema::NEW_INDEX: - msg = msg + " new_index files:" + std::to_string(new_index_count); + case (int)TableFileSchema::NEW_INDEX:msg = msg + " new_index files:" + + std::to_string(new_index_count); break; - case (int)TableFileSchema::TO_INDEX: - msg = msg + " to_index files:" + std::to_string(to_index_count); + case (int)TableFileSchema::TO_INDEX:msg = msg + " to_index files:" + std::to_string(to_index_count); break; - case (int)TableFileSchema::INDEX: - msg = msg + " index files:" + std::to_string(index_count); + case (int)TableFileSchema::INDEX:msg = msg + " index files:" + std::to_string(index_count); break; - case (int)TableFileSchema::BACKUP: - msg = msg + " backup files:" + std::to_string(backup_count); - break; - default: + case (int)TableFileSchema::BACKUP:msg = msg + " backup files:" + std::to_string(backup_count); break; + default:break; } } ENGINE_LOG_DEBUG << msg; diff --git a/core/src/scheduler/task/BuildIndexTask.cpp b/core/src/scheduler/task/BuildIndexTask.cpp index 13d3b4a611..666b977731 100644 --- a/core/src/scheduler/task/BuildIndexTask.cpp +++ b/core/src/scheduler/task/BuildIndexTask.cpp @@ -17,6 +17,7 @@ #include #include +#include "db/Utils.h" #include "db/engine/EngineFactory.h" #include "metrics/Metrics.h" #include "scheduler/job/BuildIndexJob.h" @@ -35,8 +36,8 @@ XBuildIndexTask::XBuildIndexTask(TableFileSchemaPtr file, TaskLabelPtr label) if (file->file_type_ == TableFileSchema::FILE_TYPE::RAW || file->file_type_ == TableFileSchema::FILE_TYPE::TO_INDEX || file->file_type_ == TableFileSchema::FILE_TYPE::BACKUP) { - engine_type = server::ValidationUtil::IsBinaryMetricType(file->metric_type_) ? EngineType::FAISS_BIN_IDMAP - : EngineType::FAISS_IDMAP; + engine_type = engine::utils::IsBinaryMetricType(file->metric_type_) ? EngineType::FAISS_BIN_IDMAP + : EngineType::FAISS_IDMAP; } else { engine_type = (EngineType)file->engine_type_; } diff --git a/core/src/scheduler/task/SearchTask.cpp b/core/src/scheduler/task/SearchTask.cpp index 0b32f35ab5..6331c727c7 100644 --- a/core/src/scheduler/task/SearchTask.cpp +++ b/core/src/scheduler/task/SearchTask.cpp @@ -109,8 +109,8 @@ XSearchTask::XSearchTask(const std::shared_ptr& context, TableF if (file->file_type_ == TableFileSchema::FILE_TYPE::RAW || file->file_type_ == TableFileSchema::FILE_TYPE::TO_INDEX || file->file_type_ == TableFileSchema::FILE_TYPE::BACKUP) { - engine_type = server::ValidationUtil::IsBinaryMetricType(file->metric_type_) ? EngineType::FAISS_BIN_IDMAP - : EngineType::FAISS_IDMAP; + engine_type = engine::utils::IsBinaryMetricType(file->metric_type_) ? EngineType::FAISS_BIN_IDMAP + : EngineType::FAISS_IDMAP; } else { engine_type = (EngineType)file->engine_type_; } diff --git a/core/src/server/delivery/request/CreateIndexRequest.cpp b/core/src/server/delivery/request/CreateIndexRequest.cpp index 5836b3cc09..521fe7aa58 100644 --- a/core/src/server/delivery/request/CreateIndexRequest.cpp +++ b/core/src/server/delivery/request/CreateIndexRequest.cpp @@ -10,6 +10,7 @@ // or implied. See the License for the specific language governing permissions and limitations under the License. #include "server/delivery/request/CreateIndexRequest.h" +#include "db/Utils.h" #include "server/Config.h" #include "server/DBWrapper.h" #include "utils/Log.h" @@ -83,7 +84,7 @@ CreateIndexRequest::OnExecute() { status = DBWrapper::DB()->DescribeTable(table_info); int32_t adapter_index_type = index_type_; - if (ValidationUtil::IsBinaryMetricType(table_info.metric_type_)) { // binary vector not allow + if (engine::utils::IsBinaryMetricType(table_info.metric_type_)) { // binary vector not allow if (adapter_index_type == static_cast(engine::EngineType::FAISS_IDMAP)) { adapter_index_type = static_cast(engine::EngineType::FAISS_BIN_IDMAP); } else if (adapter_index_type == static_cast(engine::EngineType::FAISS_IVFFLAT)) { diff --git a/core/src/server/delivery/request/CreateTableRequest.cpp b/core/src/server/delivery/request/CreateTableRequest.cpp index 15d07b8cfd..d92db5a94d 100644 --- a/core/src/server/delivery/request/CreateTableRequest.cpp +++ b/core/src/server/delivery/request/CreateTableRequest.cpp @@ -10,6 +10,7 @@ // or implied. See the License for the specific language governing permissions and limitations under the License. #include "server/delivery/request/CreateTableRequest.h" +#include "db/Utils.h" #include "server/DBWrapper.h" #include "server/delivery/request/BaseRequest.h" #include "utils/Log.h" @@ -78,7 +79,7 @@ CreateTableRequest::OnExecute() { table_info.metric_type_ = metric_type_; // some metric type only support binary vector, adapt the index type - if (ValidationUtil::IsBinaryMetricType(metric_type_)) { + if (engine::utils::IsBinaryMetricType(metric_type_)) { if (table_info.engine_type_ == static_cast(engine::EngineType::FAISS_IDMAP)) { table_info.engine_type_ = static_cast(engine::EngineType::FAISS_BIN_IDMAP); } else if (table_info.engine_type_ == static_cast(engine::EngineType::FAISS_IVFFLAT)) { diff --git a/core/src/server/delivery/request/InsertRequest.cpp b/core/src/server/delivery/request/InsertRequest.cpp index 729dc4d292..a739d86b9f 100644 --- a/core/src/server/delivery/request/InsertRequest.cpp +++ b/core/src/server/delivery/request/InsertRequest.cpp @@ -10,6 +10,7 @@ // or implied. See the License for the specific language governing permissions and limitations under the License. #include "server/delivery/request/InsertRequest.h" +#include "db/Utils.h" #include "server/DBWrapper.h" #include "utils/CommonUtil.h" #include "utils/Log.h" @@ -115,7 +116,7 @@ InsertRequest::OnExecute() { #endif // step 4: some metric type doesn't support float vectors if (!vectors_data_.float_data_.empty()) { // insert float vectors - if (ValidationUtil::IsBinaryMetricType(table_schema.metric_type_)) { + if (engine::utils::IsBinaryMetricType(table_schema.metric_type_)) { return Status(SERVER_INVALID_ROWRECORD_ARRAY, "Table metric type doesn't support float vectors."); } @@ -131,7 +132,7 @@ InsertRequest::OnExecute() { "The vector dimension must be equal to the table dimension."); } } else if (!vectors_data_.binary_data_.empty()) { // insert binary vectors - if (!ValidationUtil::IsBinaryMetricType(table_schema.metric_type_)) { + if (!engine::utils::IsBinaryMetricType(table_schema.metric_type_)) { return Status(SERVER_INVALID_ROWRECORD_ARRAY, "Table metric type doesn't support binary vectors."); } diff --git a/core/src/server/delivery/request/SearchRequest.cpp b/core/src/server/delivery/request/SearchRequest.cpp index 0b12f91f5d..e31be87b40 100644 --- a/core/src/server/delivery/request/SearchRequest.cpp +++ b/core/src/server/delivery/request/SearchRequest.cpp @@ -10,6 +10,7 @@ // or implied. See the License for the specific language governing permissions and limitations under the License. #include "server/delivery/request/SearchRequest.h" +#include "db/Utils.h" #include "server/DBWrapper.h" #include "utils/CommonUtil.h" #include "utils/Log.h" @@ -103,7 +104,7 @@ SearchRequest::OnExecute() { rc.RecordSection("check validation"); // step 4: check metric type - if (ValidationUtil::IsBinaryMetricType(table_schema.metric_type_)) { + if (engine::utils::IsBinaryMetricType(table_schema.metric_type_)) { // check prepared binary data if (vectors_data_.binary_data_.size() % vector_count != 0) { return Status(SERVER_INVALID_ROWRECORD_ARRAY, diff --git a/core/src/utils/ValidationUtil.cpp b/core/src/utils/ValidationUtil.cpp index 84f260c508..616375f08e 100644 --- a/core/src/utils/ValidationUtil.cpp +++ b/core/src/utils/ValidationUtil.cpp @@ -264,12 +264,6 @@ ValidationUtil::ValidateSearchParams(const milvus::json& search_params, const en return Status::OK(); } -bool -ValidationUtil::IsBinaryIndexType(int32_t index_type) { - return (index_type == static_cast(engine::EngineType::FAISS_BIN_IDMAP)) || - (index_type == static_cast(engine::EngineType::FAISS_BIN_IVFFLAT)); -} - Status ValidationUtil::ValidateTableIndexFileSize(int64_t index_file_size) { if (index_file_size <= 0 || index_file_size > INDEX_FILE_SIZE_LIMIT) { @@ -294,13 +288,6 @@ ValidationUtil::ValidateTableIndexMetricType(int32_t metric_type) { return Status::OK(); } -bool -ValidationUtil::IsBinaryMetricType(int32_t metric_type) { - return (metric_type == static_cast(engine::MetricType::HAMMING)) || - (metric_type == static_cast(engine::MetricType::JACCARD)) || - (metric_type == static_cast(engine::MetricType::TANIMOTO)); -} - Status ValidationUtil::ValidateSearchTopk(int64_t top_k, const engine::meta::TableSchema& table_schema) { if (top_k <= 0 || top_k > 2048) { diff --git a/core/src/utils/ValidationUtil.h b/core/src/utils/ValidationUtil.h index 4fa2df1619..481cb31f1b 100644 --- a/core/src/utils/ValidationUtil.h +++ b/core/src/utils/ValidationUtil.h @@ -43,18 +43,12 @@ class ValidationUtil { ValidateSearchParams(const milvus::json& search_params, const engine::meta::TableSchema& table_schema, int64_t topk); - static bool - IsBinaryIndexType(int32_t index_type); - static Status ValidateTableIndexFileSize(int64_t index_file_size); static Status ValidateTableIndexMetricType(int32_t metric_type); - static bool - IsBinaryMetricType(int32_t metric_type); - static Status ValidateSearchTopk(int64_t top_k, const engine::meta::TableSchema& table_schema); From b3bba221d049015c7c159a76dbe214bbc131af39 Mon Sep 17 00:00:00 2001 From: Zhiru Zhu Date: Mon, 9 Mar 2020 11:43:33 +0800 Subject: [PATCH 21/58] update Signed-off-by: Zhiru Zhu --- .../src/config/handler/GpuBuildResHandler.cpp | 61 ++++++ core/src/config/handler/GpuBuildResHandler.h | 44 ++++ .../config/handler/GpuResourcesHandler.cpp | 64 ++++++ core/src/config/handler/GpuResourcesHandler.h | 50 +++++ .../config/handler/GpuSearchResHandler.cpp | 99 +++++++++ core/src/config/handler/GpuSearchResHandler.h | 55 +++++ core/src/db/insert/MemTable.cpp | 38 ++-- core/src/wrapper/VecIndex.h | 7 +- core/unittest/db/test_delete.cpp | 193 ++++++++++++------ core/unittest/db/test_search_by_id.cpp | 60 +++--- 10 files changed, 566 insertions(+), 105 deletions(-) create mode 100644 core/src/config/handler/GpuBuildResHandler.cpp create mode 100644 core/src/config/handler/GpuBuildResHandler.h create mode 100644 core/src/config/handler/GpuResourcesHandler.cpp create mode 100644 core/src/config/handler/GpuResourcesHandler.h create mode 100644 core/src/config/handler/GpuSearchResHandler.cpp create mode 100644 core/src/config/handler/GpuSearchResHandler.h diff --git a/core/src/config/handler/GpuBuildResHandler.cpp b/core/src/config/handler/GpuBuildResHandler.cpp new file mode 100644 index 0000000000..6a57812b85 --- /dev/null +++ b/core/src/config/handler/GpuBuildResHandler.cpp @@ -0,0 +1,61 @@ +// Copyright (C) 2019-2020 Zilliz. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software distributed under the License +// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +// or implied. See the License for the specific language governing permissions and limitations under the License. +#ifdef MILVUS_GPU_VERSION +#include "config/handler/GpuBuildResHandler.h" + +#include +#include + +namespace milvus { +namespace server { + +GpuBuildResHandler::GpuBuildResHandler() { + server::Config& config = server::Config::GetInstance(); + config.GetGpuResourceConfigBuildIndexResources(build_gpus_); +} + +GpuBuildResHandler::~GpuBuildResHandler() { + server::Config& config = server::Config::GetInstance(); + config.CancelCallBack(server::CONFIG_GPU_RESOURCE, server::CONFIG_GPU_RESOURCE_BUILD_INDEX_RESOURCES, identity_); +} + +//////////////////////////////////////////////////////////////// +void +GpuBuildResHandler::OnGpuBuildResChanged(const std::vector& gpus) { + build_gpus_ = gpus; +} + +void +GpuBuildResHandler::AddGpuBuildResListener() { + server::Config& config = server::Config::GetInstance(); + server::ConfigCallBackF lambda = [this](const std::string& value) -> Status { + server::Config& config = server::Config::GetInstance(); + std::vector gpu_ids; + auto status = config.GetGpuResourceConfigSearchResources(gpu_ids); + if (status.ok()) { + OnGpuBuildResChanged(gpu_ids); + } + + return status; + }; + config.RegisterCallBack(server::CONFIG_GPU_RESOURCE, server::CONFIG_GPU_RESOURCE_BUILD_INDEX_RESOURCES, identity_, + lambda); +} + +void +GpuBuildResHandler::RemoveGpuBuildResListener() { + auto& config = server::Config::GetInstance(); + config.CancelCallBack(server::CONFIG_GPU_RESOURCE, server::CONFIG_GPU_RESOURCE_BUILD_INDEX_RESOURCES, identity_); +} + +} // namespace server +} // namespace milvus +#endif diff --git a/core/src/config/handler/GpuBuildResHandler.h b/core/src/config/handler/GpuBuildResHandler.h new file mode 100644 index 0000000000..6047836bcb --- /dev/null +++ b/core/src/config/handler/GpuBuildResHandler.h @@ -0,0 +1,44 @@ +// Copyright (C) 2019-2020 Zilliz. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software distributed under the License +// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +// or implied. See the License for the specific language governing permissions and limitations under the License. +#ifdef MILVUS_GPU_VERSION +#pragma once + +#include + +#include "config/handler/GpuResourcesHandler.h" + +namespace milvus { +namespace server { + +class GpuBuildResHandler : virtual public GpuResourcesHandler { + public: + GpuBuildResHandler(); + + ~GpuBuildResHandler(); + + public: + virtual void + OnGpuBuildResChanged(const std::vector& gpus); + + protected: + void + AddGpuBuildResListener(); + + void + RemoveGpuBuildResListener(); + + protected: + std::vector build_gpus_; +}; + +} // namespace server +} // namespace milvus +#endif diff --git a/core/src/config/handler/GpuResourcesHandler.cpp b/core/src/config/handler/GpuResourcesHandler.cpp new file mode 100644 index 0000000000..65659985d6 --- /dev/null +++ b/core/src/config/handler/GpuResourcesHandler.cpp @@ -0,0 +1,64 @@ + +// Copyright (C) 2019-2020 Zilliz. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software distributed under the License +// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +// or implied. See the License for the specific language governing permissions and limitations under the License. +#ifdef MILVUS_GPU_VERSION +#include "config/handler/GpuResourcesHandler.h" + +namespace milvus { +namespace server { + +GpuResourcesHandler::GpuResourcesHandler() { + server::Config& config = server::Config::GetInstance(); + config.GetGpuResourceConfigEnable(gpu_enable_); +} + +GpuResourcesHandler::~GpuResourcesHandler() { + RemoveGpuEnableListener(); +} + +////////////////////////////////////////////////////////////// +void +GpuResourcesHandler::OnGpuEnableChanged(bool enable) { + gpu_enable_ = enable; +} + +void +GpuResourcesHandler::SetIdentity(const std::string& identity) { + server::Config& config = server::Config::GetInstance(); + config.GenUniqueIdentityID(identity, identity_); +} + +void +GpuResourcesHandler::AddGpuEnableListener() { + server::Config& config = server::Config::GetInstance(); + + server::ConfigCallBackF lambda = [this](const std::string& value) -> Status { + server::Config& config = server::Config::GetInstance(); + bool enable; + auto status = config.GetGpuResourceConfigEnable(enable); + if (status.ok()) { + OnGpuEnableChanged(enable); + } + + return status; + }; + config.RegisterCallBack(server::CONFIG_GPU_RESOURCE, server::CONFIG_GPU_RESOURCE_ENABLE, identity_, lambda); +} + +void +GpuResourcesHandler::RemoveGpuEnableListener() { + server::Config& config = server::Config::GetInstance(); + config.CancelCallBack(server::CONFIG_GPU_RESOURCE, server::CONFIG_GPU_RESOURCE_ENABLE, identity_); +} + +} // namespace server +} // namespace milvus +#endif diff --git a/core/src/config/handler/GpuResourcesHandler.h b/core/src/config/handler/GpuResourcesHandler.h new file mode 100644 index 0000000000..d9e5dc601e --- /dev/null +++ b/core/src/config/handler/GpuResourcesHandler.h @@ -0,0 +1,50 @@ +// Copyright (C) 2019-2020 Zilliz. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software distributed under the License +// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +// or implied. See the License for the specific language governing permissions and limitations under the License. +#ifdef MILVUS_GPU_VERSION +#pragma once + +#include +#include +#include + +#include "server/Config.h" + +namespace milvus { +namespace server { + +class GpuResourcesHandler { + public: + GpuResourcesHandler(); + + ~GpuResourcesHandler(); + + protected: + virtual void + OnGpuEnableChanged(bool enable); + + protected: + void + SetIdentity(const std::string& identity); + + void + AddGpuEnableListener(); + + void + RemoveGpuEnableListener(); + + protected: + bool gpu_enable_ = true; + std::string identity_; +}; + +} // namespace server +} // namespace milvus +#endif diff --git a/core/src/config/handler/GpuSearchResHandler.cpp b/core/src/config/handler/GpuSearchResHandler.cpp new file mode 100644 index 0000000000..781e5aa527 --- /dev/null +++ b/core/src/config/handler/GpuSearchResHandler.cpp @@ -0,0 +1,99 @@ +// Copyright (C) 2019-2020 Zilliz. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software distributed under the License +// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +// or implied. See the License for the specific language governing permissions and limitations under the License. +#ifdef MILVUS_GPU_VERSION +#include "config/handler/GpuSearchResHandler.h" + +#include +#include + +#include "server/Config.h" + +namespace milvus { +namespace server { + +GpuSearchResHandler::GpuSearchResHandler() { + server::Config& config = server::Config::GetInstance(); + + Status s = config.GetEngineConfigGpuSearchThreshold(threshold_); + if (!s.ok()) { + threshold_ = std::numeric_limits::max(); + } + + config.GetGpuResourceConfigSearchResources(search_gpus_); +} + +GpuSearchResHandler::~GpuSearchResHandler() { + RemoveGpuSearchThresholdListener(); + RemoveGpuSearchResListener(); +} + +//////////////////////////////////////////////////////////////////////// +void +GpuSearchResHandler::OnGpuSearchThresholdChanged(int64_t threshold) { + threshold_ = threshold; +} + +void +GpuSearchResHandler::OnGpuSearchResChanged(const std::vector& gpus) { + search_gpus_ = gpus; +} + +void +GpuSearchResHandler::AddGpuSearchThresholdListener() { + server::Config& config = server::Config::GetInstance(); + + server::ConfigCallBackF lambda_gpu_threshold = [this](const std::string& value) -> Status { + server::Config& config = server::Config::GetInstance(); + int64_t threshold; + auto status = config.GetEngineConfigGpuSearchThreshold(threshold); + if (status.ok()) { + OnGpuSearchThresholdChanged(threshold); + } + + return status; + }; + config.RegisterCallBack(server::CONFIG_ENGINE, server::CONFIG_ENGINE_GPU_SEARCH_THRESHOLD, identity_, + lambda_gpu_threshold); +} + +void +GpuSearchResHandler::AddGpuSearchResListener() { + server::Config& config = server::Config::GetInstance(); + + server::ConfigCallBackF lambda_gpu_search_res = [this](const std::string& value) -> Status { + server::Config& config = server::Config::GetInstance(); + std::vector gpu_ids; + auto status = config.GetGpuResourceConfigSearchResources(gpu_ids); + if (status.ok()) { + OnGpuSearchResChanged(gpu_ids); + } + + return status; + }; + config.RegisterCallBack(server::CONFIG_GPU_RESOURCE, server::CONFIG_GPU_RESOURCE_SEARCH_RESOURCES, identity_, + lambda_gpu_search_res); +} + +void +GpuSearchResHandler::RemoveGpuSearchThresholdListener() { + server::Config& config = server::Config::GetInstance(); + config.CancelCallBack(server::CONFIG_ENGINE, server::CONFIG_ENGINE_GPU_SEARCH_THRESHOLD, identity_); +} + +void +GpuSearchResHandler::RemoveGpuSearchResListener() { + auto& config = server::Config::GetInstance(); + config.CancelCallBack(server::CONFIG_GPU_RESOURCE, server::CONFIG_GPU_RESOURCE_SEARCH_RESOURCES, identity_); +} + +} // namespace server +} // namespace milvus +#endif diff --git a/core/src/config/handler/GpuSearchResHandler.h b/core/src/config/handler/GpuSearchResHandler.h new file mode 100644 index 0000000000..390db85bb0 --- /dev/null +++ b/core/src/config/handler/GpuSearchResHandler.h @@ -0,0 +1,55 @@ +// Copyright (C) 2019-2020 Zilliz. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software distributed under the License +// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +// or implied. See the License for the specific language governing permissions and limitations under the License. +#ifdef MILVUS_GPU_VERSION +#pragma once + +#include +#include + +#include "config/handler/GpuResourcesHandler.h" + +namespace milvus { +namespace server { + +class GpuSearchResHandler : virtual public GpuResourcesHandler { + public: + GpuSearchResHandler(); + + ~GpuSearchResHandler(); + + public: + virtual void + OnGpuSearchThresholdChanged(int64_t threshold); + + virtual void + OnGpuSearchResChanged(const std::vector& gpus); + + protected: + void + AddGpuSearchThresholdListener(); + + void + AddGpuSearchResListener(); + + void + RemoveGpuSearchThresholdListener(); + + void + RemoveGpuSearchResListener(); + + protected: + int64_t threshold_ = std::numeric_limits::max(); + std::vector search_gpus_; +}; + +} // namespace server +} // namespace milvus +#endif diff --git a/core/src/db/insert/MemTable.cpp b/core/src/db/insert/MemTable.cpp index f0bdb7dedf..f91efbabe3 100644 --- a/core/src/db/insert/MemTable.cpp +++ b/core/src/db/insert/MemTable.cpp @@ -236,11 +236,27 @@ MemTable::ApplyDeletes() { utils::GetParentPath(table_file.location_, segment_dir); segment::SegmentReader segment_reader(segment_dir); - auto index = - std::static_pointer_cast(cache::CpuCacheMgr::GetInstance()->GetIndex(table_file.location_)); - faiss::ConcurrentBitsetPtr blacklist = nullptr; - if (index != nullptr) { - status = index->GetBlacklist(blacklist); + auto& segment_id = table_file.segment_id_; + meta::TableFilesSchema segment_files; + status = meta_->GetTableFilesBySegmentId(segment_id, segment_files); + if (!status.ok()) { + break; + } + + // Get all index that contains blacklist in cache + std::vector indexes; + std::vector blacklists; + for (auto& file : segment_files) { + auto index = + std::static_pointer_cast(cache::CpuCacheMgr::GetInstance()->GetIndex(file.location_)); + faiss::ConcurrentBitsetPtr blacklist = nullptr; + if (index != nullptr) { + index->GetBlacklist(blacklist); + if (blacklist != nullptr) { + indexes.emplace_back(index); + blacklists.emplace_back(blacklist); + } + } } std::vector uids; @@ -293,7 +309,7 @@ MemTable::ApplyDeletes() { id_bloom_filter_ptr->Remove(uids[i]); } - if (blacklist != nullptr) { + for (auto& blacklist : blacklists) { if (!blacklist->test(i)) { blacklist->set(i); } @@ -308,8 +324,8 @@ MemTable::ApplyDeletes() { << find_diff.count() << " s in total"; ENGINE_LOG_DEBUG << "Setting deleted docs and bloom filter took " << set_diff.count() << " s in total"; - if (index != nullptr) { - index->SetBlacklist(blacklist); + for (auto i = 0; i < indexes.size(); ++i) { + indexes[i]->SetBlacklist(blacklists[i]); } start = std::chrono::high_resolution_clock::now(); @@ -339,12 +355,6 @@ MemTable::ApplyDeletes() { << " s"; // Update table file row count - auto& segment_id = table_file.segment_id_; - meta::TableFilesSchema segment_files; - status = meta_->GetTableFilesBySegmentId(segment_id, segment_files); - if (!status.ok()) { - break; - } for (auto& file : segment_files) { if (file.file_type_ == meta::TableFileSchema::RAW || file.file_type_ == meta::TableFileSchema::TO_INDEX || file.file_type_ == meta::TableFileSchema::INDEX || file.file_type_ == meta::TableFileSchema::BACKUP) { diff --git a/core/src/wrapper/VecIndex.h b/core/src/wrapper/VecIndex.h index f7cf50b5bc..7a98e971fe 100644 --- a/core/src/wrapper/VecIndex.h +++ b/core/src/wrapper/VecIndex.h @@ -12,10 +12,10 @@ #pragma once #include -#include #include #include +#include #include #include @@ -180,13 +180,14 @@ class VecIndex : public cache::DataObj { virtual Status SetBlacklist(faiss::ConcurrentBitsetPtr list) { - ENGINE_LOG_ERROR << "SetBlacklist not support"; + // ENGINE_LOG_ERROR << "SetBlacklist not support"; return Status::OK(); } virtual Status GetBlacklist(faiss::ConcurrentBitsetPtr& list) { - ENGINE_LOG_ERROR << "GetBlacklist not support"; + // ENGINE_LOG_ERROR << "GetBlacklist not support"; + ENGINE_LOG_WARNING << "Deletion on unsupported index type"; return Status::OK(); } diff --git a/core/unittest/db/test_delete.cpp b/core/unittest/db/test_delete.cpp index 4d6051455d..b01802b20a 100644 --- a/core/unittest/db/test_delete.cpp +++ b/core/unittest/db/test_delete.cpp @@ -68,7 +68,7 @@ TEST_F(DeleteTest, delete_in_mem) { auto stat = db_->CreateTable(table_info); milvus::engine::meta::TableSchema table_info_get; - table_info_get.table_id_ = GetTableName(); + table_info_get.table_id_ = table_info.table_id_; stat = db_->DescribeTable(table_info_get); ASSERT_TRUE(stat.ok()); ASSERT_EQ(table_info_get.dimension_, TABLE_DIM); @@ -81,7 +81,7 @@ TEST_F(DeleteTest, delete_in_mem) { xb.id_array_.push_back(i); } - stat = db_->InsertVectors(GetTableName(), "", xb); + stat = db_->InsertVectors(table_info.table_id_, "", xb); ASSERT_TRUE(stat.ok()); std::random_device rd; @@ -105,7 +105,7 @@ TEST_F(DeleteTest, delete_in_mem) { ids_to_delete.emplace_back(kv.first); } - stat = db_->DeleteVectors(GetTableName(), ids_to_delete); + stat = db_->DeleteVectors(table_info.table_id_, ids_to_delete); ASSERT_TRUE(stat.ok()); // std::this_thread::sleep_for(std::chrono::seconds(3)); // ensure raw data write to disk @@ -113,7 +113,7 @@ TEST_F(DeleteTest, delete_in_mem) { ASSERT_TRUE(stat.ok()); uint64_t row_count; - stat = db_->GetTableRowCount(GetTableName(), row_count); + stat = db_->GetTableRowCount(table_info.table_id_, row_count); ASSERT_TRUE(stat.ok()); ASSERT_EQ(row_count, nb - search_vectors.size()); @@ -124,7 +124,7 @@ TEST_F(DeleteTest, delete_in_mem) { std::vector tags; milvus::engine::ResultIds result_ids; milvus::engine::ResultDistances result_distances; - stat = db_->Query(dummy_context_, GetTableName(), tags, topk, nprobe, search, result_ids, result_distances); + stat = db_->Query(dummy_context_, table_info.table_id_, tags, topk, {{"nprobe", nprobe}}, search, result_ids, result_distances); ASSERT_NE(result_ids[0], pair.first); // ASSERT_LT(result_distances[0], 1e-4); ASSERT_GT(result_distances[0], 1); @@ -136,7 +136,7 @@ TEST_F(DeleteTest, delete_on_disk) { auto stat = db_->CreateTable(table_info); milvus::engine::meta::TableSchema table_info_get; - table_info_get.table_id_ = GetTableName(); + table_info_get.table_id_ = table_info.table_id_; stat = db_->DescribeTable(table_info_get); ASSERT_TRUE(stat.ok()); ASSERT_EQ(table_info_get.dimension_, TABLE_DIM); @@ -149,7 +149,7 @@ TEST_F(DeleteTest, delete_on_disk) { xb.id_array_.push_back(i); } - stat = db_->InsertVectors(GetTableName(), "", xb); + stat = db_->InsertVectors(table_info.table_id_, "", xb); ASSERT_TRUE(stat.ok()); std::random_device rd; @@ -173,7 +173,7 @@ TEST_F(DeleteTest, delete_on_disk) { ASSERT_TRUE(stat.ok()); for (auto& kv : search_vectors) { - stat = db_->DeleteVector(GetTableName(), kv.first); + stat = db_->DeleteVector(table_info.table_id_, kv.first); ASSERT_TRUE(stat.ok()); } @@ -181,7 +181,7 @@ TEST_F(DeleteTest, delete_on_disk) { ASSERT_TRUE(stat.ok()); uint64_t row_count; - stat = db_->GetTableRowCount(GetTableName(), row_count); + stat = db_->GetTableRowCount(table_info.table_id_, row_count); ASSERT_TRUE(stat.ok()); ASSERT_EQ(row_count, nb - search_vectors.size()); @@ -192,7 +192,7 @@ TEST_F(DeleteTest, delete_on_disk) { std::vector tags; milvus::engine::ResultIds result_ids; milvus::engine::ResultDistances result_distances; - stat = db_->Query(dummy_context_, GetTableName(), tags, topk, nprobe, search, result_ids, result_distances); + stat = db_->Query(dummy_context_, table_info.table_id_, tags, topk, {{"nprobe", nprobe}}, search, result_ids, result_distances); ASSERT_NE(result_ids[0], pair.first); // ASSERT_LT(result_distances[0], 1e-4); ASSERT_GT(result_distances[0], 1); @@ -204,7 +204,7 @@ TEST_F(DeleteTest, delete_multiple_times) { auto stat = db_->CreateTable(table_info); milvus::engine::meta::TableSchema table_info_get; - table_info_get.table_id_ = GetTableName(); + table_info_get.table_id_ = table_info.table_id_; stat = db_->DescribeTable(table_info_get); ASSERT_TRUE(stat.ok()); ASSERT_EQ(table_info_get.dimension_, TABLE_DIM); @@ -217,7 +217,7 @@ TEST_F(DeleteTest, delete_multiple_times) { xb.id_array_.push_back(i); } - stat = db_->InsertVectors(GetTableName(), "", xb); + stat = db_->InsertVectors(table_info.table_id_, "", xb); ASSERT_TRUE(stat.ok()); std::random_device rd; @@ -243,7 +243,7 @@ TEST_F(DeleteTest, delete_multiple_times) { int topk = 10, nprobe = 10; for (auto& pair : search_vectors) { std::vector to_delete{pair.first}; - stat = db_->DeleteVectors(GetTableName(), to_delete); + stat = db_->DeleteVectors(table_info.table_id_, to_delete); ASSERT_TRUE(stat.ok()); stat = db_->Flush(); @@ -254,7 +254,7 @@ TEST_F(DeleteTest, delete_multiple_times) { std::vector tags; milvus::engine::ResultIds result_ids; milvus::engine::ResultDistances result_distances; - stat = db_->Query(dummy_context_, GetTableName(), tags, topk, nprobe, search, result_ids, result_distances); + stat = db_->Query(dummy_context_, table_info.table_id_, tags, topk, {{"nprobe", nprobe}}, search, result_ids, result_distances); ASSERT_NE(result_ids[0], pair.first); // ASSERT_LT(result_distances[0], 1e-4); ASSERT_GT(result_distances[0], 1); @@ -267,7 +267,7 @@ TEST_F(DeleteTest, delete_with_index) { auto stat = db_->CreateTable(table_info); milvus::engine::meta::TableSchema table_info_get; - table_info_get.table_id_ = GetTableName(); + table_info_get.table_id_ = table_info.table_id_; stat = db_->DescribeTable(table_info_get); ASSERT_TRUE(stat.ok()); ASSERT_EQ(table_info_get.dimension_, TABLE_DIM); @@ -280,7 +280,7 @@ TEST_F(DeleteTest, delete_with_index) { xb.id_array_.push_back(i); } - stat = db_->InsertVectors(GetTableName(), "", xb); + stat = db_->InsertVectors(table_info.table_id_, "", xb); ASSERT_TRUE(stat.ok()); std::random_device rd; @@ -302,7 +302,7 @@ TEST_F(DeleteTest, delete_with_index) { milvus::engine::TableIndex index; index.engine_type_ = (int)milvus::engine::EngineType::FAISS_IVFSQ8; index.extra_params_ = {{"nlist", 100}}; - stat = db_->CreateIndex(GetTableName(), index); + stat = db_->CreateIndex(table_info.table_id_, index); ASSERT_TRUE(stat.ok()); // std::this_thread::sleep_for(std::chrono::seconds(3)); // ensure raw data write to disk @@ -313,13 +313,13 @@ TEST_F(DeleteTest, delete_with_index) { for (auto& kv : search_vectors) { ids_to_delete.emplace_back(kv.first); } - stat = db_->DeleteVectors(GetTableName(), ids_to_delete); + stat = db_->DeleteVectors(table_info.table_id_, ids_to_delete); stat = db_->Flush(); ASSERT_TRUE(stat.ok()); uint64_t row_count; - stat = db_->GetTableRowCount(GetTableName(), row_count); + stat = db_->GetTableRowCount(table_info.table_id_, row_count); ASSERT_TRUE(stat.ok()); ASSERT_EQ(row_count, nb - ids_to_delete.size()); @@ -330,7 +330,84 @@ TEST_F(DeleteTest, delete_with_index) { std::vector tags; milvus::engine::ResultIds result_ids; milvus::engine::ResultDistances result_distances; - stat = db_->Query(dummy_context_, GetTableName(), tags, topk, nprobe, search, result_ids, result_distances); + stat = db_->Query(dummy_context_, table_info.table_id_, tags, topk, {{"nprobe", nprobe}}, search, result_ids, result_distances); + ASSERT_NE(result_ids[0], pair.first); + // ASSERT_LT(result_distances[0], 1e-4); + ASSERT_GT(result_distances[0], 1); + } +} + +TEST_F(DeleteTest, delete_multiple_times_with_index) { + milvus::engine::meta::TableSchema table_info = BuildTableSchema(); + auto stat = db_->CreateTable(table_info); + + milvus::engine::meta::TableSchema table_info_get; + table_info_get.table_id_ = table_info.table_id_; + stat = db_->DescribeTable(table_info_get); + ASSERT_TRUE(stat.ok()); + ASSERT_EQ(table_info_get.dimension_, TABLE_DIM); + + int64_t nb = 100000; + milvus::engine::VectorsData xb; + BuildVectors(nb, xb); + + for (int64_t i = 0; i < nb; i++) { + xb.id_array_.push_back(i); + } + + stat = db_->InsertVectors(table_info.table_id_, "", xb); + ASSERT_TRUE(stat.ok()); + + std::random_device rd; + std::mt19937 gen(rd()); + std::uniform_int_distribution dis(0, nb - 1); + + int64_t num_query = 10; + std::map search_vectors; + for (int64_t i = 0; i < num_query; ++i) { + int64_t index = dis(gen); + milvus::engine::VectorsData search; + search.vector_count_ = 1; + for (int64_t j = 0; j < TABLE_DIM; j++) { + search.float_data_.push_back(xb.float_data_[index * TABLE_DIM + j]); + } + search_vectors.insert(std::make_pair(xb.id_array_[index], search)); + } + + // std::this_thread::sleep_for(std::chrono::seconds(3)); // ensure raw data write to disk + stat = db_->Flush(); + ASSERT_TRUE(stat.ok()); + + milvus::engine::TableIndex index; + index.engine_type_ = (int)milvus::engine::EngineType::FAISS_IVFFLAT; + index.extra_params_ = {{"nlist", 1}}; + stat = db_->CreateIndex(table_info.table_id_, index); + ASSERT_TRUE(stat.ok()); + + int topk = 10, nprobe = 10; + int deleted = 0; + for (auto& pair : search_vectors) { + std::vector to_delete{pair.first}; + stat = db_->DeleteVectors(table_info.table_id_, to_delete); + ASSERT_TRUE(stat.ok()); + + stat = db_->Flush(); + ASSERT_TRUE(stat.ok()); + + ++deleted; + + uint64_t row_count; + stat = db_->GetTableRowCount(table_info.table_id_, row_count); + ASSERT_TRUE(stat.ok()); + ASSERT_EQ(row_count, nb - deleted); + + auto& search = pair.second; + + std::vector tags; + milvus::engine::ResultIds result_ids; + milvus::engine::ResultDistances result_distances; + stat = db_->Query(dummy_context_, table_info.table_id_, tags, topk, {{"nprobe", nprobe}}, search, result_ids, result_distances); + ASSERT_TRUE(stat.ok()); ASSERT_NE(result_ids[0], pair.first); // ASSERT_LT(result_distances[0], 1e-4); ASSERT_GT(result_distances[0], 1); @@ -342,7 +419,7 @@ TEST_F(DeleteTest, delete_single_vector) { auto stat = db_->CreateTable(table_info); milvus::engine::meta::TableSchema table_info_get; - table_info_get.table_id_ = GetTableName(); + table_info_get.table_id_ = table_info.table_id_; stat = db_->DescribeTable(table_info_get); ASSERT_TRUE(stat.ok()); ASSERT_EQ(table_info_get.dimension_, TABLE_DIM); @@ -351,21 +428,21 @@ TEST_F(DeleteTest, delete_single_vector) { milvus::engine::VectorsData xb; BuildVectors(nb, xb); - stat = db_->InsertVectors(GetTableName(), "", xb); + stat = db_->InsertVectors(table_info.table_id_, "", xb); ASSERT_TRUE(stat.ok()); // std::this_thread::sleep_for(std::chrono::seconds(3)); // ensure raw data write to disk stat = db_->Flush(); ASSERT_TRUE(stat.ok()); - stat = db_->DeleteVectors(GetTableName(), xb.id_array_); + stat = db_->DeleteVectors(table_info.table_id_, xb.id_array_); ASSERT_TRUE(stat.ok()); stat = db_->Flush(); ASSERT_TRUE(stat.ok()); uint64_t row_count; - stat = db_->GetTableRowCount(GetTableName(), row_count); + stat = db_->GetTableRowCount(table_info.table_id_, row_count); ASSERT_TRUE(stat.ok()); ASSERT_EQ(row_count, 0); @@ -375,7 +452,7 @@ TEST_F(DeleteTest, delete_single_vector) { std::vector tags; milvus::engine::ResultIds result_ids; milvus::engine::ResultDistances result_distances; - stat = db_->Query(dummy_context_, GetTableName(), tags, topk, json_params, xb, result_ids, result_distances); + stat = db_->Query(dummy_context_, table_info.table_id_, tags, topk, json_params, xb, result_ids, result_distances); ASSERT_TRUE(result_ids.empty()); ASSERT_TRUE(result_distances.empty()); // ASSERT_EQ(result_ids[0], -1); @@ -388,7 +465,7 @@ TEST_F(DeleteTest, delete_add_create_index) { auto stat = db_->CreateTable(table_info); milvus::engine::meta::TableSchema table_info_get; - table_info_get.table_id_ = GetTableName(); + table_info_get.table_id_ = table_info.table_id_; stat = db_->DescribeTable(table_info_get); ASSERT_TRUE(stat.ok()); ASSERT_EQ(table_info_get.dimension_, TABLE_DIM); @@ -397,7 +474,7 @@ TEST_F(DeleteTest, delete_add_create_index) { milvus::engine::VectorsData xb; BuildVectors(nb, xb); - stat = db_->InsertVectors(GetTableName(), "", xb); + stat = db_->InsertVectors(table_info.table_id_, "", xb); ASSERT_TRUE(stat.ok()); // stat = db_->Flush(); @@ -405,27 +482,27 @@ TEST_F(DeleteTest, delete_add_create_index) { milvus::engine::TableIndex index; index.engine_type_ = (int)milvus::engine::EngineType::FAISS_IVFSQ8; index.extra_params_ = {{"nlist", 100}}; - stat = db_->CreateIndex(GetTableName(), index); + stat = db_->CreateIndex(table_info.table_id_, index); ASSERT_TRUE(stat.ok()); std::vector ids_to_delete; ids_to_delete.emplace_back(xb.id_array_.front()); - stat = db_->DeleteVectors(GetTableName(), ids_to_delete); + stat = db_->DeleteVectors(table_info.table_id_, ids_to_delete); ASSERT_TRUE(stat.ok()); milvus::engine::VectorsData xb2 = xb; xb2.id_array_.clear(); // same vector, different id - stat = db_->InsertVectors(GetTableName(), "", xb2); + stat = db_->InsertVectors(table_info.table_id_, "", xb2); ASSERT_TRUE(stat.ok()); // stat = db_->Flush(); // ASSERT_TRUE(stat.ok()); - stat = db_->CreateIndex(GetTableName(), index); + stat = db_->CreateIndex(table_info.table_id_, index); ASSERT_TRUE(stat.ok()); uint64_t row_count; - stat = db_->GetTableRowCount(GetTableName(), row_count); + stat = db_->GetTableRowCount(table_info.table_id_, row_count); ASSERT_TRUE(stat.ok()); ASSERT_EQ(row_count, nb * 2 - 1); @@ -439,14 +516,14 @@ TEST_F(DeleteTest, delete_add_create_index) { qb.float_data_.resize(TABLE_DIM); qb.vector_count_ = 1; qb.id_array_.clear(); - stat = db_->Query(dummy_context_, GetTableName(), tags, topk, json_params, qb, result_ids, result_distances); + stat = db_->Query(dummy_context_, table_info.table_id_, tags, topk, json_params, qb, result_ids, result_distances); ASSERT_EQ(result_ids[0], xb2.id_array_.front()); ASSERT_LT(result_distances[0], 1e-4); result_ids.clear(); result_distances.clear(); - stat = db_->QueryByID(dummy_context_, GetTableName(), tags, topk, json_params, ids_to_delete.front(), result_ids, + stat = db_->QueryByID(dummy_context_, table_info.table_id_, tags, topk, json_params, ids_to_delete.front(), result_ids, result_distances); ASSERT_EQ(result_ids[0], -1); ASSERT_EQ(result_distances[0], std::numeric_limits::max()); @@ -457,7 +534,7 @@ TEST_F(DeleteTest, delete_add_auto_flush) { auto stat = db_->CreateTable(table_info); milvus::engine::meta::TableSchema table_info_get; - table_info_get.table_id_ = GetTableName(); + table_info_get.table_id_ = table_info.table_id_; stat = db_->DescribeTable(table_info_get); ASSERT_TRUE(stat.ok()); ASSERT_EQ(table_info_get.dimension_, TABLE_DIM); @@ -466,7 +543,7 @@ TEST_F(DeleteTest, delete_add_auto_flush) { milvus::engine::VectorsData xb; BuildVectors(nb, xb); - stat = db_->InsertVectors(GetTableName(), "", xb); + stat = db_->InsertVectors(table_info.table_id_, "", xb); ASSERT_TRUE(stat.ok()); std::this_thread::sleep_for(std::chrono::seconds(2)); @@ -475,28 +552,28 @@ TEST_F(DeleteTest, delete_add_auto_flush) { // ASSERT_TRUE(stat.ok()); // milvus::engine::TableIndex index; // index.engine_type_ = (int)milvus::engine::EngineType::FAISS_IVFSQ8; - // stat = db_->CreateIndex(GetTableName(), index); + // stat = db_->CreateIndex(table_info.table_id_, index); // ASSERT_TRUE(stat.ok()); std::vector ids_to_delete; ids_to_delete.emplace_back(xb.id_array_.front()); - stat = db_->DeleteVectors(GetTableName(), ids_to_delete); + stat = db_->DeleteVectors(table_info.table_id_, ids_to_delete); ASSERT_TRUE(stat.ok()); milvus::engine::VectorsData xb2 = xb; xb2.id_array_.clear(); // same vector, different id - stat = db_->InsertVectors(GetTableName(), "", xb2); + stat = db_->InsertVectors(table_info.table_id_, "", xb2); ASSERT_TRUE(stat.ok()); std::this_thread::sleep_for(std::chrono::seconds(2)); // stat = db_->Flush(); // ASSERT_TRUE(stat.ok()); - // stat = db_->CreateIndex(GetTableName(), index); + // stat = db_->CreateIndex(table_info.table_id_, index); // ASSERT_TRUE(stat.ok()); uint64_t row_count; - stat = db_->GetTableRowCount(GetTableName(), row_count); + stat = db_->GetTableRowCount(table_info.table_id_, row_count); ASSERT_TRUE(stat.ok()); ASSERT_EQ(row_count, nb * 2 - 1); @@ -510,14 +587,14 @@ TEST_F(DeleteTest, delete_add_auto_flush) { qb.float_data_.resize(TABLE_DIM); qb.vector_count_ = 1; qb.id_array_.clear(); - stat = db_->Query(dummy_context_, GetTableName(), tags, topk, json_params, qb, result_ids, result_distances); + stat = db_->Query(dummy_context_, table_info.table_id_, tags, topk, json_params, qb, result_ids, result_distances); ASSERT_EQ(result_ids[0], xb2.id_array_.front()); ASSERT_LT(result_distances[0], 1e-4); result_ids.clear(); result_distances.clear(); - stat = db_->QueryByID(dummy_context_, GetTableName(), tags, topk, nprobe, ids_to_delete.front(), result_ids, + stat = db_->QueryByID(dummy_context_, table_info.table_id_, tags, topk, {{"nprobe", nprobe}}, ids_to_delete.front(), result_ids, result_distances); ASSERT_EQ(result_ids[0], -1); ASSERT_EQ(result_distances[0], std::numeric_limits::max()); @@ -528,7 +605,7 @@ TEST_F(CompactTest, compact_basic) { auto stat = db_->CreateTable(table_info); milvus::engine::meta::TableSchema table_info_get; - table_info_get.table_id_ = GetTableName(); + table_info_get.table_id_ = table_info.table_id_; stat = db_->DescribeTable(table_info_get); ASSERT_TRUE(stat.ok()); ASSERT_EQ(table_info_get.dimension_, TABLE_DIM); @@ -537,7 +614,7 @@ TEST_F(CompactTest, compact_basic) { milvus::engine::VectorsData xb; BuildVectors(nb, xb); - stat = db_->InsertVectors(GetTableName(), "", xb); + stat = db_->InsertVectors(table_info.table_id_, "", xb); ASSERT_TRUE(stat.ok()); stat = db_->Flush(); @@ -546,18 +623,18 @@ TEST_F(CompactTest, compact_basic) { std::vector ids_to_delete; ids_to_delete.emplace_back(xb.id_array_.front()); ids_to_delete.emplace_back(xb.id_array_.back()); - stat = db_->DeleteVectors(GetTableName(), ids_to_delete); + stat = db_->DeleteVectors(table_info.table_id_, ids_to_delete); ASSERT_TRUE(stat.ok()); stat = db_->Flush(); ASSERT_TRUE(stat.ok()); uint64_t row_count; - stat = db_->GetTableRowCount(GetTableName(), row_count); + stat = db_->GetTableRowCount(table_info.table_id_, row_count); ASSERT_TRUE(stat.ok()); ASSERT_EQ(row_count, nb - 2); - stat = db_->Compact(GetTableName()); + stat = db_->Compact(table_info.table_id_); ASSERT_TRUE(stat.ok()); const int topk = 1, nprobe = 1; @@ -570,7 +647,7 @@ TEST_F(CompactTest, compact_basic) { for (auto& id : ids_to_delete) { stat = - db_->QueryByID(dummy_context_, GetTableName(), tags, topk, json_params, id, result_ids, result_distances); + db_->QueryByID(dummy_context_, table_info.table_id_, tags, topk, json_params, id, result_ids, result_distances); ASSERT_EQ(result_ids[0], -1); ASSERT_EQ(result_distances[0], std::numeric_limits::max()); } @@ -583,7 +660,7 @@ TEST_F(CompactTest, compact_with_index) { auto stat = db_->CreateTable(table_info); milvus::engine::meta::TableSchema table_info_get; - table_info_get.table_id_ = GetTableName(); + table_info_get.table_id_ = table_info.table_id_; stat = db_->DescribeTable(table_info_get); ASSERT_TRUE(stat.ok()); ASSERT_EQ(table_info_get.dimension_, TABLE_DIM); @@ -597,7 +674,7 @@ TEST_F(CompactTest, compact_with_index) { xb.id_array_.emplace_back(i); } - stat = db_->InsertVectors(GetTableName(), "", xb); + stat = db_->InsertVectors(table_info.table_id_, "", xb); ASSERT_TRUE(stat.ok()); std::random_device rd; @@ -618,7 +695,7 @@ TEST_F(CompactTest, compact_with_index) { milvus::engine::TableIndex index; index.engine_type_ = (int)milvus::engine::EngineType::FAISS_IVFSQ8; - stat = db_->CreateIndex(GetTableName(), index); + stat = db_->CreateIndex(table_info.table_id_, index); ASSERT_TRUE(stat.ok()); stat = db_->Flush(); @@ -628,25 +705,25 @@ TEST_F(CompactTest, compact_with_index) { for (auto& kv : search_vectors) { ids_to_delete.emplace_back(kv.first); } - stat = db_->DeleteVectors(GetTableName(), ids_to_delete); + stat = db_->DeleteVectors(table_info.table_id_, ids_to_delete); stat = db_->Flush(); ASSERT_TRUE(stat.ok()); uint64_t row_count; - stat = db_->GetTableRowCount(GetTableName(), row_count); + stat = db_->GetTableRowCount(table_info.table_id_, row_count); ASSERT_TRUE(stat.ok()); ASSERT_EQ(row_count, nb - ids_to_delete.size()); - stat = db_->Compact(GetTableName()); + stat = db_->Compact(table_info.table_id_); ASSERT_TRUE(stat.ok()); - stat = db_->GetTableRowCount(GetTableName(), row_count); + stat = db_->GetTableRowCount(table_info.table_id_, row_count); ASSERT_TRUE(stat.ok()); ASSERT_EQ(row_count, nb - ids_to_delete.size()); milvus::engine::TableIndex table_index; - stat = db_->DescribeIndex(GetTableName(), table_index); + stat = db_->DescribeIndex(table_info.table_id_, table_index); ASSERT_TRUE(stat.ok()); ASSERT_FLOAT_EQ(table_index.engine_type_, index.engine_type_); @@ -660,7 +737,7 @@ TEST_F(CompactTest, compact_with_index) { milvus::engine::ResultIds result_ids; milvus::engine::ResultDistances result_distances; stat = - db_->Query(dummy_context_, GetTableName(), tags, topk, json_params, search, result_ids, result_distances); + db_->Query(dummy_context_, table_info.table_id_, tags, topk, json_params, search, result_ids, result_distances); ASSERT_NE(result_ids[0], pair.first); // ASSERT_LT(result_distances[0], 1e-4); ASSERT_GT(result_distances[0], 1); diff --git a/core/unittest/db/test_search_by_id.cpp b/core/unittest/db/test_search_by_id.cpp index a4edb301b0..5c1ad7993f 100644 --- a/core/unittest/db/test_search_by_id.cpp +++ b/core/unittest/db/test_search_by_id.cpp @@ -68,7 +68,7 @@ TEST_F(SearchByIdTest, basic) { auto stat = db_->CreateTable(table_info); milvus::engine::meta::TableSchema table_info_get; - table_info_get.table_id_ = GetTableName(); + table_info_get.table_id_ = table_info.table_id_; stat = db_->DescribeTable(table_info_get); ASSERT_TRUE(stat.ok()); ASSERT_EQ(table_info_get.dimension_, TABLE_DIM); @@ -81,7 +81,7 @@ TEST_F(SearchByIdTest, basic) { xb.id_array_.push_back(i); } - stat = db_->InsertVectors(GetTableName(), "", xb); + stat = db_->InsertVectors(table_info.table_id_, "", xb); ASSERT_TRUE(stat.ok()); std::random_device rd; @@ -108,7 +108,7 @@ TEST_F(SearchByIdTest, basic) { milvus::engine::ResultIds result_ids; milvus::engine::ResultDistances result_distances; - stat = db_->QueryByID(dummy_context_, GetTableName(), tags, topk, json_params, i, result_ids, result_distances); + stat = db_->QueryByID(dummy_context_, table_info.table_id_, tags, topk, json_params, i, result_ids, result_distances); ASSERT_EQ(result_ids[0], i); ASSERT_LT(result_distances[0], 1e-4); } @@ -119,7 +119,7 @@ TEST_F(SearchByIdTest, with_index) { auto stat = db_->CreateTable(table_info); milvus::engine::meta::TableSchema table_info_get; - table_info_get.table_id_ = GetTableName(); + table_info_get.table_id_ = table_info.table_id_; stat = db_->DescribeTable(table_info_get); ASSERT_TRUE(stat.ok()); ASSERT_EQ(table_info_get.dimension_, TABLE_DIM); @@ -132,7 +132,7 @@ TEST_F(SearchByIdTest, with_index) { xb.id_array_.push_back(i); } - stat = db_->InsertVectors(GetTableName(), "", xb); + stat = db_->InsertVectors(table_info.table_id_, "", xb); ASSERT_TRUE(stat.ok()); std::random_device rd; @@ -153,7 +153,7 @@ TEST_F(SearchByIdTest, with_index) { milvus::engine::TableIndex index; index.engine_type_ = (int)milvus::engine::EngineType::FAISS_IVFSQ8; index.extra_params_ = {{"nlist", 10}}; - stat = db_->CreateIndex(GetTableName(), index); + stat = db_->CreateIndex(table_info.table_id_, index); ASSERT_TRUE(stat.ok()); const int topk = 10, nprobe = 10; @@ -165,7 +165,7 @@ TEST_F(SearchByIdTest, with_index) { milvus::engine::ResultIds result_ids; milvus::engine::ResultDistances result_distances; - stat = db_->QueryByID(dummy_context_, GetTableName(), tags, topk, json_params, i, result_ids, result_distances); + stat = db_->QueryByID(dummy_context_, table_info.table_id_, tags, topk, json_params, i, result_ids, result_distances); ASSERT_EQ(result_ids[0], i); ASSERT_LT(result_distances[0], 1e-3); } @@ -176,7 +176,7 @@ TEST_F(SearchByIdTest, with_delete) { auto stat = db_->CreateTable(table_info); milvus::engine::meta::TableSchema table_info_get; - table_info_get.table_id_ = GetTableName(); + table_info_get.table_id_ = table_info.table_id_; stat = db_->DescribeTable(table_info_get); ASSERT_TRUE(stat.ok()); ASSERT_EQ(table_info_get.dimension_, TABLE_DIM); @@ -189,7 +189,7 @@ TEST_F(SearchByIdTest, with_delete) { xb.id_array_.push_back(i); } - stat = db_->InsertVectors(GetTableName(), "", xb); + stat = db_->InsertVectors(table_info.table_id_, "", xb); ASSERT_TRUE(stat.ok()); std::random_device rd; @@ -211,7 +211,7 @@ TEST_F(SearchByIdTest, with_delete) { for (auto& id : ids_to_search) { ids_to_delete.emplace_back(id); } - stat = db_->DeleteVectors(GetTableName(), ids_to_delete); + stat = db_->DeleteVectors(table_info.table_id_, ids_to_delete); stat = db_->Flush(); ASSERT_TRUE(stat.ok()); @@ -225,7 +225,7 @@ TEST_F(SearchByIdTest, with_delete) { milvus::engine::ResultIds result_ids; milvus::engine::ResultDistances result_distances; - stat = db_->QueryByID(dummy_context_, GetTableName(), tags, topk, json_params, i, result_ids, result_distances); + stat = db_->QueryByID(dummy_context_, table_info.table_id_, tags, topk, json_params, i, result_ids, result_distances); ASSERT_EQ(result_ids[0], -1); ASSERT_EQ(result_distances[0], std::numeric_limits::max()); } @@ -236,7 +236,7 @@ TEST_F(GetVectorByIdTest, basic) { auto stat = db_->CreateTable(table_info); milvus::engine::meta::TableSchema table_info_get; - table_info_get.table_id_ = GetTableName(); + table_info_get.table_id_ = table_info.table_id_; stat = db_->DescribeTable(table_info_get); ASSERT_TRUE(stat.ok()); ASSERT_EQ(table_info_get.dimension_, TABLE_DIM); @@ -249,7 +249,7 @@ TEST_F(GetVectorByIdTest, basic) { xb.id_array_.push_back(i); } - stat = db_->InsertVectors(GetTableName(), "", xb); + stat = db_->InsertVectors(table_info.table_id_, "", xb); ASSERT_TRUE(stat.ok()); std::random_device rd; @@ -277,11 +277,11 @@ TEST_F(GetVectorByIdTest, basic) { milvus::engine::ResultDistances result_distances; milvus::engine::VectorsData vector; - stat = db_->GetVectorByID(GetTableName(), id, vector); + stat = db_->GetVectorByID(table_info.table_id_, id, vector); ASSERT_TRUE(stat.ok()); stat = - db_->Query(dummy_context_, GetTableName(), tags, topk, json_params, vector, result_ids, result_distances); + db_->Query(dummy_context_, table_info.table_id_, tags, topk, json_params, vector, result_ids, result_distances); ASSERT_TRUE(stat.ok()); ASSERT_EQ(result_ids[0], id); ASSERT_LT(result_distances[0], 1e-4); @@ -293,7 +293,7 @@ TEST_F(GetVectorByIdTest, with_index) { auto stat = db_->CreateTable(table_info); milvus::engine::meta::TableSchema table_info_get; - table_info_get.table_id_ = GetTableName(); + table_info_get.table_id_ = table_info.table_id_; stat = db_->DescribeTable(table_info_get); ASSERT_TRUE(stat.ok()); ASSERT_EQ(table_info_get.dimension_, TABLE_DIM); @@ -306,7 +306,7 @@ TEST_F(GetVectorByIdTest, with_index) { xb.id_array_.push_back(i); } - stat = db_->InsertVectors(GetTableName(), "", xb); + stat = db_->InsertVectors(table_info.table_id_, "", xb); ASSERT_TRUE(stat.ok()); std::random_device rd; @@ -327,7 +327,7 @@ TEST_F(GetVectorByIdTest, with_index) { milvus::engine::TableIndex index; index.extra_params_ = {{"nlist", 10}}; index.engine_type_ = (int)milvus::engine::EngineType::FAISS_IVFSQ8; - stat = db_->CreateIndex(GetTableName(), index); + stat = db_->CreateIndex(table_info.table_id_, index); ASSERT_TRUE(stat.ok()); const int topk = 10, nprobe = 10; @@ -340,11 +340,11 @@ TEST_F(GetVectorByIdTest, with_index) { milvus::engine::ResultDistances result_distances; milvus::engine::VectorsData vector; - stat = db_->GetVectorByID(GetTableName(), id, vector); + stat = db_->GetVectorByID(table_info.table_id_, id, vector); ASSERT_TRUE(stat.ok()); stat = - db_->Query(dummy_context_, GetTableName(), tags, topk, json_params, vector, result_ids, result_distances); + db_->Query(dummy_context_, table_info.table_id_, tags, topk, json_params, vector, result_ids, result_distances); ASSERT_EQ(result_ids[0], id); ASSERT_LT(result_distances[0], 1e-3); } @@ -355,7 +355,7 @@ TEST_F(GetVectorByIdTest, with_delete) { auto stat = db_->CreateTable(table_info); milvus::engine::meta::TableSchema table_info_get; - table_info_get.table_id_ = GetTableName(); + table_info_get.table_id_ = table_info.table_id_; stat = db_->DescribeTable(table_info_get); ASSERT_TRUE(stat.ok()); ASSERT_EQ(table_info_get.dimension_, TABLE_DIM); @@ -368,7 +368,7 @@ TEST_F(GetVectorByIdTest, with_delete) { xb.id_array_.push_back(i); } - stat = db_->InsertVectors(GetTableName(), "", xb); + stat = db_->InsertVectors(table_info.table_id_, "", xb); ASSERT_TRUE(stat.ok()); std::random_device rd; @@ -390,7 +390,7 @@ TEST_F(GetVectorByIdTest, with_delete) { for (auto& id : ids_to_search) { ids_to_delete.emplace_back(id); } - stat = db_->DeleteVectors(GetTableName(), ids_to_delete); + stat = db_->DeleteVectors(table_info.table_id_, ids_to_delete); stat = db_->Flush(); ASSERT_TRUE(stat.ok()); @@ -402,7 +402,7 @@ TEST_F(GetVectorByIdTest, with_delete) { milvus::engine::ResultDistances result_distances; milvus::engine::VectorsData vector; - stat = db_->GetVectorByID(GetTableName(), id, vector); + stat = db_->GetVectorByID(table_info.table_id_, id, vector); ASSERT_TRUE(stat.ok()); ASSERT_TRUE(vector.float_data_.empty()); ASSERT_EQ(vector.vector_count_, 0); @@ -419,7 +419,7 @@ TEST_F(SearchByIdTest, BINARY) { ASSERT_TRUE(stat.ok()); milvus::engine::meta::TableSchema table_info_get; - table_info_get.table_id_ = GetTableName(); + table_info_get.table_id_ = table_info.table_id_; stat = db_->DescribeTable(table_info_get); ASSERT_TRUE(stat.ok()); ASSERT_EQ(table_info_get.dimension_, TABLE_DIM); @@ -445,7 +445,7 @@ TEST_F(SearchByIdTest, BINARY) { vectors.id_array_.emplace_back(k * nb + i); } - stat = db_->InsertVectors(GetTableName(), "", vectors); + stat = db_->InsertVectors(table_info.table_id_, "", vectors); ASSERT_TRUE(stat.ok()); } @@ -465,7 +465,7 @@ TEST_F(SearchByIdTest, BINARY) { ASSERT_TRUE(stat.ok()); uint64_t row_count; - stat = db_->GetTableRowCount(GetTableName(), row_count); + stat = db_->GetTableRowCount(table_info.table_id_, row_count); ASSERT_TRUE(stat.ok()); ASSERT_EQ(row_count, nb * insert_loop); @@ -479,12 +479,12 @@ TEST_F(SearchByIdTest, BINARY) { milvus::engine::ResultDistances result_distances; milvus::engine::VectorsData vector; - stat = db_->GetVectorByID(GetTableName(), id, vector); + stat = db_->GetVectorByID(table_info.table_id_, id, vector); ASSERT_TRUE(stat.ok()); ASSERT_EQ(vector.vector_count_, 1); stat = - db_->Query(dummy_context_, GetTableName(), tags, topk, json_params, vector, result_ids, result_distances); + db_->Query(dummy_context_, table_info.table_id_, tags, topk, json_params, vector, result_ids, result_distances); ASSERT_TRUE(stat.ok()); ASSERT_EQ(result_ids[0], id); ASSERT_LT(result_distances[0], 1e-4); @@ -494,7 +494,7 @@ TEST_F(SearchByIdTest, BINARY) { result_distances.clear(); stat = - db_->QueryByID(dummy_context_, GetTableName(), tags, topk, json_params, id, result_ids, result_distances); + db_->QueryByID(dummy_context_, table_info.table_id_, tags, topk, json_params, id, result_ids, result_distances); ASSERT_TRUE(stat.ok()); ASSERT_EQ(result_ids[0], id); ASSERT_LT(result_distances[0], 1e-4); From 7f3abf1c6427603a538c4f55fa3659541773be4f Mon Sep 17 00:00:00 2001 From: Zhiru Zhu Date: Mon, 9 Mar 2020 11:44:28 +0800 Subject: [PATCH 22/58] update Signed-off-by: Zhiru Zhu --- core/unittest/db/test_delete.cpp | 31 +++++++++++++++----------- core/unittest/db/test_search_by_id.cpp | 25 ++++++++++++--------- 2 files changed, 32 insertions(+), 24 deletions(-) diff --git a/core/unittest/db/test_delete.cpp b/core/unittest/db/test_delete.cpp index b01802b20a..15618a950e 100644 --- a/core/unittest/db/test_delete.cpp +++ b/core/unittest/db/test_delete.cpp @@ -124,7 +124,8 @@ TEST_F(DeleteTest, delete_in_mem) { std::vector tags; milvus::engine::ResultIds result_ids; milvus::engine::ResultDistances result_distances; - stat = db_->Query(dummy_context_, table_info.table_id_, tags, topk, {{"nprobe", nprobe}}, search, result_ids, result_distances); + stat = db_->Query(dummy_context_, table_info.table_id_, tags, topk, {{"nprobe", nprobe}}, search, result_ids, + result_distances); ASSERT_NE(result_ids[0], pair.first); // ASSERT_LT(result_distances[0], 1e-4); ASSERT_GT(result_distances[0], 1); @@ -192,7 +193,8 @@ TEST_F(DeleteTest, delete_on_disk) { std::vector tags; milvus::engine::ResultIds result_ids; milvus::engine::ResultDistances result_distances; - stat = db_->Query(dummy_context_, table_info.table_id_, tags, topk, {{"nprobe", nprobe}}, search, result_ids, result_distances); + stat = db_->Query(dummy_context_, table_info.table_id_, tags, topk, {{"nprobe", nprobe}}, search, result_ids, + result_distances); ASSERT_NE(result_ids[0], pair.first); // ASSERT_LT(result_distances[0], 1e-4); ASSERT_GT(result_distances[0], 1); @@ -254,7 +256,8 @@ TEST_F(DeleteTest, delete_multiple_times) { std::vector tags; milvus::engine::ResultIds result_ids; milvus::engine::ResultDistances result_distances; - stat = db_->Query(dummy_context_, table_info.table_id_, tags, topk, {{"nprobe", nprobe}}, search, result_ids, result_distances); + stat = db_->Query(dummy_context_, table_info.table_id_, tags, topk, {{"nprobe", nprobe}}, search, result_ids, + result_distances); ASSERT_NE(result_ids[0], pair.first); // ASSERT_LT(result_distances[0], 1e-4); ASSERT_GT(result_distances[0], 1); @@ -330,7 +333,8 @@ TEST_F(DeleteTest, delete_with_index) { std::vector tags; milvus::engine::ResultIds result_ids; milvus::engine::ResultDistances result_distances; - stat = db_->Query(dummy_context_, table_info.table_id_, tags, topk, {{"nprobe", nprobe}}, search, result_ids, result_distances); + stat = db_->Query(dummy_context_, table_info.table_id_, tags, topk, {{"nprobe", nprobe}}, search, result_ids, + result_distances); ASSERT_NE(result_ids[0], pair.first); // ASSERT_LT(result_distances[0], 1e-4); ASSERT_GT(result_distances[0], 1); @@ -406,7 +410,8 @@ TEST_F(DeleteTest, delete_multiple_times_with_index) { std::vector tags; milvus::engine::ResultIds result_ids; milvus::engine::ResultDistances result_distances; - stat = db_->Query(dummy_context_, table_info.table_id_, tags, topk, {{"nprobe", nprobe}}, search, result_ids, result_distances); + stat = db_->Query(dummy_context_, table_info.table_id_, tags, topk, {{"nprobe", nprobe}}, search, result_ids, + result_distances); ASSERT_TRUE(stat.ok()); ASSERT_NE(result_ids[0], pair.first); // ASSERT_LT(result_distances[0], 1e-4); @@ -523,8 +528,8 @@ TEST_F(DeleteTest, delete_add_create_index) { result_ids.clear(); result_distances.clear(); - stat = db_->QueryByID(dummy_context_, table_info.table_id_, tags, topk, json_params, ids_to_delete.front(), result_ids, - result_distances); + stat = db_->QueryByID(dummy_context_, table_info.table_id_, tags, topk, json_params, ids_to_delete.front(), + result_ids, result_distances); ASSERT_EQ(result_ids[0], -1); ASSERT_EQ(result_distances[0], std::numeric_limits::max()); } @@ -594,8 +599,8 @@ TEST_F(DeleteTest, delete_add_auto_flush) { result_ids.clear(); result_distances.clear(); - stat = db_->QueryByID(dummy_context_, table_info.table_id_, tags, topk, {{"nprobe", nprobe}}, ids_to_delete.front(), result_ids, - result_distances); + stat = db_->QueryByID(dummy_context_, table_info.table_id_, tags, topk, {{"nprobe", nprobe}}, ids_to_delete.front(), + result_ids, result_distances); ASSERT_EQ(result_ids[0], -1); ASSERT_EQ(result_distances[0], std::numeric_limits::max()); } @@ -646,8 +651,8 @@ TEST_F(CompactTest, compact_basic) { milvus::engine::VectorsData qb = xb; for (auto& id : ids_to_delete) { - stat = - db_->QueryByID(dummy_context_, table_info.table_id_, tags, topk, json_params, id, result_ids, result_distances); + stat = db_->QueryByID(dummy_context_, table_info.table_id_, tags, topk, json_params, id, result_ids, + result_distances); ASSERT_EQ(result_ids[0], -1); ASSERT_EQ(result_distances[0], std::numeric_limits::max()); } @@ -736,8 +741,8 @@ TEST_F(CompactTest, compact_with_index) { std::vector tags; milvus::engine::ResultIds result_ids; milvus::engine::ResultDistances result_distances; - stat = - db_->Query(dummy_context_, table_info.table_id_, tags, topk, json_params, search, result_ids, result_distances); + stat = db_->Query(dummy_context_, table_info.table_id_, tags, topk, json_params, search, result_ids, + result_distances); ASSERT_NE(result_ids[0], pair.first); // ASSERT_LT(result_distances[0], 1e-4); ASSERT_GT(result_distances[0], 1); diff --git a/core/unittest/db/test_search_by_id.cpp b/core/unittest/db/test_search_by_id.cpp index 5c1ad7993f..568626296b 100644 --- a/core/unittest/db/test_search_by_id.cpp +++ b/core/unittest/db/test_search_by_id.cpp @@ -108,7 +108,8 @@ TEST_F(SearchByIdTest, basic) { milvus::engine::ResultIds result_ids; milvus::engine::ResultDistances result_distances; - stat = db_->QueryByID(dummy_context_, table_info.table_id_, tags, topk, json_params, i, result_ids, result_distances); + stat = db_->QueryByID(dummy_context_, table_info.table_id_, tags, topk, json_params, i, result_ids, + result_distances); ASSERT_EQ(result_ids[0], i); ASSERT_LT(result_distances[0], 1e-4); } @@ -165,7 +166,8 @@ TEST_F(SearchByIdTest, with_index) { milvus::engine::ResultIds result_ids; milvus::engine::ResultDistances result_distances; - stat = db_->QueryByID(dummy_context_, table_info.table_id_, tags, topk, json_params, i, result_ids, result_distances); + stat = db_->QueryByID(dummy_context_, table_info.table_id_, tags, topk, json_params, i, result_ids, + result_distances); ASSERT_EQ(result_ids[0], i); ASSERT_LT(result_distances[0], 1e-3); } @@ -225,7 +227,8 @@ TEST_F(SearchByIdTest, with_delete) { milvus::engine::ResultIds result_ids; milvus::engine::ResultDistances result_distances; - stat = db_->QueryByID(dummy_context_, table_info.table_id_, tags, topk, json_params, i, result_ids, result_distances); + stat = db_->QueryByID(dummy_context_, table_info.table_id_, tags, topk, json_params, i, result_ids, + result_distances); ASSERT_EQ(result_ids[0], -1); ASSERT_EQ(result_distances[0], std::numeric_limits::max()); } @@ -280,8 +283,8 @@ TEST_F(GetVectorByIdTest, basic) { stat = db_->GetVectorByID(table_info.table_id_, id, vector); ASSERT_TRUE(stat.ok()); - stat = - db_->Query(dummy_context_, table_info.table_id_, tags, topk, json_params, vector, result_ids, result_distances); + stat = db_->Query(dummy_context_, table_info.table_id_, tags, topk, json_params, vector, result_ids, + result_distances); ASSERT_TRUE(stat.ok()); ASSERT_EQ(result_ids[0], id); ASSERT_LT(result_distances[0], 1e-4); @@ -343,8 +346,8 @@ TEST_F(GetVectorByIdTest, with_index) { stat = db_->GetVectorByID(table_info.table_id_, id, vector); ASSERT_TRUE(stat.ok()); - stat = - db_->Query(dummy_context_, table_info.table_id_, tags, topk, json_params, vector, result_ids, result_distances); + stat = db_->Query(dummy_context_, table_info.table_id_, tags, topk, json_params, vector, result_ids, + result_distances); ASSERT_EQ(result_ids[0], id); ASSERT_LT(result_distances[0], 1e-3); } @@ -483,8 +486,8 @@ TEST_F(SearchByIdTest, BINARY) { ASSERT_TRUE(stat.ok()); ASSERT_EQ(vector.vector_count_, 1); - stat = - db_->Query(dummy_context_, table_info.table_id_, tags, topk, json_params, vector, result_ids, result_distances); + stat = db_->Query(dummy_context_, table_info.table_id_, tags, topk, json_params, vector, result_ids, + result_distances); ASSERT_TRUE(stat.ok()); ASSERT_EQ(result_ids[0], id); ASSERT_LT(result_distances[0], 1e-4); @@ -493,8 +496,8 @@ TEST_F(SearchByIdTest, BINARY) { result_ids.clear(); result_distances.clear(); - stat = - db_->QueryByID(dummy_context_, table_info.table_id_, tags, topk, json_params, id, result_ids, result_distances); + stat = db_->QueryByID(dummy_context_, table_info.table_id_, tags, topk, json_params, id, result_ids, + result_distances); ASSERT_TRUE(stat.ok()); ASSERT_EQ(result_ids[0], id); ASSERT_LT(result_distances[0], 1e-4); From a267c9a6fc4bbecad926bc5adf74f5a467a1099c Mon Sep 17 00:00:00 2001 From: Zhiru Zhu Date: Mon, 9 Mar 2020 11:46:15 +0800 Subject: [PATCH 23/58] update Signed-off-by: Zhiru Zhu --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 69e40ea6fb..bd6d243309 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -45,6 +45,7 @@ Please mark all change in change log and use the issue from GitHub - \#1535 Degradation searching performance with metric_type: binary_idmap - \#1556 Index file not created after table and index created - \#1560 Search crashed with Super-high dimensional binary vector +- \#1574 Set all existing bitset in cache when applying deletes ## Feature - \#216 Add CLI to get server info From f69861c2fdf56b2dca6fef19bfa4abdae73c12e9 Mon Sep 17 00:00:00 2001 From: Zhiru Zhu Date: Mon, 9 Mar 2020 11:52:52 +0800 Subject: [PATCH 24/58] update Signed-off-by: Zhiru Zhu --- .../src/config/handler/GpuBuildResHandler.cpp | 61 ------------ core/src/config/handler/GpuBuildResHandler.h | 44 --------- .../config/handler/GpuResourcesHandler.cpp | 64 ------------ core/src/config/handler/GpuResourcesHandler.h | 50 ---------- .../config/handler/GpuSearchResHandler.cpp | 99 ------------------- core/src/config/handler/GpuSearchResHandler.h | 55 ----------- 6 files changed, 373 deletions(-) delete mode 100644 core/src/config/handler/GpuBuildResHandler.cpp delete mode 100644 core/src/config/handler/GpuBuildResHandler.h delete mode 100644 core/src/config/handler/GpuResourcesHandler.cpp delete mode 100644 core/src/config/handler/GpuResourcesHandler.h delete mode 100644 core/src/config/handler/GpuSearchResHandler.cpp delete mode 100644 core/src/config/handler/GpuSearchResHandler.h diff --git a/core/src/config/handler/GpuBuildResHandler.cpp b/core/src/config/handler/GpuBuildResHandler.cpp deleted file mode 100644 index 6a57812b85..0000000000 --- a/core/src/config/handler/GpuBuildResHandler.cpp +++ /dev/null @@ -1,61 +0,0 @@ -// Copyright (C) 2019-2020 Zilliz. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software distributed under the License -// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -// or implied. See the License for the specific language governing permissions and limitations under the License. -#ifdef MILVUS_GPU_VERSION -#include "config/handler/GpuBuildResHandler.h" - -#include -#include - -namespace milvus { -namespace server { - -GpuBuildResHandler::GpuBuildResHandler() { - server::Config& config = server::Config::GetInstance(); - config.GetGpuResourceConfigBuildIndexResources(build_gpus_); -} - -GpuBuildResHandler::~GpuBuildResHandler() { - server::Config& config = server::Config::GetInstance(); - config.CancelCallBack(server::CONFIG_GPU_RESOURCE, server::CONFIG_GPU_RESOURCE_BUILD_INDEX_RESOURCES, identity_); -} - -//////////////////////////////////////////////////////////////// -void -GpuBuildResHandler::OnGpuBuildResChanged(const std::vector& gpus) { - build_gpus_ = gpus; -} - -void -GpuBuildResHandler::AddGpuBuildResListener() { - server::Config& config = server::Config::GetInstance(); - server::ConfigCallBackF lambda = [this](const std::string& value) -> Status { - server::Config& config = server::Config::GetInstance(); - std::vector gpu_ids; - auto status = config.GetGpuResourceConfigSearchResources(gpu_ids); - if (status.ok()) { - OnGpuBuildResChanged(gpu_ids); - } - - return status; - }; - config.RegisterCallBack(server::CONFIG_GPU_RESOURCE, server::CONFIG_GPU_RESOURCE_BUILD_INDEX_RESOURCES, identity_, - lambda); -} - -void -GpuBuildResHandler::RemoveGpuBuildResListener() { - auto& config = server::Config::GetInstance(); - config.CancelCallBack(server::CONFIG_GPU_RESOURCE, server::CONFIG_GPU_RESOURCE_BUILD_INDEX_RESOURCES, identity_); -} - -} // namespace server -} // namespace milvus -#endif diff --git a/core/src/config/handler/GpuBuildResHandler.h b/core/src/config/handler/GpuBuildResHandler.h deleted file mode 100644 index 6047836bcb..0000000000 --- a/core/src/config/handler/GpuBuildResHandler.h +++ /dev/null @@ -1,44 +0,0 @@ -// Copyright (C) 2019-2020 Zilliz. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software distributed under the License -// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -// or implied. See the License for the specific language governing permissions and limitations under the License. -#ifdef MILVUS_GPU_VERSION -#pragma once - -#include - -#include "config/handler/GpuResourcesHandler.h" - -namespace milvus { -namespace server { - -class GpuBuildResHandler : virtual public GpuResourcesHandler { - public: - GpuBuildResHandler(); - - ~GpuBuildResHandler(); - - public: - virtual void - OnGpuBuildResChanged(const std::vector& gpus); - - protected: - void - AddGpuBuildResListener(); - - void - RemoveGpuBuildResListener(); - - protected: - std::vector build_gpus_; -}; - -} // namespace server -} // namespace milvus -#endif diff --git a/core/src/config/handler/GpuResourcesHandler.cpp b/core/src/config/handler/GpuResourcesHandler.cpp deleted file mode 100644 index 65659985d6..0000000000 --- a/core/src/config/handler/GpuResourcesHandler.cpp +++ /dev/null @@ -1,64 +0,0 @@ - -// Copyright (C) 2019-2020 Zilliz. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software distributed under the License -// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -// or implied. See the License for the specific language governing permissions and limitations under the License. -#ifdef MILVUS_GPU_VERSION -#include "config/handler/GpuResourcesHandler.h" - -namespace milvus { -namespace server { - -GpuResourcesHandler::GpuResourcesHandler() { - server::Config& config = server::Config::GetInstance(); - config.GetGpuResourceConfigEnable(gpu_enable_); -} - -GpuResourcesHandler::~GpuResourcesHandler() { - RemoveGpuEnableListener(); -} - -////////////////////////////////////////////////////////////// -void -GpuResourcesHandler::OnGpuEnableChanged(bool enable) { - gpu_enable_ = enable; -} - -void -GpuResourcesHandler::SetIdentity(const std::string& identity) { - server::Config& config = server::Config::GetInstance(); - config.GenUniqueIdentityID(identity, identity_); -} - -void -GpuResourcesHandler::AddGpuEnableListener() { - server::Config& config = server::Config::GetInstance(); - - server::ConfigCallBackF lambda = [this](const std::string& value) -> Status { - server::Config& config = server::Config::GetInstance(); - bool enable; - auto status = config.GetGpuResourceConfigEnable(enable); - if (status.ok()) { - OnGpuEnableChanged(enable); - } - - return status; - }; - config.RegisterCallBack(server::CONFIG_GPU_RESOURCE, server::CONFIG_GPU_RESOURCE_ENABLE, identity_, lambda); -} - -void -GpuResourcesHandler::RemoveGpuEnableListener() { - server::Config& config = server::Config::GetInstance(); - config.CancelCallBack(server::CONFIG_GPU_RESOURCE, server::CONFIG_GPU_RESOURCE_ENABLE, identity_); -} - -} // namespace server -} // namespace milvus -#endif diff --git a/core/src/config/handler/GpuResourcesHandler.h b/core/src/config/handler/GpuResourcesHandler.h deleted file mode 100644 index d9e5dc601e..0000000000 --- a/core/src/config/handler/GpuResourcesHandler.h +++ /dev/null @@ -1,50 +0,0 @@ -// Copyright (C) 2019-2020 Zilliz. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software distributed under the License -// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -// or implied. See the License for the specific language governing permissions and limitations under the License. -#ifdef MILVUS_GPU_VERSION -#pragma once - -#include -#include -#include - -#include "server/Config.h" - -namespace milvus { -namespace server { - -class GpuResourcesHandler { - public: - GpuResourcesHandler(); - - ~GpuResourcesHandler(); - - protected: - virtual void - OnGpuEnableChanged(bool enable); - - protected: - void - SetIdentity(const std::string& identity); - - void - AddGpuEnableListener(); - - void - RemoveGpuEnableListener(); - - protected: - bool gpu_enable_ = true; - std::string identity_; -}; - -} // namespace server -} // namespace milvus -#endif diff --git a/core/src/config/handler/GpuSearchResHandler.cpp b/core/src/config/handler/GpuSearchResHandler.cpp deleted file mode 100644 index 781e5aa527..0000000000 --- a/core/src/config/handler/GpuSearchResHandler.cpp +++ /dev/null @@ -1,99 +0,0 @@ -// Copyright (C) 2019-2020 Zilliz. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software distributed under the License -// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -// or implied. See the License for the specific language governing permissions and limitations under the License. -#ifdef MILVUS_GPU_VERSION -#include "config/handler/GpuSearchResHandler.h" - -#include -#include - -#include "server/Config.h" - -namespace milvus { -namespace server { - -GpuSearchResHandler::GpuSearchResHandler() { - server::Config& config = server::Config::GetInstance(); - - Status s = config.GetEngineConfigGpuSearchThreshold(threshold_); - if (!s.ok()) { - threshold_ = std::numeric_limits::max(); - } - - config.GetGpuResourceConfigSearchResources(search_gpus_); -} - -GpuSearchResHandler::~GpuSearchResHandler() { - RemoveGpuSearchThresholdListener(); - RemoveGpuSearchResListener(); -} - -//////////////////////////////////////////////////////////////////////// -void -GpuSearchResHandler::OnGpuSearchThresholdChanged(int64_t threshold) { - threshold_ = threshold; -} - -void -GpuSearchResHandler::OnGpuSearchResChanged(const std::vector& gpus) { - search_gpus_ = gpus; -} - -void -GpuSearchResHandler::AddGpuSearchThresholdListener() { - server::Config& config = server::Config::GetInstance(); - - server::ConfigCallBackF lambda_gpu_threshold = [this](const std::string& value) -> Status { - server::Config& config = server::Config::GetInstance(); - int64_t threshold; - auto status = config.GetEngineConfigGpuSearchThreshold(threshold); - if (status.ok()) { - OnGpuSearchThresholdChanged(threshold); - } - - return status; - }; - config.RegisterCallBack(server::CONFIG_ENGINE, server::CONFIG_ENGINE_GPU_SEARCH_THRESHOLD, identity_, - lambda_gpu_threshold); -} - -void -GpuSearchResHandler::AddGpuSearchResListener() { - server::Config& config = server::Config::GetInstance(); - - server::ConfigCallBackF lambda_gpu_search_res = [this](const std::string& value) -> Status { - server::Config& config = server::Config::GetInstance(); - std::vector gpu_ids; - auto status = config.GetGpuResourceConfigSearchResources(gpu_ids); - if (status.ok()) { - OnGpuSearchResChanged(gpu_ids); - } - - return status; - }; - config.RegisterCallBack(server::CONFIG_GPU_RESOURCE, server::CONFIG_GPU_RESOURCE_SEARCH_RESOURCES, identity_, - lambda_gpu_search_res); -} - -void -GpuSearchResHandler::RemoveGpuSearchThresholdListener() { - server::Config& config = server::Config::GetInstance(); - config.CancelCallBack(server::CONFIG_ENGINE, server::CONFIG_ENGINE_GPU_SEARCH_THRESHOLD, identity_); -} - -void -GpuSearchResHandler::RemoveGpuSearchResListener() { - auto& config = server::Config::GetInstance(); - config.CancelCallBack(server::CONFIG_GPU_RESOURCE, server::CONFIG_GPU_RESOURCE_SEARCH_RESOURCES, identity_); -} - -} // namespace server -} // namespace milvus -#endif diff --git a/core/src/config/handler/GpuSearchResHandler.h b/core/src/config/handler/GpuSearchResHandler.h deleted file mode 100644 index 390db85bb0..0000000000 --- a/core/src/config/handler/GpuSearchResHandler.h +++ /dev/null @@ -1,55 +0,0 @@ -// Copyright (C) 2019-2020 Zilliz. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software distributed under the License -// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -// or implied. See the License for the specific language governing permissions and limitations under the License. -#ifdef MILVUS_GPU_VERSION -#pragma once - -#include -#include - -#include "config/handler/GpuResourcesHandler.h" - -namespace milvus { -namespace server { - -class GpuSearchResHandler : virtual public GpuResourcesHandler { - public: - GpuSearchResHandler(); - - ~GpuSearchResHandler(); - - public: - virtual void - OnGpuSearchThresholdChanged(int64_t threshold); - - virtual void - OnGpuSearchResChanged(const std::vector& gpus); - - protected: - void - AddGpuSearchThresholdListener(); - - void - AddGpuSearchResListener(); - - void - RemoveGpuSearchThresholdListener(); - - void - RemoveGpuSearchResListener(); - - protected: - int64_t threshold_ = std::numeric_limits::max(); - std::vector search_gpus_; -}; - -} // namespace server -} // namespace milvus -#endif From e18e0a11cca8697aa0cc887ee8289b055fe74b20 Mon Sep 17 00:00:00 2001 From: groot Date: Mon, 9 Mar 2020 11:59:57 +0800 Subject: [PATCH 25/58] #1571 Meta engine type become IDMAP after drop index for BINARY table Signed-off-by: groot --- core/src/db/meta/MySQLMetaImpl.cpp | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/core/src/db/meta/MySQLMetaImpl.cpp b/core/src/db/meta/MySQLMetaImpl.cpp index b1d28c2d34..bd0d4de96d 100644 --- a/core/src/db/meta/MySQLMetaImpl.cpp +++ b/core/src/db/meta/MySQLMetaImpl.cpp @@ -1326,8 +1326,12 @@ MySQLMetaImpl::DropTableIndex(const std::string& table_id) { } // set table index type to raw - dropTableIndexQuery << "UPDATE " << META_TABLES - << " SET engine_type = " << std::to_string(DEFAULT_ENGINE_TYPE) + dropTableIndexQuery << "UPDATE " << META_TABLES << " SET engine_type = " + << " (CASE" + << " WHEN metric_type in (" << (int32_t)MetricType::HAMMING << " ," + << (int32_t)MetricType::JACCARD << " ," << (int32_t)MetricType::TANIMOTO << ")" + << " THEN " << (int32_t)EngineType::FAISS_BIN_IDMAP << " ELSE " + << (int32_t)EngineType::FAISS_IDMAP << " END)" << " , index_params = '{}'" << " WHERE table_id = " << mysqlpp::quote << table_id << ";"; From e483ca06d13faf45fa471d8634b1dfb60c2c7ec6 Mon Sep 17 00:00:00 2001 From: Yhz Date: Mon, 9 Mar 2020 12:36:03 +0800 Subject: [PATCH 26/58] update README.md Signed-off-by: Yhz --- core/src/server/web_impl/README.md | 646 +++++++++++++++++++++-------- 1 file changed, 481 insertions(+), 165 deletions(-) diff --git a/core/src/server/web_impl/README.md b/core/src/server/web_impl/README.md index 67535ca2fa..efa12c8090 100644 --- a/core/src/server/web_impl/README.md +++ b/core/src/server/web_impl/README.md @@ -12,25 +12,28 @@ - [`/config/gpu_resources` (GET)](#configgpu_resources-get) - [`/config/gpu_resources` (PUT)](#configgpu_resources-put) - [`/config/gpu_resources` (OPTIONS)](#configgpu_resources-options) - - [`/tables` (GET)](#tables-get) - - [`/tables` (POST)](#tables-post) - - [`/tables` (OPTIONS)](#tables-options) - - [`/tables/{table_name}` (GET)](#tablestable_name-get) - - [`/tables/{table_name}` (DELETE)](#tablestable_name-delete) - - [`/tables/{table_name}` (OPTIONS)](#tablestable_name-options) - - [`/tables/{table_name}/indexes` (GET)](#tablestable_nameindexes-get) - - [`/tables/{table_name}/indexes` (POST)](#tablestable_nameindexes-post) - - [`/tables/{table_name}/indexes` (DELETE)](#tablestable_nameindexes-delete) - - [`/tables/{table_name}/indexes` (OPTIONS)](#tablestable_nameindexes-options) - - [`/tables/{table_name}/partitions` (GET)](#tablestable_namepartitions-get) - - [`/tables/{table_name}/partitions` (POST)](#tablestable_namepartitions-post) - - [`/tables/{table_name}/partitions` (OPTIONS)](#tablestable_namepartitions-options) - - [`/tables/{table_name}/partitions/{partition_tag}` (DELETE)](#tablestable_namepartitionspartition_tag-delete) - - [`/tables/{table_name}/partitions/{partition_tag}` (OPTIONS)](#tablestable_namepartitionspartition_tag-options) - - [`/tables/{table_name}/vectors` (PUT)](#tablestable_namevectors-put) - - [`/tables/{table_name}/vectors` (POST)](#tablestable_namevectors-post) - - [`/tables/{table_name}/vectors` (OPTIONS)](#tablestable_namevectors-options) + - [`/collections` (GET)](#collections-get) + - [`/collections` (POST)](#collections-post) + - [`/collections` (OPTIONS)](#collections-options) + - [`/collections/{collection_name}` (GET)](#collectionscollection_name-get) + - [`/collections/{collection_name}` (DELETE)](#collectionscollection_name-delete) + - [`/collections/{collection_name}` (OPTIONS)](#collectionscollection_name-options) + - [`/collections/{collection_name}/indexes` (GET)](#collectionscollection_nameindexes-get) + - [`/collections/{collection_name}/indexes` (POST)](#collectionscollection_nameindexes-post) + - [`/collections/{collection_name}/indexes` (DELETE)](#collectionscollection_nameindexes-delete) + - [`/collections/{collection_name}/indexes` (OPTIONS)](#collectionscollection_nameindexes-options) + - [`/collections/{collection_name}/partitions` (GET)](#collectionscollection_namepartitions-get) + - [`/collections/{collection_name}/partitions` (POST)](#collectionscollection_namepartitions-post) + - [`/collections/{collection_name}/partitions` (OPTIONS)](#collectionscollection_namepartitions-options) + - [`/collections/{collection_name}/partitions` (DELETE)](#collectionscollection_namepartitions-delete) + - [`/collections/{collection_name}/segments` (GET)](#collectionscollection_namesegments-get) + - [`/collections/{collection_name}/segments/{segment_name}/vectors` (GET)](#collectionscollection_namesegmentssegment_namevectors-get) + - [`/collections/{collection_name}/segments/{segment_name}/ids` (GET)](#collectionscollection_namesegmentssegment_nameids-get) + - [`/collections/{collection_name}/vectors` (PUT)](#collectionscollection_namevectors-put) + - [`/collections/{collection_name}/vectors` (POST)](#collectionscollection_namevectors-post) + - [`/collections/{collection_name}/vectors` (OPTIONS)](#collectionscollection_namevectors-options) - [`/system/{msg}` (GET)](#systemmsg-get) + - [`system/{op}` (PUT)](#systemop-put) - [Error Codes](#error-codes) @@ -334,15 +337,15 @@ Use this API for Cross-Origin Resource Sharing (CORS). $ curl -X OPTIONS "http://192.168.1.65:19121/config/gpu_resources" ``` -### `/tables` (GET) +### `/collections` (GET) -Gets all tables starting from `offset` and ends with `page_size`. +Gets all collections starting from `offset` and ends with `page_size`. #### Request | Request Component | Value | |-----------------|---| -| Name | `/tables` | +| Name | `/collections` | | Header | `accept: application/json` | | Body | N/A | | Method | GET | @@ -367,19 +370,19 @@ Gets all tables starting from `offset` and ends with `page_size`. ##### Request ```shell -$ curl -X GET "http://192.168.1.65:19121/tables?offset=0&page_size=1" -H "accept: application/json" +$ curl -X GET "http://192.168.1.65:19121/collections?offset=0&page_size=1" -H "accept: application/json" ``` ##### Response ```json -{"tables":[{"table_name":"test_table","dimension":1,"index_file_size":10,"metric_type":"L2","count":0,"index":"FLAT","nlist":16384}],"count":58} +{"collections":[{"collection_name":"test_collection","dimension":1,"index_file_size":10,"metric_type":"L2","count":0,"index":"FLAT","nlist":16384}],"count":58} ``` -### `/tables` (POST) +### `/collections` (POST) -Creates a table. +Creates a collection. #### Request @@ -389,7 +392,7 @@ Creates a table. Header 
accept: application/json
Body

 {
-  "table_name": string,
+  "collection_name": string,
   "dimension": integer($int64),
   "index_file_size": integer($int64),
   "metric_type": string
@@ -403,8 +406,8 @@ Creates a table.
 
 | Parameter  | Description  |  Required? |
 |-----------------|---|------|
-| `table_name`     |   The name of the table to create, which must be unique within its database.  | Yes   |
-| `dimension`  |  The dimension of the vectors that are to be inserted into the created table. |  Yes  |
+| `collection_name`     |   The name of the collection to create, which must be unique within its database.  | Yes   |
+| `dimension`  |  The dimension of the vectors that are to be inserted into the created collection. |  Yes  |
 | `index_file_size`    |  Threshold value that triggers index building for raw data files. The default is 1024.   |  No |
 | `metric_type`    |   The method vector distances are compared in Milvus. The default is L2. Currently supported metrics include `L2` (Euclidean distance), `IP` (Inner Product), `HAMMING` (Hamming distance), `JACCARD` (Jaccard distance), and `TANIMOTO` (Tanomoto distance).    |   No  |
 
@@ -420,7 +423,7 @@ Creates a table.
 ##### Request
 
 ```shell
-$ curl -X POST "http://192.168.1.65:19121/tables" -H "accept: application/json" -H "Content-Type: application/json" -d "{\"table_name\":\"test_table\",\"dimension\":1,\"index_file_size\":10,\"metric_type\":\"L2\"}"
+$ curl -X POST "http://192.168.1.65:19121/collections" -H "accept: application/json" -H "Content-Type: application/json" -d "{\"collection_name\":\"test_collection\",\"dimension\":1,\"index_file_size\":10,\"metric_type\":\"L2\"}"
 ```
 
 ##### Response
@@ -430,7 +433,7 @@ $ curl -X POST "http://192.168.1.65:19121/tables" -H "accept: application/json"
 {"message":"OK","code":0}
 ```
 
-### `/tables` (OPTIONS)
+### `/collections` (OPTIONS)
 
 Use this API for Cross-Origin Resource Sharing (CORS).
 
@@ -438,7 +441,7 @@ Use this API for Cross-Origin Resource Sharing (CORS).
 
 | Request Component     | Value  |
 |-----------------|---|
-| Name     | `/tables`  |
+| Name     | `/collections`  |
 | Header  | N/A  |
 | Body    |   N/A |
 | Method    |   OPTIONS |
@@ -449,18 +452,18 @@ Use this API for Cross-Origin Resource Sharing (CORS).
 ##### Request
 
 ```shell
-$ curl -X OPTIONS "http://192.168.1.65:19121/tables"
+$ curl -X OPTIONS "http://192.168.1.65:19121/collections"
 ```
 
-### `/tables/{table_name}` (GET)
+### `/collections/{collection_name}` (GET)
 
-Gets all information about a table by name.
+Gets all information about a collection by name.
 
 #### Request
 
 | Request Component     | Value  |
 |-----------------|---|
-| Name     | `/tables/{table_name}`  |
+| Name     | `/collections/{collection_name}`  |
 | Header  | `accept: application/json`  |
 | Body    |   N/A |
 | Method    |   GET |
@@ -470,8 +473,8 @@ Gets all information about a table by name.
 
 | Parameter  | Description  |  Required? |
 |-----------------|---|------|
-| `table_name`     | Name of the table.   | Yes   |
-| `info`     | Type of information to acquire. `info` must either be empty or `stat`. When `info` is empty, Milvus returns table name, dimension, index file size, metric type, offset, index type, and nlist of the table. When `info` is `stat`, Milvus returns the table offset, partition status, and segment status.  | No   |
+| `collection_name`     | Name of the collection.   | Yes   |
+| `info`     | Type of information to acquire. `info` must either be empty or `stat`. When `info` is empty, Milvus returns collection name, dimension, index file size, metric type, offset, index type, and nlist of the collection. When `info` is `stat`, Milvus returns the collection offset, partition status, and segment status.  | No   |
 
 
 #### Response
@@ -487,24 +490,24 @@ Gets all information about a table by name.
 ##### Request
 
 ```shell
-$ curl -X GET "http://192.168.1.65:19121/tables/test_table" -H "accept: application/json"
+$ curl -X GET "http://192.168.1.65:19121/collections/test_collection" -H "accept: application/json"
 ```
 
 ##### Response
 
 ```json
-{"table_name":"test_table","dimension":1,"index_file_size":10,"metric_type":"L2","count":0,"index":"FLAT","nlist":16384}
+{"collection_name":"test_collection","dimension":1,"index_file_size":10,"metric_type":"L2","count":0,"index":"FLAT","nlist":16384}
 ```
 
-### `/tables/{table_name}` (DELETE)
+### `/collections/{collection_name}` (DELETE)
 
-Drops a table by name.
+Drops a collection by name.
 
 #### Request
 
 | Request Component     | Value  |
 |-----------------|-----|
-| Name     | `/tables/{table_name}`  |
+| Name     | `/collections/{collection_name}`  |
 | Header  | `accept: application/json`  |
 | Body    |   N/A |
 | Method    |   DELETE |
@@ -513,7 +516,7 @@ Drops a table by name.
 
 | Parameter  | Description  |  Required? |
 |-----------------|---|------|
-| `table_name`     | Name of the table.   | Yes   |
+| `collection_name`     | Name of the collection.   | Yes   |
 
 #### Response
 
@@ -529,12 +532,12 @@ Drops a table by name.
 
 
 ```shell
-$ curl -X DELETE "http://192.168.1.65:19121/tables/test_table" -H "accept: application/json"
+$ curl -X DELETE "http://192.168.1.65:19121/collections/test_collection" -H "accept: application/json"
 ```
 
 If the deletion is successful, no message will be returned.
 
-### `/tables/{table_name}` (OPTIONS)
+### `/collections/{collection_name}` (OPTIONS)
 
 Use this API for Cross-Origin Resource Sharing (CORS).
 
@@ -542,7 +545,7 @@ Use this API for Cross-Origin Resource Sharing (CORS).
 
 | Request Component     | Value  |
 |-----------------|-----|
-| Name     | `/tables/{table_name}`  |
+| Name     | `/collections/{collection_name}`  |
 | Header  | N/A  |
 | Body    |   N/A |
 | Method    |   OPTIONS |
@@ -551,7 +554,7 @@ Use this API for Cross-Origin Resource Sharing (CORS).
 
 | Parameter  | Description  |  Required? |
 |-----------------|---|------|
-| `table_name`     | Name of the table.   | Yes   |
+| `collection_name`     | Name of the collection.   | Yes   |
 
 
 #### Example
@@ -559,18 +562,18 @@ Use this API for Cross-Origin Resource Sharing (CORS).
 ##### Request
 
 ```shell
-$ curl -X OPTIONS "http://192.168.1.65:19121/tables/test_table"
+$ curl -X OPTIONS "http://192.168.1.65:19121/collections/test_collection"
 ```
 
-### `/tables/{table_name}/indexes` (GET)
+### `/collections/{collection_name}/indexes` (GET)
 
-Gets the index type and nlist of a table.
+Gets the index type and nlist of a collection.
 
 #### Request
 
 | Request Component     | Value  |
 |-----------------|-----|
-| Name     | `/tables/{table_name}/indexes`  |
+| Name     | `/collections/{collection_name}/indexes`  |
 | Header  | `accept: application/json`  |
 | Body    |   N/A |
 | Method    |   GET |
@@ -579,7 +582,7 @@ Gets the index type and nlist of a table.
 
 | Parameter  | Description  |  Required? |
 |-----------------|---|------|
-| `table_name`     | Name of the table.   | Yes   |
+| `collection_name`     | Name of the collection.   | Yes   |
 
 #### Response
 
@@ -594,7 +597,7 @@ Gets the index type and nlist of a table.
 ##### Request
 
 ```shell
-$ curl -X GET "http://192.168.1.65:19121/tables/test_table/indexes" -H "accept: application/json"
+$ curl -X GET "http://192.168.1.65:19121/collections/test_collection/indexes" -H "accept: application/json"
 ```
 
 ##### Response
@@ -604,9 +607,9 @@ $ curl -X GET "http://192.168.1.65:19121/tables/test_table/indexes" -H "accept:
 {"index_type":"FLAT","nlist":16384}
 ```
 
-### `/tables/{table_name}/indexes` (POST)
+### `/collections/{collection_name}/indexes` (POST)
 
-Updates the index type and nlist of a table.
+Updates the index type and nlist of a collection.
 
 #### Request
 
@@ -628,14 +631,14 @@ Updates the index type and nlist of a table.
 
 | Parameter  | Description  |  Required? |
 |-----------------|---|------|
-| `index_type`     | The type of indexing method to query the table. Please refer to [Index Types](https://www.milvus.io/docs/reference/index.md) for detailed introduction of supported indexes. The default is "FLAT".  | No   |
+| `index_type`     | The type of indexing method to query the collection. Please refer to [Index Types](https://www.milvus.io/docs/reference/index.md) for detailed introduction of supported indexes. The default is "FLAT".  | No   |
 | `nlist`     |  Number of vector buckets in a file. The default is 16384.   | No   |
 
 ##### Query Parameters
 
 | Parameter  | Description  |  Required? |
 |-----------------|---|------|
-| `table_name`     | Name of the table.   | Yes   |
+| `collection_name`     | Name of the collection.   | Yes   |
 
 #### Response
 
@@ -658,7 +661,7 @@ Updates the index type and nlist of a table.
 ##### Request
 
 ```shell
-$ curl -X POST "http://192.168.1.65:19121/tables/test_table/indexes" -H "accept: application/json" -H "Content-Type: application/json" -d "{\"index_type\":\"FLAT\",\"nlist\":16384}"
+$ curl -X POST "http://192.168.1.65:19121/collections/test_collection/indexes" -H "accept: application/json" -H "Content-Type: application/json" -d "{\"index_type\":\"FLAT\",\"nlist\":16384}"
 ```
 
 ##### Response
@@ -667,15 +670,15 @@ $ curl -X POST "http://192.168.1.65:19121/tables/test_table/indexes" -H "accept:
 {"message":"OK","code":0}
 ```
 
-### `/tables/{table_name}/indexes` (DELETE)
+### `/collections/{collection_name}/indexes` (DELETE)
 
-Drops an index for a table.
+Drops an index for a collection.
 
 #### Request
 
 | Request Component     | Value  |
 |-----------------|---|
-| Name     | `/tables/{table_name}/indexes`  |
+| Name     | `/collections/{collection_name}/indexes`  |
 | Header  | `accept: application/json`  |
 | Body    |   N/A |
 | Method    |   DELETE |
@@ -684,7 +687,7 @@ Drops an index for a table.
 
 | Parameter  | Description  |  Required? |
 |-----------------|---|------|
-| `table_name`     | Name of the table.   | Yes   |
+| `collection_name`     | Name of the collection.   | Yes   |
 
 #### Response
 
@@ -700,13 +703,13 @@ Drops an index for a table.
 ##### Request
 
 ```shell
-$ curl -X DELETE "http://192.168.1.65:19121/tables/test_table/indexes" -H "accept: application/json"
+$ curl -X DELETE "http://192.168.1.65:19121/collections/test_collection/indexes" -H "accept: application/json"
 ```
 
 If the deletion is successful, no message will be returned.
 
 
-### `/tables/{table_name}/indexes` (OPTIONS)
+### `/collections/{collection_name}/indexes` (OPTIONS)
 
 Use this API for Cross-Origin Resource Sharing (CORS).
 
@@ -714,7 +717,7 @@ Use this API for Cross-Origin Resource Sharing (CORS).
 
 | Request Component     | Value  |
 |-----------------|---|
-| Name     | `/tables/{table_name}/indexes`  |
+| Name     | `/collections/{collection_name}/indexes`  |
 | Header  | N/A  |
 | Body    |   N/A |
 | Method    |   OPTIONS |
@@ -723,25 +726,25 @@ Use this API for Cross-Origin Resource Sharing (CORS).
 
 | Parameter  | Description  |  Required? |
 |-----------------|---|------|
-| `table_name`     | Name of the table.   | Yes   |
+| `collection_name`     | Name of the collection.   | Yes   |
 
 #### Example
 
 ##### Request
 
 ```shell
-$ curl -X OPTIONS "http://192.168.1.65:19121/tables/test_table/indexes"
+$ curl -X OPTIONS "http://192.168.1.65:19121/collections/test_collection/indexes"
 ```
 
-### `/tables/{table_name}/partitions` (GET)
+### `/collections/{collection_name}/partitions` (GET)
 
-Gets all partitions in a table starting from `offset` and ends with `page_size`.
+Gets all partitions in a collection starting from `offset` and ends with `page_size`.
 
 #### Request
 
 | Request Component     | Value  |
 |-----------------|-----------|
-| Name     | `/tables/{table_name}/partitions`  |
+| Name     | `/collections/{collection_name}/partitions`  |
 | Header  | `accept: application/json`  |
 | Body    |   N/A |
 | Method    |   GET |
@@ -750,7 +753,7 @@ Gets all partitions in a table starting from `offset` and ends with `page_size`.
 
 | Parameter  | Description  |  Required? |
 |-----------------|---|------|
-| `table_name`              |        Name of the table.                  |   Yes     |
+| `collection_name`              |        Name of the collection.                  |   Yes     |
 | `offset`     |  Row offset from which the data page starts. The default is 0.    | No   |
 | `page_size`  |  Size of the data page. The default is 10.   |  No  |
 
@@ -767,7 +770,7 @@ Gets all partitions in a table starting from `offset` and ends with `page_size`.
 ##### Request
 
 ```shell
-$ curl -X GET "http://192.168.1.65:19121/tables/test_table/partitions?offset=0&page_size=3" -H "accept: application/json"
+$ curl -X GET "http://192.168.1.65:19121/collections/test_collection/partitions?offset=0&page_size=3" -H "accept: application/json"
 ```
 
 ##### Response
@@ -776,15 +779,15 @@ $ curl -X GET "http://192.168.1.65:19121/tables/test_table/partitions?offset=0&p
 {"partitions":[{"partition_name":"partition_1","partition_tag":"test_tag"},{"partition_name":"partition_2","partition_tag":"test_2"},{"partition_name":"partition_3","partition_tag":"test_3"}]}
 ```
 
-### `/tables/{table_name}/partitions` (POST)
+### `/collections/{collection_name}/partitions` (POST)
 
-Creates a partition in a table.
+Creates a partition in a collection.
 
 #### Request
 
 | Request Component     | Value  |
 |-----------------|-----------|
-| Name     | `/tables/{table_name}/partitions`  |
+| Name     | `/collections/{collection_name}/partitions`  |
 | Header  | `accept: application/json`  |
 | Body    |   N/A |
 | Method    |   POST |
@@ -802,7 +805,7 @@ Creates a partition in a table.
 ##### Request
 
 ```shell
-$ curl -X POST "http://192.168.1.65:19121/tables/test_table/partitions" -H "accept: application/json" -H "Content-Type: application/json" -d "{\"partition_name\": \"partition_1\",\"partition_tag\": \"test\"}"
+$ curl -X POST "http://192.168.1.65:19121/collections/test_collection/partitions" -H "accept: application/json" -H "Content-Type: application/json" -d "{\"partition_name\": \"partition_1\",\"partition_tag\": \"test\"}"
 ```
 
 ##### Response
@@ -811,7 +814,7 @@ $ curl -X POST "http://192.168.1.65:19121/tables/test_table/partitions" -H "acce
 {"message":"OK","code":0}
 ```
 
-### `/tables/{table_name}/partitions` (OPTIONS)
+### `/collections/{collection_name}/partitions` (OPTIONS)
 
 Use this API for Cross-Origin Resource Sharing (CORS).
 
@@ -819,7 +822,7 @@ Use this API for Cross-Origin Resource Sharing (CORS).
 
 | Request Component     | Value  |
 |-----------------|---|
-| Name     | `/tables/{table_name}/partitions`  |
+| Name     | `/collections/{collection_name}/partitions`  |
 | Header  | N/A  |
 | Body    |   N/A |
 | Method    |   OPTIONS |
@@ -828,7 +831,7 @@ Use this API for Cross-Origin Resource Sharing (CORS).
 
 | Parameter  | Description  |  Required? |
 |-----------------|---|------|
-| `table_name`              |        Name of the table.                  |   Yes     |
+| `collection_name`              |        Name of the collection.                  |   Yes     |
 
 
 #### Example
@@ -836,27 +839,33 @@ Use this API for Cross-Origin Resource Sharing (CORS).
 ##### Request
 
 ```shell
-$ curl -X OPTIONS "http://192.168.1.65:19121/tables/test_table/partitions"
+$ curl -X OPTIONS "http://192.168.1.65:19121/collections/test_collection/partitions"
 ```
 
-### `/tables/{table_name}/partitions/{partition_tag}` (DELETE)
+### `/collections/{collection_name}/partitions` (DELETE)
 
 Deletes a partition by tag.
 
 #### Request
 
-| Request Component     | Value  |
-|-----------------|-----------|
-| Name     | `/tables/{table_name}/partitions/{partition_tag}`  |
-| Header  | `accept: application/json`  |
-| Body    |   N/A |
-| Method    |   DELETE |
+
+
+
+
+
+
+
+
Request ComponentValue
 Name
/collections/{collection_name}/partitions
Header 
accept: application/json
 
Body

+{
+  "partition_tag": string
+}
+
 
MethodPOST

 
 ##### Query Parameters
 
 | Parameter  | Description  |  Required? |
 |-----------------|---|------|
-| `table_name`              |        Name of the table that contains the partition.               |   Yes     |
+| `collection_name`              |        Name of the collection that contains the partition.               |   Yes     |
 | `partition_tag` |    Tag of the partition to delete.      |   yes |
 
 #### Response
@@ -872,81 +881,32 @@ Deletes a partition by tag.
 ##### Request
 
 ```shell
-$ curl -X DELETE "http://192.168.1.65:19121/tables/test_table/partitions/tags_01" -H "accept: application/json"
+$ curl -X DELETE "http://192.168.1.65:19121/collections/test_collection/partitions/tags_01" -H "accept: application/json"
 ```
 
 The deletion is successful if no information is returned.
 
-### `/tables/{table_name}/partitions/{partition_tag}` (OPTIONS)
 
-Use this API for Cross-Origin Resource Sharing (CORS).
+### `/collections/{collection_name}/segments` (GET)
+
+Gets all segments in a collection starting from `offset` and ends with `page_size`.
 
 #### Request
 
 | Request Component     | Value  |
-|-----------------|---|
-| Name     | `/tables/{table_name}/partitions/{partition_tag}`  |
-| Header  | N/A  |
+|-----------------|-----------|
+| Name     | `/collections/{collection_name}/segments`  |
+| Header  | `accept: application/json`  |
 | Body    |   N/A |
-| Method    |   OPTIONS |
+| Method    |   GET |
 
 ##### Query Parameters
 
 | Parameter  | Description  |  Required? |
 |-----------------|---|------|
-| `table_name`              |        Name of the table.                  |   Yes     |
-| `partition_tag` |    Tag of the partition      |   yes |
-
-#### Example
-
-##### Request
-
-```shell
-$ curl -X OPTIONS "http://192.168.1.65:19121/tables/test_table/partitions/tag"
-```
-
-### `/tables/{table_name}/vectors` (PUT)
-
-Searches vectors in a table.
-
-#### Request
-
-
-
-
-
-
-
-
Request ComponentValue
 Name
/tables/{table_name}/vectors
Header 
accept: application/json
 
Body

-{
-  "topk": integer($int64),
-  "nprobe": integer($int64),
-  "tags": [string],
-  "file_ids": [string],
-  "records": [[number($float)]],
-  "records_bin": [[number($uint64)]]
-}
-
 
MethodPUT

-
-##### Body Parameters
-
-| Parameter  | Description  |  Required? |
-|-----------------|---|------|
-| `topk`     |  The top k most similar results of each query vector.   | Yes   |
-| `nprobe`  |  Number of queried vector buckets. |  Yes  |
-| `tags`    |  Tags of partitions that you need to search. You do not have to specify this value if the table is not partitioned or you wish to search the whole table.   |  No |
-| `file_ids`    |  IDs of the vector files. You do not have to specify this value if you do not use Milvus in distributed scenarios. Also, if you assign a value to `file_ids`, the value of `tags` is ignored.    |   No  |
-| `records`  |  Numeric vectors to insert to the table.  |  Yes  |
-| `records_bin` | Binary vectors to insert to the table. |    Yes   |
-
-> Note: Select `records` or `records_bin` depending on the metric used by the table. If the table uses `L2` or `IP`, you must use `records`. If the table uses `HAMMING`, `JACCARD`, or `TANIMOTO`, you must use `records_bin`.
-
-
-##### Query Parameters
-
-| Parameter  | Description  |  Required? |
-|-----------------|---|------|
-| `table_name` |  Name of the table.      |   Yes     |
+| `collection_name`              |        Name of the collection.                  |   Yes     |
+| `offset`     |  Row offset from which the data page starts. The default is 0.    | No   |
+| `page_size`  |  Size of the data page. The default is 10.   |  No  |
 
 #### Response
 
@@ -961,7 +921,189 @@ Searches vectors in a table.
 ##### Request
 
 ```shell
-$ curl -X PUT "http://192.168.1.65:19121/tables/test_table/vectors" -H "accept: application/json" -H "Content-Type: application/json" -d "{\"topk\":2,\"nprobe\":16,\"records\":[[0.1]]}"
+$ curl -X GET "http://192.168.1.65:19121/collections/test_collection/segments?offset=0&page_size=3" -H "accept: application/json"
+```
+
+##### Response
+
+```json
+{
+   "code":0,
+   "message":"OK",
+   "count":2,
+   "segments":[
+      {
+         "count":10000,
+         "index":"IVFFLAT",
+         "partition_tag":"_default",
+         "segment_name":"1583727470444700000",
+         "size":5284922
+      }
+   ]
+}
+```
+
+### `/collections/{collection_name}/segments/{segment_name}/vectors` (GET)
+
+Gets all vectors of segment in a collection starting from `offset` and ends with `page_size`.
+
+#### Request
+
+| Request Component     | Value  |
+|-----------------|-----------|
+| Name     | `/collections/{collection_name}/segments`  |
+| Header  | `accept: application/json`  |
+| Body    |   N/A |
+| Method    |   GET |
+
+##### Query Parameters
+
+| Parameter  | Description  |  Required? |
+|-----------------|---|------|
+| `collection_name`              |        Name of the collection.                  |   Yes     |
+| `segment_name`              |        Name of the segment.                  |   Yes     |
+| `offset`     |  Row offset from which the data page starts. The default is 0.    | No   |
+| `page_size`  |  Size of the data page. The default is 10.   |  No  |
+
+#### Response
+
+| Status code    | Description |
+|-----------------|---|
+| 200     | The request is successful.|
+| 400     | The request is incorrect. Refer to the error message for details. |
+| 404     | The required resource does not exist. |
+
+#### Example
+
+##### Request
+
+```shell
+$ curl -X GET "http://192.168.1.65:19121/collections/test_collection/segments/1583727470444700000/vectors?offset=0&page_size=1" -H "accept: application/json"
+```
+
+##### Response
+
+```json
+{
+   "code":0,
+   "message":"OK",
+   "count":2,
+   "vectors": [
+      {
+         "vector": [], 
+         "id": ""
+      }
+   ]
+}
+```
+
+### `/collections/{collection_name}/segments/{segment_name}/ids` (GET)
+
+Gets all vector ids of segment in a collection starting from `offset` and ends with `page_size`.
+
+#### Request
+
+| Request Component     | Value  |
+|-----------------|-----------|
+| Name     | `/collections/{collection_name}/segments`  |
+| Header  | `accept: application/json`  |
+| Body    |   N/A |
+| Method    |   GET |
+
+##### Query Parameters
+
+| Parameter  | Description  |  Required? |
+|-----------------|---|------|
+| `collection_name`              |        Name of the collection.                  |   Yes     |
+| `segment_name`              |        Name of the segment.                  |   Yes     |
+| `offset`     |  Row offset from which the data page starts. The default is 0.    | No   |
+| `page_size`  |  Size of the data page. The default is 10.   |  No  |
+
+#### Response
+
+| Status code    | Description |
+|-----------------|---|
+| 200     | The request is successful.|
+| 400     | The request is incorrect. Refer to the error message for details. |
+| 404     | The required resource does not exist. |
+
+#### Example
+
+##### Request
+
+```shell
+$ curl -X GET "http://192.168.1.65:19121/collections/test_collection/segments/1583727470444700000/ids?offset=0&page_size=1" -H "accept: application/json"
+```
+
+##### Response
+
+```json
+{
+   "ids": [],
+   "count": 0
+}
+```
+
+
+### `/collections/{collection_name}/vectors` (PUT)
+
+1. Searches vectors in a collection.
+
+#### Request
+
+
+
+
+
+
+
+
Request ComponentValue
 Name
/tables/{table_name}/vectors
Header 
accept: application/json
 
Body

+{
+  "search": {
+      "topk": integer($int64),
+      "tags": [string],
+      "file_ids": [string],
+      "vectors": [[number($float/$uint8)]]
+      "params": {
+          "nprobe": 16
+      }
+  }
+}
+
 
MethodPUT

+
+##### Body Parameters
+
+| Parameter  | Description  |  Required? |
+|-----------------|---|------|
+| `topk`     |  The top k most similar results of each query vector.   | Yes   |
+| `tags`    |  Tags of partitions that you need to search. You do not have to specify this value if the collection is not partitioned or you wish to search the whole collection.   |  No |
+| `file_ids`    |  IDs of the vector files. You do not have to specify this value if you do not use Milvus in distributed scenarios. Also, if you assign a value to `file_ids`, the value of `tags` is ignored.    |   No  |
+| `records`  |  Numeric vectors to insert to the collection.  |  Yes  |
+| `records_bin` | Binary vectors to insert to the collection. |    Yes   |
+
+> Note: Type of items of vectors depends on the metric used by the collection. If the collection uses `L2` or `IP`, you must use `float`. If the collection uses `HAMMING`, `JACCARD`, or `TANIMOTO`, you must use `uint8`.
+
+
+##### Query Parameters
+
+| Parameter  | Description  |  Required? |
+|-----------------|---|------|
+| `collection_name` |  Name of the collection.      |   Yes     |
+
+#### Response
+
+| Status code    | Description |
+|-----------------|---|
+| 200     | The request is successful.|
+| 400     | The request is incorrect. Refer to the error message for details. |
+| 404     | The required resource does not exist. |
+
+#### Example
+
+##### Request
+
+```shell
+$ curl -X PUT "http://192.168.1.65:19121/collections/test_collection/vectors" -H "accept: application/json" -H "Content-Type: application/json" -d "{\"topk\":2,\"nprobe\":16,\"records\":[[0.1]]}"
 ```
 
 ##### Response
@@ -970,9 +1112,62 @@ $ curl -X PUT "http://192.168.1.65:19121/tables/test_table/vectors" -H "accept:
 {"num":1,"results":[[{"id":"1578989029645098000","distance":"0.000000"},{"id":"1578989029645098001","distance":"0.010000"}]]}
 ```
 
-### `/tables/{table_name}/vectors` (POST)
+2. Delete vectors
 
-Inserts vectors to a table.
+#### Request
+
+
+
+
+
+
+
+
Request ComponentValue
 Name
/tables/{table_name}/vectors
Header 
accept: application/json
 
Body

+{
+  "delete": {
+     "ids": [$string]
+  }
+}
+
 
MethodPUT

+
+##### Body Parameters
+
+| Parameter  | Description  |  Required? |
+|-----------------|---|------|
+| ids | IDs of vectors. | Yes |
+
+
+##### Query Parameters
+
+| Parameter  | Description  |  Required? |
+|-----------------|---|------|
+| `collection_name` |  Name of the collection.      |   Yes     |
+
+#### Response
+
+| Status code    | Description |
+|-----------------|---|
+| 200     | The request is successful.|
+| 400     | The request is incorrect. Refer to the error message for details. |
+| 404     | The required resource does not exist. |
+
+#### Example
+
+##### Request
+
+```shell
+$ curl -X PUT "http://192.168.1.65:19121/collections/test_collection/vectors" -H "accept: application/json" -H "Content-Type: application/json" -d "{"delete": {"ids": ["1578989029645098000"]}}"
+```
+
+##### Response
+
+```json
+{"code": 0, "message": "success"}
+```
+
+### `/collections/{collection_name}/vectors` (POST)
+
+Inserts vectors to a collection.
 
 > Note: It is recommended that you do not insert more than 1 million vectors per request.
 
@@ -985,8 +1180,7 @@ Inserts vectors to a table.
 Body

 {
   "tag": string,
-  "records": [[number($float)]],
-  “records_bin”:[[number($uint64)]]
+  "vectors": [[number($float/$uint8)]],
   "ids": [integer($int64)]
 }
 
 
@@ -998,17 +1192,16 @@ Inserts vectors to a table.
 | Parameter  | Description  |  Required? |
 |-----------------|---|------|
 | `tag`     |  Tag of the partition to insert vectors to.   | No   |
-| `records`  |  Numeric vectors to insert to the table.  |  Yes  |
-| `records_bin` | Binary vectors to insert to the table.  |    Yes    |
-| `ids`    |  IDs of the vectors to insert to the table. If you assign IDs to the vectors, you must provide IDs for all vectors in the table. If you do not specify this parameter, Milvus automatically assigns IDs to the vectors. |  No |
+| `vectors`  |  Vectors to insert to the collection.  |  Yes  |
+| `ids`    |  IDs of the vectors to insert to the collection. If you assign IDs to the vectors, you must provide IDs for all vectors in the collection. If you do not specify this parameter, Milvus automatically assigns IDs to the vectors. |  No |
 
-> Note: Select `records` or `records_bin` depending on the metric used by the table. If the table uses `L2` or `IP`, you must use `records`. If the table uses `HAMMING`, `JACCARD`, or `TANIMOTO`, you must use `records_bin`.
+> Note: Type of items of `vectors` depends on the metric used by the collection. If the collection uses `L2` or `IP`, you must use `float`. If the collection uses `HAMMING`, `JACCARD`, or `TANIMOTO`, you must use `uint8`.
 
 ##### Query Parameters
 
 | Parameter  | Description  |  Required? |
 |-----------------|---|------|
-| `table_name` |  Name of the table.      |   Yes     |
+| `collection_name` |  Name of the collection.      |   Yes     |
 
 #### Response
 
@@ -1023,7 +1216,7 @@ Inserts vectors to a table.
 ##### Request
 
 ```shell
-$ curl -X POST "http://192.168.1.65:19121/tables/test_table/vectors" -H "accept: application/json" -H "Content-Type: application/json" -d "{\"records\":[[0.1],[0.2],[0.3],[0.4]]}"
+$ curl -X POST "http://192.168.1.65:19121/collections/test_collection/vectors" -H "accept: application/json" -H "Content-Type: application/json" -d "{\"vectors\":[[0.1],[0.2],[0.3],[0.4]]}"
 ```
 
 ##### Response
@@ -1032,7 +1225,7 @@ $ curl -X POST "http://192.168.1.65:19121/tables/test_table/vectors" -H "accept:
 {"ids":["1578989029645098000","1578989029645098001","1578989029645098002","1578989029645098003"]}
 ```
 
-### `/tables/{table_name}/vectors` (OPTIONS)
+### `/collections/{collection_name}/vectors` (OPTIONS)
 
 Use this API for Cross-Origin Resource Sharing (CORS).
 
@@ -1040,7 +1233,7 @@ Use this API for Cross-Origin Resource Sharing (CORS).
 
 | Request Component     | Value  |
 |-----------------|---|
-| Name     | `/tables/{table_name}/vectors`  |
+| Name     | `/collections/{collection_name}/vectors`  |
 | Header  | N/A |
 | Body    |   N/A |
 | Method    |   OPTIONS |
@@ -1050,7 +1243,7 @@ Use this API for Cross-Origin Resource Sharing (CORS).
 ##### Request
 
 ```shell
-$ curl -X OPTIONS "http://192.168.1.65:19121/tables/test_table/vectors"
+$ curl -X OPTIONS "http://192.168.1.65:19121/collections/test_collection/vectors"
 ```
 
 ### `/system/{msg}` (GET)
@@ -1093,6 +1286,125 @@ $ curl -X GET "http://192.168.1.65:19121/system/version" -H "accept: application
 {"reply":"0.7.0"}
 ```
 
+### `system/{op}` (PUT)
+
+1. Flush collections
+
+#### Request
+
+
+
+
+
+
+
+
Request ComponentValue
 Name
/system/task
Header 
accept: application/json
 
Body

+{
+  "flush": {
+     "collection_names": [$string]
+  }
+}
+
 
MethodPUT

+
+#### Response
+
+| Status code    | Description |
+|-----------------|---|
+| 200     | The request is successful.|
+| 400     | The request is incorrect. Refer to the error message for details. |
+
+#### Example
+
+##### Request
+
+```shell
+$ curl -X PUT "http://192.168.1.65:19121/system/task" -H "accept: application/json" -d "{\"flush\": {\"collection_names\": [test_collection]}}"
+```
+
+##### Response
+
+```json
+{"code": 0, "message": "success"}
+```
+
+2. Compact collection
+
+#### Request
+
+
+
+
+
+
+
+
Request ComponentValue
 Name
/system/task
Header 
accept: application/json
 
Body

+{
+  "compact": {
+     "collection_name": $string
+  }
+}
+
 
MethodPUT

+
+#### Response
+
+| Status code    | Description |
+|-----------------|---|
+| 200     | The request is successful.|
+| 400     | The request is incorrect. Refer to the error message for details. |
+
+#### Example
+
+##### Request
+
+```shell
+$ curl -X PUT "http://192.168.1.65:19121/system/task" -H "accept: application/json" -d "{\"compact\": {\"collection_name\": test_collection}}"
+```
+
+##### Response
+
+```json
+{"code": 0, "message": "success"}
+```
+
+3. Load collection to memory
+
+#### Request
+
+
+
+
+
+
+
+
Request ComponentValue
 Name
/system/task
Header 
accept: application/json
 
Body

+{
+  "load": {
+     "collection_name": $string
+  }
+}
+
 
MethodPUT

+
+#### Response
+
+| Status code    | Description |
+|-----------------|---|
+| 200     | The request is successful.|
+| 400     | The request is incorrect. Refer to the error message for details. |
+
+#### Example
+
+##### Request
+
+```shell
+$ curl -X PUT "http://192.168.1.65:19121/system/task" -H "accept: application/json" -d "{\"load\": {\"collection_name\": test_collection}}"
+```
+
+##### Response
+
+```json
+{"code": 0, "message": "success"}
+```
+
 ## Error Codes
 
 The RESTful API returns error messages as JSON text. Each type of error message has a specific error code.
@@ -1125,6 +1437,10 @@ ILLEGAL_NLIST | 22 |
 ILLEGAL_METRIC_TYPE | 23 |
 OUT_OF_MEMORY | 24 |
 PATH_PARAM_LOSS | 31 |
-QUERY_PARAM_LOSS | 32 |
-BODY_FIELD_LOSS | 33 |
-ILLEGAL_QUERY_PARAM | 36 |
+UNKNOWN_PATH | 32 |
+QUERY_PARAM_LOSS | 33 |
+BODY_FIELD_LOSS | 34 |
+ILLEGAL_BODY | 35 |
+BODY_PARSE_FAIL | 36 |
+ILLEGAL_QUERY_PARAM | 37 |
+

From e9d32dc9de1b48c62b26b3ea3d630ab630f01c86 Mon Sep 17 00:00:00 2001
From: groot 
Date: Mon, 9 Mar 2020 13:33:57 +0800
Subject: [PATCH 27/58] typo

Signed-off-by: groot 
---
 CHANGELOG.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 6ffed248af..493b118bb2 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -45,7 +45,7 @@ Please mark all change in change log and use the issue from GitHub
 -   \#1535 Degradation searching performance with metric_type: binary_idmap
 -   \#1556 Index file not created after table and index created
 -   \#1560 Search crashed with Super-high dimensional binary vector
--   \#1571 Meta engine type become IDMAP after drop index for BINARY table
+-   \#1571 Meta engine type become IDMAP after dropping index for BINARY table
 
 ## Feature
 -   \#216 Add CLI to get server info

From f891a590b5b45eb088be4c36dfe73241bf377a3b Mon Sep 17 00:00:00 2001
From: "xiaojun.lin" 
Date: Mon, 9 Mar 2020 15:06:26 +0800
Subject: [PATCH 28/58] fix

Signed-off-by: xiaojun.lin 
---
 CHANGELOG.md                           | 1 +
 core/src/scheduler/task/SearchTask.cpp | 3 ++-
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index f86f15976a..2ba1428637 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -46,6 +46,7 @@ Please mark all change in change log and use the issue from GitHub
 -   \#1549 Fix server/wal config setting bug
 -   \#1556 Index file not created after table and index created
 -   \#1560 Search crashed with Super-high dimensional binary vector
+-   \#1564 Too low recall for glove-200-angular, ivf_pq index
 
 ## Feature
 -   \#216 Add CLI to get server info
diff --git a/core/src/scheduler/task/SearchTask.cpp b/core/src/scheduler/task/SearchTask.cpp
index 0b32f35ab5..cf96dc504a 100644
--- a/core/src/scheduler/task/SearchTask.cpp
+++ b/core/src/scheduler/task/SearchTask.cpp
@@ -101,7 +101,8 @@ XSearchTask::XSearchTask(const std::shared_ptr& context, TableF
     if (file_) {
         // distance -- value 0 means two vectors equal, ascending reduce, L2/HAMMING/JACCARD/TONIMOTO ...
         // similarity -- infinity value means two vectors equal, descending reduce, IP
-        if (file_->metric_type_ == static_cast(MetricType::IP)) {
+        if (file_->metric_type_ == static_cast(MetricType::IP) &&
+            file_->engine_type_ != static_cast(EngineType::FAISS_PQ)) {
             ascending_reduce = false;
         }
 

From b8b86d77c46ed1be2541c68c30da4b6498b381e8 Mon Sep 17 00:00:00 2001
From: groot 
Date: Mon, 9 Mar 2020 15:08:54 +0800
Subject: [PATCH 29/58] typo

Signed-off-by: groot 
---
 CHANGELOG.md | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 493b118bb2..46a7dcd0cb 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -12,16 +12,16 @@ Please mark all change in change log and use the issue from GitHub
 -   \#831 Judge branch error in CommonUtil.cpp
 -   \#977 Server crash when create tables concurrently
 -   \#990 Check gpu resources setting when assign repeated value
--   \#995 table count set to 0 if no tables found
+-   \#995 Table count set to 0 if no tables found
 -   \#1010 Improve error message when offset or page_size is equal 0
--   \#1022 check if partition name is legal
--   \#1028 check if table exists when show partitions
--   \#1029 check if table exists when try to delete partition
--   \#1066 optimize http insert and search speed
+-   \#1022 Check if partition name is legal
+-   \#1028 Check if table exists when show partitions
+-   \#1029 Check if table exists when try to delete partition
+-   \#1066 Optimize http insert and search speed
 -   \#1067 Add binary vectors support in http server
 -   \#1075 Improve error message when page size or offset is illegal
 -   \#1082 Check page_size or offset value to avoid float
--   \#1115 http server support load table into memory
+-   \#1115 Http server support load table into memory
 -   \#1152 Error log output continuously after server start
 -   \#1211 Server down caused by searching with index_type: HNSW
 -   \#1240 Update license declaration

From 281d8ec53f03acb8db5ef32b37740227a629cc97 Mon Sep 17 00:00:00 2001
From: Zhiru Zhu 
Date: Mon, 9 Mar 2020 15:16:36 +0800
Subject: [PATCH 30/58] update

Signed-off-by: Zhiru Zhu 
---
 CHANGELOG.md                               |  1 +
 core/src/scheduler/task/BuildIndexTask.cpp |  3 +-
 core/unittest/db/test_delete.cpp           | 76 ++++++++++++++++++++++
 3 files changed, 79 insertions(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 70ac6ced65..1486fc5633 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -47,6 +47,7 @@ Please mark all change in change log and use the issue from GitHub
 -   \#1556 Index file not created after table and index created
 -   \#1560 Search crashed with Super-high dimensional binary vector
 -   \#1574 Set all existing bitset in cache when applying deletes
+-   \#1577 Row count incorrect if delete vectors then create index
 
 ## Feature
 -   \#216 Add CLI to get server info
diff --git a/core/src/scheduler/task/BuildIndexTask.cpp b/core/src/scheduler/task/BuildIndexTask.cpp
index 13d3b4a611..1ae0def7b7 100644
--- a/core/src/scheduler/task/BuildIndexTask.cpp
+++ b/core/src/scheduler/task/BuildIndexTask.cpp
@@ -12,6 +12,7 @@
 #include "scheduler/task/BuildIndexTask.h"
 
 #include 
+
 #include 
 #include 
 #include 
@@ -206,7 +207,7 @@ XBuildIndexTask::Execute() {
         // step 6: update meta
         table_file.file_type_ = engine::meta::TableFileSchema::INDEX;
         table_file.file_size_ = index->PhysicalSize();
-        table_file.row_count_ = index->Count();
+        table_file.row_count_ = file_->row_count_;  // index->Count();
 
         auto origin_file = *file_;
         origin_file.file_type_ = engine::meta::TableFileSchema::BACKUP;
diff --git a/core/unittest/db/test_delete.cpp b/core/unittest/db/test_delete.cpp
index 15618a950e..97c91fab64 100644
--- a/core/unittest/db/test_delete.cpp
+++ b/core/unittest/db/test_delete.cpp
@@ -264,6 +264,82 @@ TEST_F(DeleteTest, delete_multiple_times) {
     }
 }
 
+TEST_F(DeleteTest, delete_before_create_index) {
+    milvus::engine::meta::TableSchema table_info = BuildTableSchema();
+    table_info.engine_type_ = (int32_t)milvus::engine::EngineType::FAISS_IVFFLAT;
+    auto stat = db_->CreateTable(table_info);
+
+    milvus::engine::meta::TableSchema table_info_get;
+    table_info_get.table_id_ = table_info.table_id_;
+    stat = db_->DescribeTable(table_info_get);
+    ASSERT_TRUE(stat.ok());
+    ASSERT_EQ(table_info_get.dimension_, TABLE_DIM);
+
+    int64_t nb = 10000;
+    milvus::engine::VectorsData xb;
+    BuildVectors(nb, xb);
+
+    for (int64_t i = 0; i < nb; i++) {
+        xb.id_array_.push_back(i);
+    }
+
+    stat = db_->InsertVectors(table_info.table_id_, "", xb);
+    ASSERT_TRUE(stat.ok());
+
+    stat = db_->Flush();
+    ASSERT_TRUE(stat.ok());
+
+    std::random_device rd;
+    std::mt19937 gen(rd());
+    std::uniform_int_distribution dis(0, nb - 1);
+
+    int64_t num_query = 10;
+    std::map search_vectors;
+    for (int64_t i = 0; i < num_query; ++i) {
+        int64_t index = dis(gen);
+        milvus::engine::VectorsData search;
+        search.vector_count_ = 1;
+        for (int64_t j = 0; j < TABLE_DIM; j++) {
+            search.float_data_.push_back(xb.float_data_[index * TABLE_DIM + j]);
+        }
+        search_vectors.insert(std::make_pair(xb.id_array_[index], search));
+    }
+
+    milvus::engine::IDNumbers ids_to_delete;
+    for (auto& kv : search_vectors) {
+        ids_to_delete.emplace_back(kv.first);
+    }
+    stat = db_->DeleteVectors(table_info.table_id_, ids_to_delete);
+
+    stat = db_->Flush();
+    ASSERT_TRUE(stat.ok());
+
+    milvus::engine::TableIndex index;
+    index.engine_type_ = (int)milvus::engine::EngineType::FAISS_IVFSQ8;
+    index.extra_params_ = {{"nlist", 100}};
+    stat = db_->CreateIndex(table_info.table_id_, index);
+    ASSERT_TRUE(stat.ok());
+
+    uint64_t row_count;
+    stat = db_->GetTableRowCount(table_info.table_id_, row_count);
+    ASSERT_TRUE(stat.ok());
+    ASSERT_EQ(row_count, nb - ids_to_delete.size());
+
+    int topk = 10, nprobe = 10;
+    for (auto& pair : search_vectors) {
+        auto& search = pair.second;
+
+        std::vector tags;
+        milvus::engine::ResultIds result_ids;
+        milvus::engine::ResultDistances result_distances;
+        stat = db_->Query(dummy_context_, table_info.table_id_, tags, topk, {{"nprobe", nprobe}}, search, result_ids,
+                          result_distances);
+        ASSERT_NE(result_ids[0], pair.first);
+        //        ASSERT_LT(result_distances[0], 1e-4);
+        ASSERT_GT(result_distances[0], 1);
+    }
+}
+
 TEST_F(DeleteTest, delete_with_index) {
     milvus::engine::meta::TableSchema table_info = BuildTableSchema();
     table_info.engine_type_ = (int32_t)milvus::engine::EngineType::FAISS_IVFFLAT;

From 34da4da6d6ffa783c2ed87ae6593e443641cf19c Mon Sep 17 00:00:00 2001
From: Yhz 
Date: Mon, 9 Mar 2020 15:43:06 +0800
Subject: [PATCH 31/58] update readmemd adding

Signed-off-by: Yhz 
---
 core/src/server/web_impl/README.md | 72 ++++++++++++++++++++++++++----
 1 file changed, 63 insertions(+), 9 deletions(-)

diff --git a/core/src/server/web_impl/README.md b/core/src/server/web_impl/README.md
index efa12c8090..5e8a08bd4a 100644
--- a/core/src/server/web_impl/README.md
+++ b/core/src/server/web_impl/README.md
@@ -31,6 +31,7 @@
     - [`/collections/{collection_name}/segments/{segment_name}/ids` (GET)](#collectionscollection_namesegmentssegment_nameids-get)
     - [`/collections/{collection_name}/vectors` (PUT)](#collectionscollection_namevectors-put)
     - [`/collections/{collection_name}/vectors` (POST)](#collectionscollection_namevectors-post)
+    - [`/collections/{collection_name}/vectors` (GET)](#collectionscollection_namevectorsidvector_id-get)
     - [`/collections/{collection_name}/vectors` (OPTIONS)](#collectionscollection_namevectors-options)
     - [`/system/{msg}` (GET)](#systemmsg-get)
     - [`system/{op}` (PUT)](#systemop-put)
@@ -620,7 +621,9 @@ Updates the index type and nlist of a collection.
 Body

 {
   "index_type": string,
-  "nlist": integer($int64)
+  "params": {
+      ......
+  }
 }
 
 
 MethodPOST
@@ -632,7 +635,7 @@ Updates the index type and nlist of a collection.
 | Parameter  | Description  |  Required? |
 |-----------------|---|------|
 | `index_type`     | The type of indexing method to query the collection. Please refer to [Index Types](https://www.milvus.io/docs/reference/index.md) for detailed introduction of supported indexes. The default is "FLAT".  | No   |
-| `nlist`     |  Number of vector buckets in a file. The default is 16384.   | No   |
+| `params`     | The extra params of indexing method to query the collection. Please refer to [Index Types](https://www.milvus.io/docs/reference/index.md) for detailed introduction of supported indexes. The default is "FLAT".  | No   |
 
 ##### Query Parameters
 
@@ -990,8 +993,8 @@ $ curl -X GET "http://192.168.1.65:19121/collections/test_collection/segments/15
    "count":2,
    "vectors": [
       {
-         "vector": [], 
-         "id": ""
+         "vector": [0.1], 
+         "id": "1583727470435045000"
       }
    ]
 }
@@ -1039,8 +1042,8 @@ $ curl -X GET "http://192.168.1.65:19121/collections/test_collection/segments/15
 
 ```json
 {
-   "ids": [],
-   "count": 0
+   "ids": ["1583727470435045000"],
+   "count": 10000
 }
 ```
 
@@ -1059,7 +1062,7 @@ $ curl -X GET "http://192.168.1.65:19121/collections/test_collection/segments/15
 {
   "search": {
       "topk": integer($int64),
-      "tags": [string],
+      "partition_tags": [string],
       "file_ids": [string],
       "vectors": [[number($float/$uint8)]]
       "params": {
@@ -1078,8 +1081,8 @@ $ curl -X GET "http://192.168.1.65:19121/collections/test_collection/segments/15
 | `topk`     |  The top k most similar results of each query vector.   | Yes   |
 | `tags`    |  Tags of partitions that you need to search. You do not have to specify this value if the collection is not partitioned or you wish to search the whole collection.   |  No |
 | `file_ids`    |  IDs of the vector files. You do not have to specify this value if you do not use Milvus in distributed scenarios. Also, if you assign a value to `file_ids`, the value of `tags` is ignored.    |   No  |
-| `records`  |  Numeric vectors to insert to the collection.  |  Yes  |
-| `records_bin` | Binary vectors to insert to the collection. |    Yes   |
+| `vectors`  |  Vectors to query.  |  Yes  |
+| `params`  |  Extra params for search.  |  Yes  |
 
 > Note: Type of items of vectors depends on the metric used by the collection. If the collection uses `L2` or `IP`, you must use `float`. If the collection uses `HAMMING`, `JACCARD`, or `TANIMOTO`, you must use `uint8`.
 
@@ -1225,6 +1228,57 @@ $ curl -X POST "http://192.168.1.65:19121/collections/test_collection/vectors" -
 {"ids":["1578989029645098000","1578989029645098001","1578989029645098002","1578989029645098003"]}
 ```
 
+### `/collections/{collection_name}/vectors?id={vector_id}` (GET)
+
+Obtain a vector to by ID.
+
+#### Request
+
+| Request Component     | Value  |
+|-----------------|-----------|
+| Name     | `/collections/{collection_name}/vectors`  |
+| Header  | `accept: application/json`  |
+| Body    |   N/A |
+| Method    |   GET |
+
+#### Query Parameters
+
+| Parameter  | Description  |  Required? |
+|-----------------|---|------|
+| `collection_name` |  Name of the collection.      |   Yes     |
+| `vector_id` | Vector id.   | Yes   |
+
+
+#### Response
+
+| Status code    | Description |
+|-----------------|---|
+| 201     | Created |
+| 400     | The request is incorrect. Refer to the error message for details. |
+| 404     | The required resource does not exist. |
+
+#### Example
+
+##### Request
+
+```shell
+$ curl -X POST "http://192.168.1.65:19121/collections/test_collection/vectors?id=1578989029645098000" -H "accept: application/json" -H "Content-Type: application/json"
+```
+
+##### Response
+
+```json
+{
+   "vectors": [
+      {
+         "id": "1578989029645098000",
+         "vector": [0.1]
+      }
+   ]
+}
+```
+
+
 ### `/collections/{collection_name}/vectors` (OPTIONS)
 
 Use this API for Cross-Origin Resource Sharing (CORS).

From 2d96646356142a5b73478085780ff24731e28089 Mon Sep 17 00:00:00 2001
From: Nicky 
Date: Mon, 9 Mar 2020 16:33:28 +0800
Subject: [PATCH 32/58] update.

Signed-off-by: Nicky 
---
 .../knowhere/knowhere/index/vector_index/IndexIVF.cpp     | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/core/src/index/knowhere/knowhere/index/vector_index/IndexIVF.cpp b/core/src/index/knowhere/knowhere/index/vector_index/IndexIVF.cpp
index 523cdbaaf3..674638bcca 100644
--- a/core/src/index/knowhere/knowhere/index/vector_index/IndexIVF.cpp
+++ b/core/src/index/knowhere/knowhere/index/vector_index/IndexIVF.cpp
@@ -118,12 +118,12 @@ IVF::Search(const DatasetPtr& dataset, const Config& config) {
 
         //    std::stringstream ss_res_id, ss_res_dist;
         //    for (int i = 0; i < 10; ++i) {
-        //        printf("%llu", res_ids[i]);
+        //        printf("%llu", p_id[i]);
         //        printf("\n");
-        //        printf("%.6f", res_dis[i]);
+        //        printf("%.6f", p_dist[i]);
         //        printf("\n");
-        //        ss_res_id << res_ids[i] << " ";
-        //        ss_res_dist << res_dis[i] << " ";
+        //        ss_res_id << p_id[i] << " ";
+        //        ss_res_dist << p_dist[i] << " ";
         //    }
         //    std::cout << std::endl << "after search: " << std::endl;
         //    std::cout << ss_res_id.str() << std::endl;

From 0ec50358b7a2c5361edb67311c875e395d8dc104 Mon Sep 17 00:00:00 2001
From: groot 
Date: Mon, 9 Mar 2020 17:36:25 +0800
Subject: [PATCH 33/58] #1583 Default index_params should be "{}" for IDMAP
 table

Signed-off-by: groot 
---
 core/src/db/meta/MetaTypes.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/core/src/db/meta/MetaTypes.h b/core/src/db/meta/MetaTypes.h
index d8df8c2622..611df80d98 100644
--- a/core/src/db/meta/MetaTypes.h
+++ b/core/src/db/meta/MetaTypes.h
@@ -54,7 +54,7 @@ struct TableSchema {
     int64_t flag_ = 0;
     int64_t index_file_size_ = DEFAULT_INDEX_FILE_SIZE;
     int32_t engine_type_ = DEFAULT_ENGINE_TYPE;
-    std::string index_params_ = "{ \"nlist\": 16384 }";
+    std::string index_params_ = "{}";
     int32_t metric_type_ = DEFAULT_METRIC_TYPE;
     std::string owner_table_;
     std::string partition_tag_;

From 81b0275cc3a525214b6836410961b9e2a2488816 Mon Sep 17 00:00:00 2001
From: groot 
Date: Mon, 9 Mar 2020 19:57:18 +0800
Subject: [PATCH 34/58] typo

Signed-off-by: groot 
---
 core/src/db/DBImpl.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/core/src/db/DBImpl.cpp b/core/src/db/DBImpl.cpp
index c32819a830..b6992a3772 100644
--- a/core/src/db/DBImpl.cpp
+++ b/core/src/db/DBImpl.cpp
@@ -731,7 +731,7 @@ DBImpl::Compact(const std::string& table_id) {
                 break;
             }
         } else {
-            ENGINE_LOG_ERROR << "Segment " << file.segment_id_ << " has no deleted data. No need to compact";
+            ENGINE_LOG_DEBUG << "Segment " << file.segment_id_ << " has no deleted data. No need to compact";
         }
     }
 

From 7b493cb83513433741b98f555110bc97e35b5057 Mon Sep 17 00:00:00 2001
From: "xiaojun.lin" 
Date: Mon, 9 Mar 2020 20:08:31 +0800
Subject: [PATCH 35/58] fix

Signed-off-by: xiaojun.lin 
---
 CHANGELOG.md                                               | 1 +
 .../index/knowhere/knowhere/index/vector_index/nsg/NSG.cpp | 7 +------
 core/src/wrapper/ConfAdapter.cpp                           | 4 +++-
 3 files changed, 5 insertions(+), 7 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 9164c50506..8e46af40b2 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -36,6 +36,7 @@ Please mark all change in change log and use the issue from GitHub
 -   \#1507 set_config for insert_buffer_size is wrong
 -   \#1510 Add set interfaces for WAL configurations
 -   \#1511 Fix big integer cannot pass to server correctly
+-   \#1517 result is not correct when search vectors in multi partition, index type is RNSG 
 -   \#1518 Table count did not match after deleting vectors and compact
 -   \#1521 Make cache_insert_data take effect in-service
 -   \#1525 Add setter API for config preload_table
diff --git a/core/src/index/knowhere/knowhere/index/vector_index/nsg/NSG.cpp b/core/src/index/knowhere/knowhere/index/vector_index/nsg/NSG.cpp
index 16d20142b3..98d0137562 100644
--- a/core/src/index/knowhere/knowhere/index/vector_index/nsg/NSG.cpp
+++ b/core/src/index/knowhere/knowhere/index/vector_index/nsg/NSG.cpp
@@ -698,13 +698,8 @@ NsgIndex::Search(const float* query, const unsigned& nq, const unsigned& dim, co
                  int64_t* ids, SearchParams& params) {
     std::vector> resset(nq);
 
-    if (k >= 45) {
-        params.search_length = k;
-    }
-
     TimeRecorder rc("NsgIndex::search", 1);
-    // TODO(linxj): when to use openmp
-    if (nq <= 4) {
+    if (nq == 1) {
         GetNeighbors(query, resset[0], nsg, ¶ms);
     } else {
 #pragma omp parallel for
diff --git a/core/src/wrapper/ConfAdapter.cpp b/core/src/wrapper/ConfAdapter.cpp
index db06504def..115568ba91 100644
--- a/core/src/wrapper/ConfAdapter.cpp
+++ b/core/src/wrapper/ConfAdapter.cpp
@@ -225,7 +225,9 @@ NSGConfAdapter::CheckTrain(milvus::json& oricfg) {
 
     // auto tune params
     oricfg[knowhere::IndexParams::nlist] = MatchNlist(oricfg[knowhere::meta::ROWS].get(), 8192, 8192);
-    oricfg[knowhere::IndexParams::nprobe] = int(oricfg[knowhere::IndexParams::nlist].get() * 0.01);
+
+    int64_t nprobe = int(oricfg[knowhere::IndexParams::nlist].get() * 0.1);
+    oricfg[knowhere::IndexParams::nprobe] = nprobe < 1 ? 1 : nprobe;
 
     return true;
 }

From c68b5eb7447d53b5b356de6c87f54cde5b34f18c Mon Sep 17 00:00:00 2001
From: groot 
Date: Mon, 9 Mar 2020 20:21:26 +0800
Subject: [PATCH 36/58] typo

Signed-off-by: groot 
---
 core/src/db/DBImpl.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/core/src/db/DBImpl.cpp b/core/src/db/DBImpl.cpp
index b6992a3772..f83c0d2b5a 100644
--- a/core/src/db/DBImpl.cpp
+++ b/core/src/db/DBImpl.cpp
@@ -739,7 +739,7 @@ DBImpl::Compact(const std::string& table_id) {
         ENGINE_LOG_DEBUG << "Finished compacting table: " << table_id;
     }
 
-    ENGINE_LOG_ERROR << "Updating meta after compaction...";
+    ENGINE_LOG_DEBUG << "Updating meta after compaction...";
 
     /*
     // Drop index again, in case some files were in the index building process during compacting

From 593077e85b1f4c4c46e06f1e8482ad183cb54338 Mon Sep 17 00:00:00 2001
From: Zhiru Zhu 
Date: Mon, 9 Mar 2020 20:40:40 +0800
Subject: [PATCH 37/58] update

Signed-off-by: Zhiru Zhu 
---
 CHANGELOG.md                                         | 1 +
 core/src/codecs/default/DefaultDeletedDocsFormat.cpp | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index ee5f9efbd1..9f189b728b 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -50,6 +50,7 @@ Please mark all change in change log and use the issue from GitHub
 -   \#1571 Meta engine type become IDMAP after dropping index for BINARY table
 -   \#1574 Set all existing bitset in cache when applying deletes
 -   \#1577 Row count incorrect if delete vectors then create index
+-   \#1590 Server down caused by failed to write file during concurrent mixed operations
 
 ## Feature
 -   \#216 Add CLI to get server info
diff --git a/core/src/codecs/default/DefaultDeletedDocsFormat.cpp b/core/src/codecs/default/DefaultDeletedDocsFormat.cpp
index fb211f6969..190600af05 100644
--- a/core/src/codecs/default/DefaultDeletedDocsFormat.cpp
+++ b/core/src/codecs/default/DefaultDeletedDocsFormat.cpp
@@ -130,7 +130,7 @@ DefaultDeletedDocsFormat::write(const store::DirectoryPtr& directory_ptr, const
         ENGINE_LOG_ERROR << err_msg;
         throw Exception(SERVER_WRITE_ERROR, err_msg);
     }
-    if (::write(del_fd, deleted_docs_list.data(), new_num_bytes) == -1) {
+    if (::write(del_fd, deleted_docs_list.data(), sizeof(segment::offset_t) * deleted_docs->GetSize()) == -1) {
         std::string err_msg = "Failed to write to file" + temp_path + ", error: " + std::strerror(errno);
         ENGINE_LOG_ERROR << err_msg;
         throw Exception(SERVER_WRITE_ERROR, err_msg);

From 13d3cff9b71ff842b3516f65b451ff13d45fe1b7 Mon Sep 17 00:00:00 2001
From: groot 
Date: Mon, 9 Mar 2020 20:52:10 +0800
Subject: [PATCH 38/58] #1580 Old segment folder not removed after merge /
 compact if create_index is called before adding data

Signed-off-by: groot 
---
 CHANGELOG.md                        |  1 +
 core/src/db/meta/MySQLMetaImpl.cpp  | 72 ++++++++++++++++++++++-------
 core/src/db/meta/SqliteMetaImpl.cpp | 56 +++++++++++++++-------
 3 files changed, 95 insertions(+), 34 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index ee5f9efbd1..da7c9bd64c 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -50,6 +50,7 @@ Please mark all change in change log and use the issue from GitHub
 -   \#1571 Meta engine type become IDMAP after dropping index for BINARY table
 -   \#1574 Set all existing bitset in cache when applying deletes
 -   \#1577 Row count incorrect if delete vectors then create index
+-   \#1580 Old segment folder not removed after merge/compact if create_index is called before adding data
 
 ## Feature
 -   \#216 Add CLI to get server info
diff --git a/core/src/db/meta/MySQLMetaImpl.cpp b/core/src/db/meta/MySQLMetaImpl.cpp
index bd0d4de96d..c638d1ae85 100644
--- a/core/src/db/meta/MySQLMetaImpl.cpp
+++ b/core/src/db/meta/MySQLMetaImpl.cpp
@@ -2026,6 +2026,7 @@ Status
 MySQLMetaImpl::CleanUpFilesWithTTL(uint64_t seconds /*, CleanUpFilter* filter*/) {
     auto now = utils::GetMicroSecTimeStamp();
     std::set table_ids;
+    std::map segment_ids;
 
     // remove to_delete files
     try {
@@ -2053,7 +2054,7 @@ MySQLMetaImpl::CleanUpFilesWithTTL(uint64_t seconds /*, CleanUpFilter* filter*/)
             mysqlpp::StoreQueryResult res = query.store();
 
             TableFileSchema table_file;
-            std::vector idsToDelete;
+            std::vector delete_ids;
 
             int64_t clean_files = 0;
             for (auto& resRow : res) {
@@ -2078,30 +2079,22 @@ MySQLMetaImpl::CleanUpFilesWithTTL(uint64_t seconds /*, CleanUpFilter* filter*/)
                 server::CommonUtil::EraseFromCache(table_file.location_);
 
                 if (table_file.file_type_ == (int)TableFileSchema::TO_DELETE) {
-                    // If we are deleting a raw table file, it means it's okay to delete the entire segment directory.
-                    // Else, we can only delete the single file
-                    // TODO(zhiru): We determine whether a table file is raw by its engine type. This is a bit hacky
-                    if (utils::IsRawIndexType(table_file.engine_type_)) {
-                        utils::DeleteSegment(options_, table_file);
-                        std::string segment_dir;
-                        utils::GetParentPath(table_file.location_, segment_dir);
-                        ENGINE_LOG_DEBUG << "Remove segment directory: " << segment_dir;
-                    } else {
-                        utils::DeleteTableFilePath(options_, table_file);
-                        ENGINE_LOG_DEBUG << "Remove table file: " << table_file.location_;
-                    }
+                    // delete file from disk storage
+                    utils::DeleteTableFilePath(options_, table_file);
+                    ENGINE_LOG_DEBUG << "Remove file id:" << table_file.id_ << " location:" << table_file.location_;
 
-                    idsToDelete.emplace_back(std::to_string(table_file.id_));
+                    delete_ids.emplace_back(std::to_string(table_file.id_));
                     table_ids.insert(table_file.table_id_);
+                    segment_ids.insert(std::make_pair(table_file.segment_id_, table_file));
 
-                    ++clean_files;
+                    clean_files++;
                 }
             }
 
             // delete file from meta
-            if (!idsToDelete.empty()) {
+            if (!delete_ids.empty()) {
                 std::stringstream idsToDeleteSS;
-                for (auto& id : idsToDelete) {
+                for (auto& id : delete_ids) {
                     idsToDeleteSS << "id = " << id << " OR ";
                 }
 
@@ -2217,6 +2210,51 @@ MySQLMetaImpl::CleanUpFilesWithTTL(uint64_t seconds /*, CleanUpFilter* filter*/)
         return HandleException("GENERAL ERROR WHEN CLEANING UP TABLES WITH TTL", e.what());
     }
 
+    // remove deleted segment folder
+    // don't remove segment folder until all its tablefiles has been deleted
+    try {
+        server::MetricCollector metric;
+
+        {
+            mysqlpp::ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_);
+
+            bool is_null_connection = (connectionPtr == nullptr);
+            fiu_do_on("MySQLMetaImpl.CleanUpFilesWithTTL.RemoveDeletedSegmentFolder_NUllConnection",
+                      is_null_connection = true);
+            fiu_do_on("MySQLMetaImpl.CleanUpFilesWithTTL.RemoveDeletedSegmentFolder_ThrowException",
+                      throw std::exception(););
+            if (is_null_connection) {
+                return Status(DB_ERROR, "Failed to connect to meta server(mysql)");
+            }
+
+            int64_t remove_segments = 0;
+            for (auto& segment_id : segment_ids) {
+                mysqlpp::Query query = connectionPtr->query();
+                query << "SELECT id"
+                      << " FROM " << META_TABLEFILES << " WHERE segment_id = " << mysqlpp::quote << segment_id.first
+                      << ";";
+
+                ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUpFilesWithTTL: " << query.str();
+
+                mysqlpp::StoreQueryResult res = query.store();
+
+                if (res.empty()) {
+                    utils::DeleteSegment(options_, segment_id.second);
+                    std::string segment_dir;
+                    utils::GetParentPath(segment_id.second.location_, segment_dir);
+                    ENGINE_LOG_DEBUG << "Remove segment directory: " << segment_dir;
+                    ++remove_segments;
+                }
+            }
+
+            if (remove_segments > 0) {
+                ENGINE_LOG_DEBUG << "Remove " << remove_segments << " segments folder";
+            }
+        }
+    } catch (std::exception& e) {
+        return HandleException("GENERAL ERROR WHEN CLEANING UP TABLES WITH TTL", e.what());
+    }
+
     return Status::OK();
 }
 
diff --git a/core/src/db/meta/SqliteMetaImpl.cpp b/core/src/db/meta/SqliteMetaImpl.cpp
index ce5ff99742..622d36175e 100644
--- a/core/src/db/meta/SqliteMetaImpl.cpp
+++ b/core/src/db/meta/SqliteMetaImpl.cpp
@@ -1231,11 +1231,13 @@ SqliteMetaImpl::FilesByType(const std::string& table_id, const std::vector&
                         break;
                     case (int)TableFileSchema::NEW:msg = msg + " new files:" + std::to_string(new_count);
                         break;
-                    case (int)TableFileSchema::NEW_MERGE:msg = msg + " new_merge files:"
-                                                               + std::to_string(new_merge_count);
+                    case (int)TableFileSchema::NEW_MERGE:
+                        msg = msg + " new_merge files:"
+                              + std::to_string(new_merge_count);
                         break;
-                    case (int)TableFileSchema::NEW_INDEX:msg = msg + " new_index files:"
-                                                               + std::to_string(new_index_count);
+                    case (int)TableFileSchema::NEW_INDEX:
+                        msg = msg + " new_index files:"
+                              + std::to_string(new_index_count);
                         break;
                     case (int)TableFileSchema::TO_INDEX:msg = msg + " to_index files:" + std::to_string(to_index_count);
                         break;
@@ -1360,6 +1362,7 @@ Status
 SqliteMetaImpl::CleanUpFilesWithTTL(uint64_t seconds /*, CleanUpFilter* filter*/) {
     auto now = utils::GetMicroSecTimeStamp();
     std::set table_ids;
+    std::map segment_ids;
 
     // remove to_delete files
     try {
@@ -1409,23 +1412,16 @@ SqliteMetaImpl::CleanUpFilesWithTTL(uint64_t seconds /*, CleanUpFilter* filter*/
                 server::CommonUtil::EraseFromCache(table_file.location_);
 
                 if (table_file.file_type_ == (int)TableFileSchema::TO_DELETE) {
-                    // If we are deleting a raw table file, it means it's okay to delete the entire segment directory.
-                    // Else, we can only delete the single file
-                    // TODO(zhiru): We determine whether a table file is raw by its engine type. This is a bit hacky
-                    if (utils::IsRawIndexType(table_file.engine_type_)) {
-                        utils::DeleteSegment(options_, table_file);
-                        std::string segment_dir;
-                        utils::GetParentPath(table_file.location_, segment_dir);
-                        ENGINE_LOG_DEBUG << "Remove segment directory: " << segment_dir;
-                    } else {
-                        utils::DeleteTableFilePath(options_, table_file);
-                        ENGINE_LOG_DEBUG << "Remove table file: " << table_file.location_;
-                    }
-
                     // delete file from meta
                     ConnectorPtr->remove(table_file.id_);
 
+                    // delete file from disk storage
+                    utils::DeleteTableFilePath(options_, table_file);
+
+                    ENGINE_LOG_DEBUG << "Remove file id:" << table_file.file_id_ << " location:"
+                                     << table_file.location_;
                     table_ids.insert(table_file.table_id_);
+                    segment_ids.insert(std::make_pair(table_file.segment_id_, table_file));
 
                     ++clean_files;
                 }
@@ -1500,6 +1496,32 @@ SqliteMetaImpl::CleanUpFilesWithTTL(uint64_t seconds /*, CleanUpFilter* filter*/
         return HandleException("Encounter exception when delete table folder", e.what());
     }
 
+    // remove deleted segment folder
+    // don't remove segment folder until all its tablefiles has been deleted
+    try {
+        fiu_do_on("SqliteMetaImpl.CleanUpFilesWithTTL.RemoveSegmentFolder_ThrowException", throw std::exception());
+        server::MetricCollector metric;
+
+        int64_t remove_segments = 0;
+        for (auto& segment_id : segment_ids) {
+            auto selected = ConnectorPtr->select(columns(&TableFileSchema::id_),
+                                                 where(c(&TableFileSchema::segment_id_) == segment_id.first));
+            if (selected.size() == 0) {
+                utils::DeleteSegment(options_, segment_id.second);
+                std::string segment_dir;
+                utils::GetParentPath(segment_id.second.location_, segment_dir);
+                ENGINE_LOG_DEBUG << "Remove segment directory: " << segment_dir;
+                ++remove_segments;
+            }
+        }
+
+        if (remove_segments > 0) {
+            ENGINE_LOG_DEBUG << "Remove " << remove_segments << " segments folder";
+        }
+    } catch (std::exception& e) {
+        return HandleException("Encounter exception when delete table folder", e.what());
+    }
+
     return Status::OK();
 }
 

From 0b32198c256e3525a0056f328f6a01e072581a8d Mon Sep 17 00:00:00 2001
From: Yhz 
Date: Mon, 9 Mar 2020 21:03:05 +0800
Subject: [PATCH 39/58] add index param

Signed-off-by: Yhz 
---
 core/src/server/web_impl/README.md | 40 ++++++++++++++++++++++++++++--
 1 file changed, 38 insertions(+), 2 deletions(-)

diff --git a/core/src/server/web_impl/README.md b/core/src/server/web_impl/README.md
index 5e8a08bd4a..3b37dff0e4 100644
--- a/core/src/server/web_impl/README.md
+++ b/core/src/server/web_impl/README.md
@@ -635,7 +635,7 @@ Updates the index type and nlist of a collection.
 | Parameter  | Description  |  Required? |
 |-----------------|---|------|
 | `index_type`     | The type of indexing method to query the collection. Please refer to [Index Types](https://www.milvus.io/docs/reference/index.md) for detailed introduction of supported indexes. The default is "FLAT".  | No   |
-| `params`     | The extra params of indexing method to query the collection. Please refer to [Index Types](https://www.milvus.io/docs/reference/index.md) for detailed introduction of supported indexes. The default is "FLAT".  | No   |
+| `params`     | The extra params of indexing method to query the collection. Please refer to - [Index params](#index_param) for detailed introduction of supported indexes.  | No   |
 
 ##### Query Parameters
 
@@ -1082,7 +1082,7 @@ $ curl -X GET "http://192.168.1.65:19121/collections/test_collection/segments/15
 | `tags`    |  Tags of partitions that you need to search. You do not have to specify this value if the collection is not partitioned or you wish to search the whole collection.   |  No |
 | `file_ids`    |  IDs of the vector files. You do not have to specify this value if you do not use Milvus in distributed scenarios. Also, if you assign a value to `file_ids`, the value of `tags` is ignored.    |   No  |
 | `vectors`  |  Vectors to query.  |  Yes  |
-| `params`  |  Extra params for search.  |  Yes  |
+| `params`  |  Extra params for search. Please refer to [Search param](#index_param) to get more detail information.  |  Yes  |
 
 > Note: Type of items of vectors depends on the metric used by the collection. If the collection uses `L2` or `IP`, you must use `float`. If the collection uses `HAMMING`, `JACCARD`, or `TANIMOTO`, you must use `uint8`.
 
@@ -1459,6 +1459,42 @@ $ curl -X PUT "http://192.168.1.65:19121/system/task" -H "accept: application/js
 {"code": 0, "message": "success"}
 ```
 
+## Index param
+
+For each index type, it has specific index param and searach param.
+
+
+
+
+ 
+ 
+ 
+
+
+ 
+ 
+ 
+
+
+ 
+ 
+ 
+
+
+ 
+ 
+ 
+
+
+ 
+ 
+ 
+
+
Index typeCreate index paramSearch param
 IVFFLAT
{"nlist": $int}
{"nprobe": $int}
 IVFPQ
{"m": $int, "nlist": $int}
{"nprobe": $int}
 IVFSQ8
{"nlist": $int}
{"nprobe": $int}
 IVFSQ8H
{"nlist": $int}
{"nprobe": $int}
 HNSW
{"M": $int, "efConstruction": $int}
{"ef": $int}

+
+You can find more details about parameters above [here](https://github.com/yamasite/docs/blob/v0.7.0/site/zh-CN/guides/index.md)
+
+
 ## Error Codes
 
 The RESTful API returns error messages as JSON text. Each type of error message has a specific error code.

From 9bb2f4b7280ddc2d588651a6cab81d500ba7c99f Mon Sep 17 00:00:00 2001
From: groot 
Date: Mon, 9 Mar 2020 21:14:14 +0800
Subject: [PATCH 40/58] typo

Signed-off-by: groot 
---
 core/src/db/meta/MySQLMetaImpl.cpp | 1 +
 1 file changed, 1 insertion(+)

diff --git a/core/src/db/meta/MySQLMetaImpl.cpp b/core/src/db/meta/MySQLMetaImpl.cpp
index c638d1ae85..ca59d82793 100644
--- a/core/src/db/meta/MySQLMetaImpl.cpp
+++ b/core/src/db/meta/MySQLMetaImpl.cpp
@@ -27,6 +27,7 @@
 #include 
 #include 
 #include 
+#include 
 
 #include "MetaConsts.h"
 #include "db/IDGenerator.h"

From f957b050e89800a30383fd90f479437b7272712a Mon Sep 17 00:00:00 2001
From: Yhz 
Date: Mon, 9 Mar 2020 21:18:12 +0800
Subject: [PATCH 41/58] update README

Signed-off-by: Yhz 
---
 core/src/server/web_impl/README.md | 23 +++++++++++++++++++++++
 1 file changed, 23 insertions(+)

diff --git a/core/src/server/web_impl/README.md b/core/src/server/web_impl/README.md
index 3b37dff0e4..22a22e6df0 100644
--- a/core/src/server/web_impl/README.md
+++ b/core/src/server/web_impl/README.md
@@ -500,6 +500,29 @@ $ curl -X GET "http://192.168.1.65:19121/collections/test_collection" -H "accept
 {"collection_name":"test_collection","dimension":1,"index_file_size":10,"metric_type":"L2","count":0,"index":"FLAT","nlist":16384}
 ```
 
+##### Request
+
+```shell
+$ curl -X GET "http://192.168.1.65:19121/collections/test_collection?info=stat" -H "accept: application/json"
+```
+
+##### Response
+
+```json
+{
+   "count":150000,
+   "partitions_stat":[
+      {
+         "count":1000,"partition_tag":"_default",
+         "segments_stat":[
+            {"count":1000,"index":"IVFFLAT","segment_name":"1583727170217439000","size":5284922}
+         ]
+      }
+   ]
+}
+```
+
+
 ### `/collections/{collection_name}` (DELETE)
 
 Drops a collection by name.

From e96d7279b212de09078aacd29ce53c42f526211a Mon Sep 17 00:00:00 2001
From: Nicky 
Date: Mon, 9 Mar 2020 21:21:36 +0800
Subject: [PATCH 42/58] update

Signed-off-by: Nicky 
---
 .../knowhere/knowhere/index/vector_index/nsg/NSG.cpp     | 9 ---------
 1 file changed, 9 deletions(-)

diff --git a/core/src/index/knowhere/knowhere/index/vector_index/nsg/NSG.cpp b/core/src/index/knowhere/knowhere/index/vector_index/nsg/NSG.cpp
index 98d0137562..e220b42c64 100644
--- a/core/src/index/knowhere/knowhere/index/vector_index/nsg/NSG.cpp
+++ b/core/src/index/knowhere/knowhere/index/vector_index/nsg/NSG.cpp
@@ -728,15 +728,6 @@ NsgIndex::Search(const float* query, const unsigned& nq, const unsigned& dim, co
         }
     }
     rc.RecordSection("merge");
-
-    // ProfilerStart("xx.prof");
-    // std::vector resset;
-    // GetNeighbors(query, resset, nsg, ¶ms);
-    // for (int i = 0; i < k; ++i) {
-    //    ids[i] = resset[i].id;
-    // dist[i] = resset[i].distance;
-    //}
-    // ProfilerStop();
 }
 
 void

From 0ffc7017ffc6e488b43b4d9d67716ddcc3e95b7f Mon Sep 17 00:00:00 2001
From: "xiaojun.lin" 
Date: Mon, 9 Mar 2020 21:33:10 +0800
Subject: [PATCH 43/58] update.

Signed-off-by: xiaojun.lin 
---
 core/src/index/unittest/test_nsg/test_nsg.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/core/src/index/unittest/test_nsg/test_nsg.cpp b/core/src/index/unittest/test_nsg/test_nsg.cpp
index 76db0cd8c2..074e3f2736 100644
--- a/core/src/index/unittest/test_nsg/test_nsg.cpp
+++ b/core/src/index/unittest/test_nsg/test_nsg.cpp
@@ -233,7 +233,7 @@ TEST_F(NSGInterfaceTest, comparetest) {
 //            }
 //        }
 //    }
-//    printf("R@1 = %.4f\n", n_1 / float(nq));
+//    printf("R@1 = %.4f\n", n_1 / float(nq));;
 //    printf("R@10 = %.4f\n", n_10 / float(nq));
 //    printf("R@100 = %.4f\n", n_100 / float(nq));
 //}

From c378eecdacbd50bec8527e11e2699afb82f198e0 Mon Sep 17 00:00:00 2001
From: Yhz 
Date: Mon, 9 Mar 2020 22:25:15 +0800
Subject: [PATCH 44/58] update README.md

Signed-off-by: Yhz 
---
 core/src/server/web_impl/README.md | 88 ++++++++++++++----------------
 1 file changed, 40 insertions(+), 48 deletions(-)

diff --git a/core/src/server/web_impl/README.md b/core/src/server/web_impl/README.md
index 22a22e6df0..dc0db1ab69 100644
--- a/core/src/server/web_impl/README.md
+++ b/core/src/server/web_impl/README.md
@@ -69,7 +69,7 @@ Checks whether the web server is running.
 ##### Request
 
 ```shell
-$ curl -X GET "http://192.168.1.65:19121/state" -H "accept: application/json"
+$ curl -X GET "http://127.0.0.1:19121/state" -H "accept: application/json"
 ```
 
 ##### Response
@@ -104,7 +104,7 @@ Gets CPU/GPU information from the host.
 ##### Request
 
 ```shell
-$ curl -X GET "http://192.168.1.65:19121/devices" -H "accept: application/json"
+$ curl -X GET "http://127.0.0.1:19121/devices" -H "accept: application/json"
 ```
 
 ##### Response
@@ -138,7 +138,7 @@ Gets the values of parameters in `cache_config` and `engine_config` of the Milvu
 ##### Request
 
 ```shell
-$ curl -X GET "http://192.168.1.65:19121/config/advanced" -H "accept: application/json"
+$ curl -X GET "http://127.0.0.1:19121/config/advanced" -H "accept: application/json"
 ```
 
 ##### Response
@@ -192,7 +192,7 @@ Updates the values of parameters in `cache_config` and `engine_config` of the Mi
 ##### Request
 
 ```shell
-$ curl -X PUT "http://192.168.1.65:19121/config/advanced" -H "accept: application/json" -H "Content-Type: application/json" -d "{\"cpu_cache_capacity\":4,\"cache_insert_data\":false,\"use_blas_threshold\":1100,\"gpu_search_threshold\":1000}"
+$ curl -X PUT "http://127.0.0.1:19121/config/advanced" -H "accept: application/json" -H "Content-Type: application/json" -d "{\"cpu_cache_capacity\":4,\"cache_insert_data\":false,\"use_blas_threshold\":1100,\"gpu_search_threshold\":1000}"
 ```
 
 ##### Response
@@ -221,7 +221,7 @@ Use this API for Cross-Origin Resource Sharing (CORS).
 ##### Request
 
 ```shell
-$ curl -X OPTIONS "http://192.168.1.65:19121/config/advanced"
+$ curl -X OPTIONS "http://127.0.0.1:19121/config/advanced"
 ```
 
 ### `/config/gpu_resources` (GET)
@@ -251,7 +251,7 @@ Gets the parameter values in `gpu_resource_config` of the Milvus configuration f
 ##### Request
 
 ```shell
-$ curl -X GET "http://192.168.1.65:19121/config/gpu_resources" -H "accept: application/json"
+$ curl -X GET "http://127.0.0.1:19121/config/gpu_resources" -H "accept: application/json"
 ```
 
 ##### Response
@@ -306,7 +306,7 @@ Updates the parameter values in `gpu_resource_config` of the Milvus configuratio
 ##### Request
 
 ```shell
-$ curl -X PUT "http://192.168.1.65:19121/config/gpu_resources" -H "accept: application/json" -H "Content-Type: application/json" -d "{\"enable\":true,\"cache_capacity\":1,\"search_resources\":[\"GPU0\"],\"build_index_resources\":[\"GPU0\"]}"
+$ curl -X PUT "http://127.0.0.1:19121/config/gpu_resources" -H "accept: application/json" -H "Content-Type: application/json" -d "{\"enable\":true,\"cache_capacity\":1,\"search_resources\":[\"GPU0\"],\"build_index_resources\":[\"GPU0\"]}"
 ```
 
 ##### Response
@@ -335,7 +335,7 @@ Use this API for Cross-Origin Resource Sharing (CORS).
 ##### Request
 
 ```shell
-$ curl -X OPTIONS "http://192.168.1.65:19121/config/gpu_resources"
+$ curl -X OPTIONS "http://127.0.0.1:19121/config/gpu_resources"
 ```
 
 ### `/collections` (GET)
@@ -371,7 +371,7 @@ Gets all collections starting from `offset` and ends with `page_size`.
 ##### Request
 
 ```shell
-$ curl -X GET "http://192.168.1.65:19121/collections?offset=0&page_size=1" -H "accept: application/json"
+$ curl -X GET "http://127.0.0.1:19121/collections?offset=0&page_size=1" -H "accept: application/json"
 ```
 
 ##### Response
@@ -424,7 +424,7 @@ Creates a collection.
 ##### Request
 
 ```shell
-$ curl -X POST "http://192.168.1.65:19121/collections" -H "accept: application/json" -H "Content-Type: application/json" -d "{\"collection_name\":\"test_collection\",\"dimension\":1,\"index_file_size\":10,\"metric_type\":\"L2\"}"
+$ curl -X POST "http://127.0.0.1:19121/collections" -H "accept: application/json" -H "Content-Type: application/json" -d "{\"collection_name\":\"test_collection\",\"dimension\":1,\"index_file_size\":10,\"metric_type\":\"L2\"}"
 ```
 
 ##### Response
@@ -453,7 +453,7 @@ Use this API for Cross-Origin Resource Sharing (CORS).
 ##### Request
 
 ```shell
-$ curl -X OPTIONS "http://192.168.1.65:19121/collections"
+$ curl -X OPTIONS "http://127.0.0.1:19121/collections"
 ```
 
 ### `/collections/{collection_name}` (GET)
@@ -491,19 +491,19 @@ Gets all information about a collection by name.
 ##### Request
 
 ```shell
-$ curl -X GET "http://192.168.1.65:19121/collections/test_collection" -H "accept: application/json"
+$ curl -X GET "http://127.0.0.1:19121/collections/test_collection" -H "accept: application/json"
 ```
 
 ##### Response
 
 ```json
-{"collection_name":"test_collection","dimension":1,"index_file_size":10,"metric_type":"L2","count":0,"index":"FLAT","nlist":16384}
+{"collection_name":"test_collection","dimension":1,"index_file_size":10,"metric_type":"L2","count":0,"index":"FLAT", "index_params":{}}
 ```
 
 ##### Request
 
 ```shell
-$ curl -X GET "http://192.168.1.65:19121/collections/test_collection?info=stat" -H "accept: application/json"
+$ curl -X GET "http://127.0.0.1:19121/collections/test_collection?info=stat" -H "accept: application/json"
 ```
 
 ##### Response
@@ -556,7 +556,7 @@ Drops a collection by name.
 
 
 ```shell
-$ curl -X DELETE "http://192.168.1.65:19121/collections/test_collection" -H "accept: application/json"
+$ curl -X DELETE "http://127.0.0.1:19121/collections/test_collection" -H "accept: application/json"
 ```
 
 If the deletion is successful, no message will be returned.
@@ -586,7 +586,7 @@ Use this API for Cross-Origin Resource Sharing (CORS).
 ##### Request
 
 ```shell
-$ curl -X OPTIONS "http://192.168.1.65:19121/collections/test_collection"
+$ curl -X OPTIONS "http://127.0.0.1:19121/collections/test_collection"
 ```
 
 ### `/collections/{collection_name}/indexes` (GET)
@@ -621,14 +621,14 @@ Gets the index type and nlist of a collection.
 ##### Request
 
 ```shell
-$ curl -X GET "http://192.168.1.65:19121/collections/test_collection/indexes" -H "accept: application/json"
+$ curl -X GET "http://127.0.0.1:19121/collections/test_collection/indexes" -H "accept: application/json"
 ```
 
 ##### Response
 
 
 ```json
-{"index_type":"FLAT","nlist":16384}
+{"index_type":"FLAT","params":{"nlist":4096}}
 ```
 
 ### `/collections/{collection_name}/indexes` (POST)
@@ -674,20 +674,12 @@ Updates the index type and nlist of a collection.
 | 400     | The request is incorrect. Refer to the error message for details. |
 | 404     | The required resource does not exist. |
 
-#### Response
-
-| Status code    | Description |
-|-----------------|---|
-| 201     | Created |
-| 400     | The request is incorrect. Refer to the error message for details. |
-| 404     | The required resource does not exist. |
-
 #### Example
 
 ##### Request
 
 ```shell
-$ curl -X POST "http://192.168.1.65:19121/collections/test_collection/indexes" -H "accept: application/json" -H "Content-Type: application/json" -d "{\"index_type\":\"FLAT\",\"nlist\":16384}"
+$ curl -X POST "http://127.0.0.1:19121/collections/test_collection/indexes" -H "accept: application/json" -H "Content-Type: application/json" -d "{\"index_type\":\"IVFFLAT\",\"params\": {\"nlist\":4096}}"
 ```
 
 ##### Response
@@ -729,7 +721,7 @@ Drops an index for a collection.
 ##### Request
 
 ```shell
-$ curl -X DELETE "http://192.168.1.65:19121/collections/test_collection/indexes" -H "accept: application/json"
+$ curl -X DELETE "http://127.0.0.1:19121/collections/test_collection/indexes" -H "accept: application/json"
 ```
 
 If the deletion is successful, no message will be returned.
@@ -759,7 +751,7 @@ Use this API for Cross-Origin Resource Sharing (CORS).
 ##### Request
 
 ```shell
-$ curl -X OPTIONS "http://192.168.1.65:19121/collections/test_collection/indexes"
+$ curl -X OPTIONS "http://127.0.0.1:19121/collections/test_collection/indexes"
 ```
 
 ### `/collections/{collection_name}/partitions` (GET)
@@ -796,13 +788,13 @@ Gets all partitions in a collection starting from `offset` and ends with `page_s
 ##### Request
 
 ```shell
-$ curl -X GET "http://192.168.1.65:19121/collections/test_collection/partitions?offset=0&page_size=3" -H "accept: application/json"
+$ curl -X GET "http://127.0.0.1:19121/collections/test_collection/partitions?offset=0&page_size=3" -H "accept: application/json"
 ```
 
 ##### Response
 
 ```json
-{"partitions":[{"partition_name":"partition_1","partition_tag":"test_tag"},{"partition_name":"partition_2","partition_tag":"test_2"},{"partition_name":"partition_3","partition_tag":"test_3"}]}
+{"partitions":[{"partition_tag": "_default"},{"partition_tag":"test_tag"},{"partition_tag":"test_2"}], "count":10}
 ```
 
 ### `/collections/{collection_name}/partitions` (POST)
@@ -831,7 +823,7 @@ Creates a partition in a collection.
 ##### Request
 
 ```shell
-$ curl -X POST "http://192.168.1.65:19121/collections/test_collection/partitions" -H "accept: application/json" -H "Content-Type: application/json" -d "{\"partition_name\": \"partition_1\",\"partition_tag\": \"test\"}"
+$ curl -X POST "http://127.0.0.1:19121/collections/test_collection/partitions" -H "accept: application/json" -H "Content-Type: application/json" -d "{\"partition_tag\": \"test\"}"
 ```
 
 ##### Response
@@ -865,7 +857,7 @@ Use this API for Cross-Origin Resource Sharing (CORS).
 ##### Request
 
 ```shell
-$ curl -X OPTIONS "http://192.168.1.65:19121/collections/test_collection/partitions"
+$ curl -X OPTIONS "http://127.0.0.1:19121/collections/test_collection/partitions"
 ```
 
 ### `/collections/{collection_name}/partitions` (DELETE)
@@ -907,7 +899,7 @@ Deletes a partition by tag.
 ##### Request
 
 ```shell
-$ curl -X DELETE "http://192.168.1.65:19121/collections/test_collection/partitions/tags_01" -H "accept: application/json"
+$ curl -X DELETE "http://127.0.0.1:19121/collections/test_collection/partitions -H "accept: application/json" -d "{\"partition_tag\": \"tags_01\"}"
 ```
 
 The deletion is successful if no information is returned.
@@ -947,7 +939,7 @@ Gets all segments in a collection starting from `offset` and ends with `page_siz
 ##### Request
 
 ```shell
-$ curl -X GET "http://192.168.1.65:19121/collections/test_collection/segments?offset=0&page_size=3" -H "accept: application/json"
+$ curl -X GET "http://127.0.0.1:19121/collections/test_collection/segments?offset=0&page_size=1" -H "accept: application/json"
 ```
 
 ##### Response
@@ -1004,7 +996,7 @@ Gets all vectors of segment in a collection starting from `offset` and ends with
 ##### Request
 
 ```shell
-$ curl -X GET "http://192.168.1.65:19121/collections/test_collection/segments/1583727470444700000/vectors?offset=0&page_size=1" -H "accept: application/json"
+$ curl -X GET "http://127.0.0.1:19121/collections/test_collection/segments/1583727470444700000/vectors?offset=0&page_size=1" -H "accept: application/json"
 ```
 
 ##### Response
@@ -1058,7 +1050,7 @@ Gets all vector ids of segment in a collection starting from `offset` and ends w
 ##### Request
 
 ```shell
-$ curl -X GET "http://192.168.1.65:19121/collections/test_collection/segments/1583727470444700000/ids?offset=0&page_size=1" -H "accept: application/json"
+$ curl -X GET "http://127.0.0.1:19121/collections/test_collection/segments/1583727470444700000/ids?offset=0&page_size=1" -H "accept: application/json"
 ```
 
 ##### Response
@@ -1129,7 +1121,7 @@ $ curl -X GET "http://192.168.1.65:19121/collections/test_collection/segments/15
 ##### Request
 
 ```shell
-$ curl -X PUT "http://192.168.1.65:19121/collections/test_collection/vectors" -H "accept: application/json" -H "Content-Type: application/json" -d "{\"topk\":2,\"nprobe\":16,\"records\":[[0.1]]}"
+$ curl -X PUT "http://127.0.0.1:19121/collections/test_collection/vectors" -H "accept: application/json" -H "Content-Type: application/json" -d "{\"topk\":2,\"vectors\":[[0.1]], \"params\":{\"nprobe\":16}}"
 ```
 
 ##### Response
@@ -1182,7 +1174,7 @@ $ curl -X PUT "http://192.168.1.65:19121/collections/test_collection/vectors" -H
 ##### Request
 
 ```shell
-$ curl -X PUT "http://192.168.1.65:19121/collections/test_collection/vectors" -H "accept: application/json" -H "Content-Type: application/json" -d "{"delete": {"ids": ["1578989029645098000"]}}"
+$ curl -X PUT "http://127.0.0.1:19121/collections/test_collection/vectors" -H "accept: application/json" -H "Content-Type: application/json" -d "{"delete": {"ids": ["1578989029645098000"]}}"
 ```
 
 ##### Response
@@ -1205,7 +1197,7 @@ Inserts vectors to a collection.
 Header 
accept: application/json
 
 Body

 {
-  "tag": string,
+  "partition_tag": string,
   "vectors": [[number($float/$uint8)]],
   "ids": [integer($int64)]
 }
@@ -1217,7 +1209,7 @@ Inserts vectors to a collection.
 
 | Parameter  | Description  |  Required? |
 |-----------------|---|------|
-| `tag`     |  Tag of the partition to insert vectors to.   | No   |
+| `partition_tag`     |  Tag of the partition to insert vectors to.   | No   |
 | `vectors`  |  Vectors to insert to the collection.  |  Yes  |
 | `ids`    |  IDs of the vectors to insert to the collection. If you assign IDs to the vectors, you must provide IDs for all vectors in the collection. If you do not specify this parameter, Milvus automatically assigns IDs to the vectors. |  No |
 
@@ -1242,7 +1234,7 @@ Inserts vectors to a collection.
 ##### Request
 
 ```shell
-$ curl -X POST "http://192.168.1.65:19121/collections/test_collection/vectors" -H "accept: application/json" -H "Content-Type: application/json" -d "{\"vectors\":[[0.1],[0.2],[0.3],[0.4]]}"
+$ curl -X POST "http://127.0.0.1:19121/collections/test_collection/vectors" -H "accept: application/json" -H "Content-Type: application/json" -d "{\"vectors\":[[0.1],[0.2],[0.3],[0.4]]}"
 ```
 
 ##### Response
@@ -1285,7 +1277,7 @@ Obtain a vector to by ID.
 ##### Request
 
 ```shell
-$ curl -X POST "http://192.168.1.65:19121/collections/test_collection/vectors?id=1578989029645098000" -H "accept: application/json" -H "Content-Type: application/json"
+$ curl -X POST "http://127.0.0.1:19121/collections/test_collection/vectors?id=1578989029645098000" -H "accept: application/json" -H "Content-Type: application/json"
 ```
 
 ##### Response
@@ -1320,7 +1312,7 @@ Use this API for Cross-Origin Resource Sharing (CORS).
 ##### Request
 
 ```shell
-$ curl -X OPTIONS "http://192.168.1.65:19121/collections/test_collection/vectors"
+$ curl -X OPTIONS "http://127.0.0.1:19121/collections/test_collection/vectors"
 ```
 
 ### `/system/{msg}` (GET)
@@ -1354,7 +1346,7 @@ Gets information about the Milvus server.
 ##### Request
 
 ```shell
-$ curl -X GET "http://192.168.1.65:19121/system/version" -H "accept: application/json"
+$ curl -X GET "http://127.0.0.1:19121/system/version" -H "accept: application/json"
 ```
 
 ##### Response
@@ -1395,7 +1387,7 @@ $ curl -X GET "http://192.168.1.65:19121/system/version" -H "accept: application
 ##### Request
 
 ```shell
-$ curl -X PUT "http://192.168.1.65:19121/system/task" -H "accept: application/json" -d "{\"flush\": {\"collection_names\": [test_collection]}}"
+$ curl -X PUT "http://127.0.0.1:19121/system/task" -H "accept: application/json" -d "{\"flush\": {\"collection_names\": [\"test_collection\"]}}"
 ```
 
 ##### Response
@@ -1434,7 +1426,7 @@ $ curl -X PUT "http://192.168.1.65:19121/system/task" -H "accept: application/js
 ##### Request
 
 ```shell
-$ curl -X PUT "http://192.168.1.65:19121/system/task" -H "accept: application/json" -d "{\"compact\": {\"collection_name\": test_collection}}"
+$ curl -X PUT "http://127.0.0.1:19121/system/task" -H "accept: application/json" -d "{\"compact\": {\"collection_name\": \"test_collection\"}}"
 ```
 
 ##### Response
@@ -1473,7 +1465,7 @@ $ curl -X PUT "http://192.168.1.65:19121/system/task" -H "accept: application/js
 ##### Request
 
 ```shell
-$ curl -X PUT "http://192.168.1.65:19121/system/task" -H "accept: application/json" -d "{\"load\": {\"collection_name\": test_collection}}"
+$ curl -X PUT "http://127.0.0.1:19121/system/task" -H "accept: application/json" -d "{\"load\": {\"collection_name\": \"test_collection\"}}"
 ```
 
 ##### Response

From f811753c62e539998e51b00c5d5891f40c4bfe80 Mon Sep 17 00:00:00 2001
From: Zhiru Zhu 
Date: Mon, 9 Mar 2020 23:33:12 +0800
Subject: [PATCH 45/58] update

Signed-off-by: Zhiru Zhu 
---
 CHANGELOG.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 9f189b728b..289836cfed 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -50,7 +50,7 @@ Please mark all change in change log and use the issue from GitHub
 -   \#1571 Meta engine type become IDMAP after dropping index for BINARY table
 -   \#1574 Set all existing bitset in cache when applying deletes
 -   \#1577 Row count incorrect if delete vectors then create index
--   \#1590 Server down caused by failed to write file during concurrent mixed operations
+-   \#1590 Server down caused by failure to write file during concurrent mixed operations
 
 ## Feature
 -   \#216 Add CLI to get server info

From 83788a3785d3df0968aad8f0553e69c63173a1d1 Mon Sep 17 00:00:00 2001
From: Lutkin Wang 
Date: Tue, 10 Mar 2020 00:31:47 +0800
Subject: [PATCH 46/58] Update README.md

---
 core/src/server/web_impl/README.md | 1110 ++++++++++++++--------------
 1 file changed, 572 insertions(+), 538 deletions(-)

diff --git a/core/src/server/web_impl/README.md b/core/src/server/web_impl/README.md
index dc0db1ab69..6100894df9 100644
--- a/core/src/server/web_impl/README.md
+++ b/core/src/server/web_impl/README.md
@@ -4,38 +4,38 @@
 
 - [Overview](#overview)
 - [API Reference](#api-reference)
-    - [`/state`](#state)
-    - [`/devices`](#devices)
-    - [`/config/advanced` (GET)](#configadvanced-get)
-    - [`/config/advanced` (PUT)](#configadvanced-put)
-    - [`/config/advanced` (OPTIONS)](#configadvanced-options)
-    - [`/config/gpu_resources` (GET)](#configgpu_resources-get)
-    - [`/config/gpu_resources` (PUT)](#configgpu_resources-put)
-    - [`/config/gpu_resources` (OPTIONS)](#configgpu_resources-options)
-    - [`/collections` (GET)](#collections-get)
-    - [`/collections` (POST)](#collections-post)
-    - [`/collections` (OPTIONS)](#collections-options)
-    - [`/collections/{collection_name}` (GET)](#collectionscollection_name-get)
-    - [`/collections/{collection_name}` (DELETE)](#collectionscollection_name-delete)
-    - [`/collections/{collection_name}` (OPTIONS)](#collectionscollection_name-options)
-    - [`/collections/{collection_name}/indexes` (GET)](#collectionscollection_nameindexes-get)
-    - [`/collections/{collection_name}/indexes` (POST)](#collectionscollection_nameindexes-post)
-    - [`/collections/{collection_name}/indexes` (DELETE)](#collectionscollection_nameindexes-delete)
-    - [`/collections/{collection_name}/indexes` (OPTIONS)](#collectionscollection_nameindexes-options)
-    - [`/collections/{collection_name}/partitions` (GET)](#collectionscollection_namepartitions-get)
-    - [`/collections/{collection_name}/partitions` (POST)](#collectionscollection_namepartitions-post)
-    - [`/collections/{collection_name}/partitions` (OPTIONS)](#collectionscollection_namepartitions-options)
-    - [`/collections/{collection_name}/partitions` (DELETE)](#collectionscollection_namepartitions-delete)
-    - [`/collections/{collection_name}/segments` (GET)](#collectionscollection_namesegments-get)
-    - [`/collections/{collection_name}/segments/{segment_name}/vectors` (GET)](#collectionscollection_namesegmentssegment_namevectors-get)
-    - [`/collections/{collection_name}/segments/{segment_name}/ids` (GET)](#collectionscollection_namesegmentssegment_nameids-get)
-    - [`/collections/{collection_name}/vectors` (PUT)](#collectionscollection_namevectors-put)
-    - [`/collections/{collection_name}/vectors` (POST)](#collectionscollection_namevectors-post)
-    - [`/collections/{collection_name}/vectors` (GET)](#collectionscollection_namevectorsidvector_id-get)
-    - [`/collections/{collection_name}/vectors` (OPTIONS)](#collectionscollection_namevectors-options)
-    - [`/system/{msg}` (GET)](#systemmsg-get)
-    - [`system/{op}` (PUT)](#systemop-put)
-- [Error Codes](#error-codes) 
+  - [`/state`](#state)
+  - [`/devices`](#devices)
+  - [`/config/advanced` (GET)](#configadvanced-get)
+  - [`/config/advanced` (PUT)](#configadvanced-put)
+  - [`/config/advanced` (OPTIONS)](#configadvanced-options)
+  - [`/config/gpu_resources` (GET)](#configgpu_resources-get)
+  - [`/config/gpu_resources` (PUT)](#configgpu_resources-put)
+  - [`/config/gpu_resources` (OPTIONS)](#configgpu_resources-options)
+  - [`/collections` (GET)](#collections-get)
+  - [`/collections` (POST)](#collections-post)
+  - [`/collections` (OPTIONS)](#collections-options)
+  - [`/collections/{collection_name}` (GET)](#collectionscollection_name-get)
+  - [`/collections/{collection_name}` (DELETE)](#collectionscollection_name-delete)
+  - [`/collections/{collection_name}` (OPTIONS)](#collectionscollection_name-options)
+  - [`/collections/{collection_name}/indexes` (GET)](#collectionscollection_nameindexes-get)
+  - [`/collections/{collection_name}/indexes` (POST)](#collectionscollection_nameindexes-post)
+  - [`/collections/{collection_name}/indexes` (DELETE)](#collectionscollection_nameindexes-delete)
+  - [`/collections/{collection_name}/indexes` (OPTIONS)](#collectionscollection_nameindexes-options)
+  - [`/collections/{collection_name}/partitions` (GET)](#collectionscollection_namepartitions-get)
+  - [`/collections/{collection_name}/partitions` (POST)](#collectionscollection_namepartitions-post)
+  - [`/collections/{collection_name}/partitions` (OPTIONS)](#collectionscollection_namepartitions-options)
+  - [`/collections/{collection_name}/partitions` (DELETE)](#collectionscollection_namepartitions-delete)
+  - [`/collections/{collection_name}/segments` (GET)](#collectionscollection_namesegments-get)
+  - [`/collections/{collection_name}/segments/{segment_name}/vectors` (GET)](#collectionscollection_namesegmentssegment_namevectors-get)
+  - [`/collections/{collection_name}/segments/{segment_name}/ids` (GET)](#collectionscollection_namesegmentssegment_nameids-get)
+  - [`/collections/{collection_name}/vectors` (PUT)](#collectionscollection_namevectors-put)
+  - [`/collections/{collection_name}/vectors` (POST)](#collectionscollection_namevectors-post)
+  - [`/collections/{collection_name}/vectors` (GET)](#collectionscollection_namevectorsidvector_id-get)
+  - [`/collections/{collection_name}/vectors` (OPTIONS)](#collectionscollection_namevectors-options)
+  - [`/system/{msg}` (GET)](#systemmsg-get)
+  - [`system/{op}` (PUT)](#systemop-put)
+- [Error Codes](#error-codes)
 
 
 
@@ -51,18 +51,18 @@ Checks whether the web server is running.
 
 #### Request
 
-| Request Component     | Value  |
-|-----------------|---|
-| Name     | `/state`  |
-| Header  | `accept: application/json`  |
-| Body    |   N/A |
-| Method    |   GET    |
+| Request Component | Value                      |
+| ----------------- | -------------------------- |
+| Name              | `/state`                   |
+| Header            | `accept: application/json` |
+| Body              | N/A                        |
+| Method            | GET                        |
 
 #### Response
 
-| Status code    | Description |
-|-----------------|---|
-| 200     | The request is successful.|
+| Status code | Description                |
+| ----------- | -------------------------- |
+| 200         | The request is successful. |
 
 #### Example
 
@@ -75,7 +75,7 @@ $ curl -X GET "http://127.0.0.1:19121/state" -H "accept: application/json"
 ##### Response
 
 ```json
-{"message":"Success","code":0}
+{ "message": "Success", "code": 0 }
 ```
 
 ### `/devices`
@@ -84,20 +84,19 @@ Gets CPU/GPU information from the host.
 
 #### Request
 
-| Request Component     | Value  |
-|-----------------|---|
-| Name     | `/devices`  |
-| Header  | `accept: application/json`  |
-| Body    |   N/A |
-| Method    |   GET    |
+| Request Component | Value                      |
+| ----------------- | -------------------------- |
+| Name              | `/devices`                 |
+| Header            | `accept: application/json` |
+| Body              | N/A                        |
+| Method            | GET                        |
 
 #### Response
 
-| Status code    | Description |
-|-----------------|---|
-| 200     | The request is successful.|
-| 400     | The request is incorrect. Refer to the error message for details. |
-
+| Status code | Description                                                       |
+| ----------- | ----------------------------------------------------------------- |
+| 200         | The request is successful.                                        |
+| 400         | The request is incorrect. Refer to the error message for details. |
 
 #### Example
 
@@ -110,7 +109,7 @@ $ curl -X GET "http://127.0.0.1:19121/devices" -H "accept: application/json"
 ##### Response
 
 ```json
-{"cpu":{"memory":31},"gpus":{"GPU0":{"memory":5}}}
+{ "cpu": { "memory": 31 }, "gpus": { "GPU0": { "memory": 5 } } }
 ```
 
 ### `/config/advanced` (GET)
@@ -119,19 +118,19 @@ Gets the values of parameters in `cache_config` and `engine_config` of the Milvu
 
 #### Request
 
-| Request Component     | Value  |
-|-----------------|---|
-| Name     | `/config/advanced`  |
-| Header  | `accept: application/json`  |
-| Body    |   N/A |
-| Method    |   GET |
+| Request Component | Value                      |
+| ----------------- | -------------------------- |
+| Name              | `/config/advanced`         |
+| Header            | `accept: application/json` |
+| Body              | N/A                        |
+| Method            | GET                        |
 
 #### Response
 
-| Status code    | Description |
-|-----------------|---|
-| 200     | The request is successful.|
-| 400     | The request is incorrect. Refer to the error message for details. |
+| Status code | Description                                                       |
+| ----------- | ----------------------------------------------------------------- |
+| 200         | The request is successful.                                        |
+| 400         | The request is incorrect. Refer to the error message for details. |
 
 #### Example
 
@@ -144,7 +143,12 @@ $ curl -X GET "http://127.0.0.1:19121/config/advanced" -H "accept: application/j
 ##### Response
 
 ```json
-{"cpu_cache_capacity":4,"cache_insert_data":false,"use_blas_threshold":1100,"gpu_search_threshold":1000}
+{
+  "cpu_cache_capacity": 4,
+  "cache_insert_data": false,
+  "use_blas_threshold": 1100,
+  "gpu_search_threshold": 1000
+}
 ```
 
 ### `/config/advanced` (PUT)
@@ -173,19 +177,19 @@ Updates the values of parameters in `cache_config` and `engine_config` of the Mi
 
 ##### Body Parameters
 
-| Parameter  | Description  |  Required? |
-|-----------------|---|------|
-| `cpu_cache_capacity`     |  Value of `cpu_cache_capacity` in the Milvus configuration file. The default is 4.| No   |
-| `cache_insert_data`  |  Value of `cache_insert_data` in the Milvus configuration file. The default is false. |  No  |
-| `use_blas_threshold`    |  Value of `use_blas_threshold` in the Milvus configuration file. The default is 1100.   |  No |
-| `gpu_search_threshold`    |  Value of `gpu_search_threshold` in the Milvus configuration file. The default is 1000.     |   No  |
+| Parameter              | Description                                                                            | Required? |
+| ---------------------- | -------------------------------------------------------------------------------------- | --------- |
+| `cpu_cache_capacity`   | Value of `cpu_cache_capacity` in the Milvus configuration file. The default is 4.      | No        |
+| `cache_insert_data`    | Value of `cache_insert_data` in the Milvus configuration file. The default is false.   | No        |
+| `use_blas_threshold`   | Value of `use_blas_threshold` in the Milvus configuration file. The default is 1100.   | No        |
+| `gpu_search_threshold` | Value of `gpu_search_threshold` in the Milvus configuration file. The default is 1000. | No        |
 
 #### Response
 
-| Status code    | Description |
-|-----------------|---|
-| 200     | The request is successful.|
-| 400     | The request is incorrect. Refer to the error message for details. |
+| Status code | Description                                                       |
+| ----------- | ----------------------------------------------------------------- |
+| 200         | The request is successful.                                        |
+| 400         | The request is incorrect. Refer to the error message for details. |
 
 #### Example
 
@@ -197,9 +201,8 @@ $ curl -X PUT "http://127.0.0.1:19121/config/advanced" -H "accept: application/j
 
 ##### Response
 
-
 ```json
-{"message": "OK","code": 0}
+{ "message": "OK", "code": 0 }
 ```
 
 ### `/config/advanced` (OPTIONS)
@@ -208,13 +211,12 @@ Use this API for Cross-Origin Resource Sharing (CORS).
 
 #### Request
 
-| Request Component     | Value  |
-|-----------------|---|
-| Name     | `/config/advanced`  |
-| Header  | N/A  |
-| Body    |   N/A |
-| Method    |   OPTIONS |
-
+| Request Component | Value              |
+| ----------------- | ------------------ |
+| Name              | `/config/advanced` |
+| Header            | N/A                |
+| Body              | N/A                |
+| Method            | OPTIONS            |
 
 #### Example
 
@@ -232,19 +234,19 @@ Gets the parameter values in `gpu_resource_config` of the Milvus configuration f
 
 #### Request
 
-| Request Component     | Value  |
-|-----------------|---|
-| Name     | `/config/gpu_resources`  |
-| Header  | `accept: application/json`  |
-| Body    |   N/A |
-| Method    |   GET |
+| Request Component | Value                      |
+| ----------------- | -------------------------- |
+| Name              | `/config/gpu_resources`    |
+| Header            | `accept: application/json` |
+| Body              | N/A                        |
+| Method            | GET                        |
 
 #### Response
 
-| Status code    | Description |
-|-----------------|---|
-| 200     | The request is successful.|
-| 400     | The request is incorrect. Refer to the error message for details. |
+| Status code | Description                                                       |
+| ----------- | ----------------------------------------------------------------- |
+| 200         | The request is successful.                                        |
+| 400         | The request is incorrect. Refer to the error message for details. |
 
 #### Example
 
@@ -256,9 +258,13 @@ $ curl -X GET "http://127.0.0.1:19121/config/gpu_resources" -H "accept: applicat
 
 ##### Response
 
-
 ```json
-{"enable":true,"cache_capacity":1,"search_resources":["GPU0"],"build_index_resources":["GPU0"]}
+{
+  "enable": true,
+  "cache_capacity": 1,
+  "search_resources": ["GPU0"],
+  "build_index_resources": ["GPU0"]
+}
 ```
 
 ### `/config/gpu_resources` (PUT)
@@ -287,19 +293,19 @@ Updates the parameter values in `gpu_resource_config` of the Milvus configuratio
 
 ##### Body Parameters
 
-| Parameter  | Description  |  Required? |
-|-----------------|---|------|
-| `enable`     |   Specifies whether to enable GPU resources.  | Yes   |
-| `cache_capacity`  | Size of GPU memory per card used for cache in GBs.  |  Yes  | 
-| `search_resources`    |  GPU devices used for search computation, must be in format `gpux`.  |  Yes |
-| `build_index_resources`    |   GPU devices used for index building, must be in format `gpux`.   |   Yes  |
+| Parameter               | Description                                                        | Required? |
+| ----------------------- | ------------------------------------------------------------------ | --------- |
+| `enable`                | Specifies whether to enable GPU resources.                         | Yes       |
+| `cache_capacity`        | Size of GPU memory per card used for cache in GBs.                 | Yes       |
+| `search_resources`      | GPU devices used for search computation, must be in format `gpux`. | Yes       |
+| `build_index_resources` | GPU devices used for index building, must be in format `gpux`.     | Yes       |
 
 #### Response
 
-| Status code    | Description |
-|-----------------|---|
-| 200     | The request is successful.|
-| 400     | The request is incorrect. Refer to the error message for details. |
+| Status code | Description                                                       |
+| ----------- | ----------------------------------------------------------------- |
+| 200         | The request is successful.                                        |
+| 400         | The request is incorrect. Refer to the error message for details. |
 
 #### Example
 
@@ -312,7 +318,7 @@ $ curl -X PUT "http://127.0.0.1:19121/config/gpu_resources" -H "accept: applicat
 ##### Response
 
 ```json
-{"message": "OK","code": 0}
+{ "message": "OK", "code": 0 }
 ```
 
 ### `/config/gpu_resources` (OPTIONS)
@@ -323,12 +329,12 @@ Use this API for Cross-Origin Resource Sharing (CORS).
 
 #### Request
 
-| Request Component     | Value  |
-|-----------------|---|
-| Name     | `/config/gpu_resources`  |
-| Header  | N/A  |
-| Body    |   N/A |
-| Method    |   OPTIONS |
+| Request Component | Value                   |
+| ----------------- | ----------------------- |
+| Name              | `/config/gpu_resources` |
+| Header            | N/A                     |
+| Body              | N/A                     |
+| Method            | OPTIONS                 |
 
 #### Example
 
@@ -344,27 +350,26 @@ Gets all collections starting from `offset` and ends with `page_size`.
 
 #### Request
 
-| Request Component     | Value  |
-|-----------------|---|
-| Name     | `/collections`  |
-| Header  | `accept: application/json`  |
-| Body    |   N/A |
-| Method    |   GET |
+| Request Component | Value                      |
+| ----------------- | -------------------------- |
+| Name              | `/collections`             |
+| Header            | `accept: application/json` |
+| Body              | N/A                        |
+| Method            | GET                        |
 
 ##### Query Parameters
 
-| Parameter  | Description  |  Required? |
-|-----------------|---|------|
-| `offset`     |  Row offset from which the data page starts. The default is 0.    | No   |
-| `page_size`  |  Size of the data page. The default is 10.   |  No  |
-
+| Parameter   | Description                                                   | Required? |
+| ----------- | ------------------------------------------------------------- | --------- |
+| `offset`    | Row offset from which the data page starts. The default is 0. | No        |
+| `page_size` | Size of the data page. The default is 10.                     | No        |
 
 #### Response
 
-| Status code    | Description |
-|-----------------|---|
-| 200     | The request is successful.|
-| 400     | The request is incorrect. Refer to the error message for details. |
+| Status code | Description                                                       |
+| ----------- | ----------------------------------------------------------------- |
+| 200         | The request is successful.                                        |
+| 400         | The request is incorrect. Refer to the error message for details. |
 
 #### Example
 
@@ -376,9 +381,21 @@ $ curl -X GET "http://127.0.0.1:19121/collections?offset=0&page_size=1" -H "acce
 
 ##### Response
 
-
 ```json
-{"collections":[{"collection_name":"test_collection","dimension":1,"index_file_size":10,"metric_type":"L2","count":0,"index":"FLAT","nlist":16384}],"count":58}
+{
+  "collections": [
+    {
+      "collection_name": "test_collection",
+      "dimension": 1,
+      "index_file_size": 10,
+      "metric_type": "L2",
+      "count": 0,
+      "index": "FLAT",
+      "nlist": 16384
+    }
+  ],
+  "count": 58
+}
 ```
 
 ### `/collections` (POST)
@@ -405,19 +422,19 @@ Creates a collection.
 
 ##### Body Parameters
 
-| Parameter  | Description  |  Required? |
-|-----------------|---|------|
-| `collection_name`     |   The name of the collection to create, which must be unique within its database.  | Yes   |
-| `dimension`  |  The dimension of the vectors that are to be inserted into the created collection. |  Yes  |
-| `index_file_size`    |  Threshold value that triggers index building for raw data files. The default is 1024.   |  No |
-| `metric_type`    |   The method vector distances are compared in Milvus. The default is L2. Currently supported metrics include `L2` (Euclidean distance), `IP` (Inner Product), `HAMMING` (Hamming distance), `JACCARD` (Jaccard distance), and `TANIMOTO` (Tanomoto distance).    |   No  |
+| Parameter         | Description                                                                                                                                                                                                                                                 | Required? |
+| ----------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------- |
+| `collection_name` | The name of the collection to create, which must be unique within its database.                                                                                                                                                                             | Yes       |
+| `dimension`       | The dimension of the vectors that are to be inserted into the created collection.                                                                                                                                                                           | Yes       |
+| `index_file_size` | Threshold value that triggers index building for raw data files. The default is 1024.                                                                                                                                                                       | No        |
+| `metric_type`     | The method vector distances are compared in Milvus. The default is L2. Currently supported metrics include `L2` (Euclidean distance), `IP` (Inner Product), `HAMMING` (Hamming distance), `JACCARD` (Jaccard distance), and `TANIMOTO` (Tanomoto distance). | No        |
 
 #### Response
 
-| Status code    | Description |
-|-----------------|---|
-| 201     | Created |
-| 400     | The request is incorrect. Refer to the error message for details. |
+| Status code | Description                                                       |
+| ----------- | ----------------------------------------------------------------- |
+| 201         | Created                                                           |
+| 400         | The request is incorrect. Refer to the error message for details. |
 
 #### Example
 
@@ -429,9 +446,8 @@ $ curl -X POST "http://127.0.0.1:19121/collections" -H "accept: application/json
 
 ##### Response
 
-
 ```json
-{"message":"OK","code":0}
+{ "message": "OK", "code": 0 }
 ```
 
 ### `/collections` (OPTIONS)
@@ -440,13 +456,12 @@ Use this API for Cross-Origin Resource Sharing (CORS).
 
 #### Request
 
-| Request Component     | Value  |
-|-----------------|---|
-| Name     | `/collections`  |
-| Header  | N/A  |
-| Body    |   N/A |
-| Method    |   OPTIONS |
-
+| Request Component | Value          |
+| ----------------- | -------------- |
+| Name              | `/collections` |
+| Header            | N/A            |
+| Body              | N/A            |
+| Method            | OPTIONS        |
 
 #### Example
 
@@ -462,29 +477,27 @@ Gets all information about a collection by name.
 
 #### Request
 
-| Request Component     | Value  |
-|-----------------|---|
-| Name     | `/collections/{collection_name}`  |
-| Header  | `accept: application/json`  |
-| Body    |   N/A |
-| Method    |   GET |
-
+| Request Component | Value                            |
+| ----------------- | -------------------------------- |
+| Name              | `/collections/{collection_name}` |
+| Header            | `accept: application/json`       |
+| Body              | N/A                              |
+| Method            | GET                              |
 
 ##### Query Parameters
 
-| Parameter  | Description  |  Required? |
-|-----------------|---|------|
-| `collection_name`     | Name of the collection.   | Yes   |
-| `info`     | Type of information to acquire. `info` must either be empty or `stat`. When `info` is empty, Milvus returns collection name, dimension, index file size, metric type, offset, index type, and nlist of the collection. When `info` is `stat`, Milvus returns the collection offset, partition status, and segment status.  | No   |
-
+| Parameter         | Description                                                                                                                                                                                                                                                                                                               | Required? |
+| ----------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------- |
+| `collection_name` | Name of the collection.                                                                                                                                                                                                                                                                                                   | Yes       |
+| `info`            | Type of information to acquire. `info` must either be empty or `stat`. When `info` is empty, Milvus returns collection name, dimension, index file size, metric type, offset, index type, and nlist of the collection. When `info` is `stat`, Milvus returns the collection offset, partition status, and segment status. | No        |
 
 #### Response
 
-| Status code    | Description |
-|-----------------|---|
-| 200     | The request is successful.|
-| 400     | The request is incorrect. Refer to the error message for details. |
-| 404     | The required resource does not exist. |
+| Status code | Description                                                       |
+| ----------- | ----------------------------------------------------------------- |
+| 200         | The request is successful.                                        |
+| 400         | The request is incorrect. Refer to the error message for details. |
+| 404         | The required resource does not exist.                             |
 
 #### Example
 
@@ -497,7 +510,15 @@ $ curl -X GET "http://127.0.0.1:19121/collections/test_collection" -H "accept: a
 ##### Response
 
 ```json
-{"collection_name":"test_collection","dimension":1,"index_file_size":10,"metric_type":"L2","count":0,"index":"FLAT", "index_params":{}}
+{
+  "collection_name": "test_collection",
+  "dimension": 1,
+  "index_file_size": 10,
+  "metric_type": "L2",
+  "count": 0,
+  "index": "FLAT",
+  "index_params": {}
+}
 ```
 
 ##### Request
@@ -510,51 +531,55 @@ $ curl -X GET "http://127.0.0.1:19121/collections/test_collection?info=stat" -H
 
 ```json
 {
-   "count":150000,
-   "partitions_stat":[
-      {
-         "count":1000,"partition_tag":"_default",
-         "segments_stat":[
-            {"count":1000,"index":"IVFFLAT","segment_name":"1583727170217439000","size":5284922}
-         ]
-      }
-   ]
+  "count": 150000,
+  "partitions_stat": [
+    {
+      "count": 1000,
+      "partition_tag": "_default",
+      "segments_stat": [
+        {
+          "count": 1000,
+          "index": "IVFFLAT",
+          "segment_name": "1583727170217439000",
+          "size": 5284922
+        }
+      ]
+    }
+  ]
 }
 ```
 
-
 ### `/collections/{collection_name}` (DELETE)
 
 Drops a collection by name.
 
 #### Request
 
-| Request Component     | Value  |
-|-----------------|-----|
-| Name     | `/collections/{collection_name}`  |
-| Header  | `accept: application/json`  |
-| Body    |   N/A |
-| Method    |   DELETE |
+| Request Component | Value                            |
+| ----------------- | -------------------------------- |
+| Name              | `/collections/{collection_name}` |
+| Header            | `accept: application/json`       |
+| Body              | N/A                              |
+| Method            | DELETE                           |
 
 ##### Query Parameters
 
-| Parameter  | Description  |  Required? |
-|-----------------|---|------|
-| `collection_name`     | Name of the collection.   | Yes   |
+| Parameter         | Description             | Required? |
+| ----------------- | ----------------------- | --------- |
+| `collection_name` | Name of the collection. | Yes       |
 
 #### Response
 
-| Status code    | Description |
-|-----------------|---|
-| 204     | Deleted|
-| 400     | The request is incorrect. Refer to the error message for details. |
-| 404     | The required resource does not exist. |
+| Status code | Description                                                       |
+| ----------- | ----------------------------------------------------------------- |
+| 204         | Deleted                                                           |
+| 400         | The request is incorrect. Refer to the error message for details. |
+| 404         | The required resource does not exist.                             |
 
 #### Example
 
 ##### Request
 
-
 ```shell
 $ curl -X DELETE "http://127.0.0.1:19121/collections/test_collection" -H "accept: application/json"
 ```
@@ -567,19 +592,18 @@ Use this API for Cross-Origin Resource Sharing (CORS).
 
 #### Request
 
-| Request Component     | Value  |
-|-----------------|-----|
-| Name     | `/collections/{collection_name}`  |
-| Header  | N/A  |
-| Body    |   N/A |
-| Method    |   OPTIONS |
+| Request Component | Value                            |
+| ----------------- | -------------------------------- |
+| Name              | `/collections/{collection_name}` |
+| Header            | N/A                              |
+| Body              | N/A                              |
+| Method            | OPTIONS                          |
 
 #### Query Parameters
 
-| Parameter  | Description  |  Required? |
-|-----------------|---|------|
-| `collection_name`     | Name of the collection.   | Yes   |
-
+| Parameter         | Description             | Required? |
+| ----------------- | ----------------------- | --------- |
+| `collection_name` | Name of the collection. | Yes       |
 
 #### Example
 
@@ -595,26 +619,26 @@ Gets the index type and nlist of a collection.
 
 #### Request
 
-| Request Component     | Value  |
-|-----------------|-----|
-| Name     | `/collections/{collection_name}/indexes`  |
-| Header  | `accept: application/json`  |
-| Body    |   N/A |
-| Method    |   GET |
+| Request Component | Value                                    |
+| ----------------- | ---------------------------------------- |
+| Name              | `/collections/{collection_name}/indexes` |
+| Header            | `accept: application/json`               |
+| Body              | N/A                                      |
+| Method            | GET                                      |
 
 ##### Query Parameters
 
-| Parameter  | Description  |  Required? |
-|-----------------|---|------|
-| `collection_name`     | Name of the collection.   | Yes   |
+| Parameter         | Description             | Required? |
+| ----------------- | ----------------------- | --------- |
+| `collection_name` | Name of the collection. | Yes       |
 
 #### Response
 
-| Status code    | Description |
-|-----------------|---|
-| 200     | The request is successful.|
-| 400     | The request is incorrect. Refer to the error message for details. |
-| 404     | The required resource does not exist. |
+| Status code | Description                                                       |
+| ----------- | ----------------------------------------------------------------- |
+| 200         | The request is successful.                                        |
+| 400         | The request is incorrect. Refer to the error message for details. |
+| 404         | The required resource does not exist.                             |
 
 #### Example
 
@@ -626,9 +650,8 @@ $ curl -X GET "http://127.0.0.1:19121/collections/test_collection/indexes" -H "a
 
 ##### Response
 
-
 ```json
-{"index_type":"FLAT","params":{"nlist":4096}}
+{ "index_type": "FLAT", "params": { "nlist": 4096 } }
 ```
 
 ### `/collections/{collection_name}/indexes` (POST)
@@ -655,24 +678,24 @@ Updates the index type and nlist of a collection.
 
 ##### Body Parameters
 
-| Parameter  | Description  |  Required? |
-|-----------------|---|------|
-| `index_type`     | The type of indexing method to query the collection. Please refer to [Index Types](https://www.milvus.io/docs/reference/index.md) for detailed introduction of supported indexes. The default is "FLAT".  | No   |
-| `params`     | The extra params of indexing method to query the collection. Please refer to - [Index params](#index_param) for detailed introduction of supported indexes.  | No   |
+| Parameter    | Description                                                                                                                                                                                              | Required? |
+| ------------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------- |
+| `index_type` | The type of indexing method to query the collection. Please refer to [Index Types](https://www.milvus.io/docs/reference/index.md) for detailed introduction of supported indexes. The default is "FLAT". | No        |
+| `params`     | The extra params of indexing method to query the collection. Please refer to - [Index params](#index_param) for detailed introduction of supported indexes.                                              | No        |
 
 ##### Query Parameters
 
-| Parameter  | Description  |  Required? |
-|-----------------|---|------|
-| `collection_name`     | Name of the collection.   | Yes   |
+| Parameter         | Description             | Required? |
+| ----------------- | ----------------------- | --------- |
+| `collection_name` | Name of the collection. | Yes       |
 
 #### Response
 
-| Status code    | Description |
-|-----------------|---|
-| 201     | Created |
-| 400     | The request is incorrect. Refer to the error message for details. |
-| 404     | The required resource does not exist. |
+| Status code | Description                                                       |
+| ----------- | ----------------------------------------------------------------- |
+| 201         | Created                                                           |
+| 400         | The request is incorrect. Refer to the error message for details. |
+| 404         | The required resource does not exist.                             |
 
 #### Example
 
@@ -685,7 +708,7 @@ $ curl -X POST "http://127.0.0.1:19121/collections/test_collection/indexes" -H "
 ##### Response
 
 ```json
-{"message":"OK","code":0}
+{ "message": "OK", "code": 0 }
 ```
 
 ### `/collections/{collection_name}/indexes` (DELETE)
@@ -694,27 +717,26 @@ Drops an index for a collection.
 
 #### Request
 
-| Request Component     | Value  |
-|-----------------|---|
-| Name     | `/collections/{collection_name}/indexes`  |
-| Header  | `accept: application/json`  |
-| Body    |   N/A |
-| Method    |   DELETE |
+| Request Component | Value                                    |
+| ----------------- | ---------------------------------------- |
+| Name              | `/collections/{collection_name}/indexes` |
+| Header            | `accept: application/json`               |
+| Body              | N/A                                      |
+| Method            | DELETE                                   |
 
 ##### Query Parameters
 
-| Parameter  | Description  |  Required? |
-|-----------------|---|------|
-| `collection_name`     | Name of the collection.   | Yes   |
+| Parameter         | Description             | Required? |
+| ----------------- | ----------------------- | --------- |
+| `collection_name` | Name of the collection. | Yes       |
 
 #### Response
 
-| Status code    | Description |
-|-----------------|---|
-| 204     | Deleted |
-| 400     | The request is incorrect. Refer to the error message for details. |
-| 404     | Resource not available |
-
+| Status code | Description                                                       |
+| ----------- | ----------------------------------------------------------------- |
+| 204         | Deleted                                                           |
+| 400         | The request is incorrect. Refer to the error message for details. |
+| 404         | Resource not available                                            |
 
 #### Example
 
@@ -726,25 +748,24 @@ $ curl -X DELETE "http://127.0.0.1:19121/collections/test_collection/indexes" -H
 
 If the deletion is successful, no message will be returned.
 
-
 ### `/collections/{collection_name}/indexes` (OPTIONS)
 
 Use this API for Cross-Origin Resource Sharing (CORS).
 
 #### Request
 
-| Request Component     | Value  |
-|-----------------|---|
-| Name     | `/collections/{collection_name}/indexes`  |
-| Header  | N/A  |
-| Body    |   N/A |
-| Method    |   OPTIONS |
+| Request Component | Value                                    |
+| ----------------- | ---------------------------------------- |
+| Name              | `/collections/{collection_name}/indexes` |
+| Header            | N/A                                      |
+| Body              | N/A                                      |
+| Method            | OPTIONS                                  |
 
 ##### Query Parameters
 
-| Parameter  | Description  |  Required? |
-|-----------------|---|------|
-| `collection_name`     | Name of the collection.   | Yes   |
+| Parameter         | Description             | Required? |
+| ----------------- | ----------------------- | --------- |
+| `collection_name` | Name of the collection. | Yes       |
 
 #### Example
 
@@ -760,28 +781,28 @@ Gets all partitions in a collection starting from `offset` and ends with `page_s
 
 #### Request
 
-| Request Component     | Value  |
-|-----------------|-----------|
-| Name     | `/collections/{collection_name}/partitions`  |
-| Header  | `accept: application/json`  |
-| Body    |   N/A |
-| Method    |   GET |
+| Request Component | Value                                       |
+| ----------------- | ------------------------------------------- |
+| Name              | `/collections/{collection_name}/partitions` |
+| Header            | `accept: application/json`                  |
+| Body              | N/A                                         |
+| Method            | GET                                         |
 
 ##### Query Parameters
 
-| Parameter  | Description  |  Required? |
-|-----------------|---|------|
-| `collection_name`              |        Name of the collection.                  |   Yes     |
-| `offset`     |  Row offset from which the data page starts. The default is 0.    | No   |
-| `page_size`  |  Size of the data page. The default is 10.   |  No  |
+| Parameter         | Description                                                   | Required? |
+| ----------------- | ------------------------------------------------------------- | --------- |
+| `collection_name` | Name of the collection.                                       | Yes       |
+| `offset`          | Row offset from which the data page starts. The default is 0. | No        |
+| `page_size`       | Size of the data page. The default is 10.                     | No        |
 
 #### Response
 
-| Status code    | Description |
-|-----------------|---|
-| 200     | The request is successful.|
-| 400     | The request is incorrect. Refer to the error message for details. |
-| 404     | The required resource does not exist. |
+| Status code | Description                                                       |
+| ----------- | ----------------------------------------------------------------- |
+| 200         | The request is successful.                                        |
+| 400         | The request is incorrect. Refer to the error message for details. |
+| 404         | The required resource does not exist.                             |
 
 #### Example
 
@@ -794,7 +815,14 @@ $ curl -X GET "http://127.0.0.1:19121/collections/test_collection/partitions?off
 ##### Response
 
 ```json
-{"partitions":[{"partition_tag": "_default"},{"partition_tag":"test_tag"},{"partition_tag":"test_2"}], "count":10}
+{
+  "partitions": [
+    { "partition_tag": "_default" },
+    { "partition_tag": "test_tag" },
+    { "partition_tag": "test_2" }
+  ],
+  "count": 10
+}
 ```
 
 ### `/collections/{collection_name}/partitions` (POST)
@@ -803,20 +831,20 @@ Creates a partition in a collection.
 
 #### Request
 
-| Request Component     | Value  |
-|-----------------|-----------|
-| Name     | `/collections/{collection_name}/partitions`  |
-| Header  | `accept: application/json`  |
-| Body    |   N/A |
-| Method    |   POST |
+| Request Component | Value                                       |
+| ----------------- | ------------------------------------------- |
+| Name              | `/collections/{collection_name}/partitions` |
+| Header            | `accept: application/json`                  |
+| Body              | N/A                                         |
+| Method            | POST                                        |
 
 #### Response
 
-| Status code    | Description |
-|-----------------|---|
-| 201     | Created |
-| 400     | The request is incorrect. Refer to the error message for details. |
-| 404     | The required resource does not exist. |
+| Status code | Description                                                       |
+| ----------- | ----------------------------------------------------------------- |
+| 201         | Created                                                           |
+| 400         | The request is incorrect. Refer to the error message for details. |
+| 404         | The required resource does not exist.                             |
 
 #### Example
 
@@ -829,7 +857,7 @@ $ curl -X POST "http://127.0.0.1:19121/collections/test_collection/partitions" -
 ##### Response
 
 ```json
-{"message":"OK","code":0}
+{ "message": "OK", "code": 0 }
 ```
 
 ### `/collections/{collection_name}/partitions` (OPTIONS)
@@ -838,19 +866,18 @@ Use this API for Cross-Origin Resource Sharing (CORS).
 
 #### Request
 
-| Request Component     | Value  |
-|-----------------|---|
-| Name     | `/collections/{collection_name}/partitions`  |
-| Header  | N/A  |
-| Body    |   N/A |
-| Method    |   OPTIONS |
+| Request Component | Value                                       |
+| ----------------- | ------------------------------------------- |
+| Name              | `/collections/{collection_name}/partitions` |
+| Header            | N/A                                         |
+| Body              | N/A                                         |
+| Method            | OPTIONS                                     |
 
 ##### Query Parameters
 
-| Parameter  | Description  |  Required? |
-|-----------------|---|------|
-| `collection_name`              |        Name of the collection.                  |   Yes     |
-
+| Parameter         | Description             | Required? |
+| ----------------- | ----------------------- | --------- |
+| `collection_name` | Name of the collection. | Yes       |
 
 #### Example
 
@@ -881,18 +908,18 @@ Deletes a partition by tag.
 
 ##### Query Parameters
 
-| Parameter  | Description  |  Required? |
-|-----------------|---|------|
-| `collection_name`              |        Name of the collection that contains the partition.               |   Yes     |
-| `partition_tag` |    Tag of the partition to delete.      |   yes |
+| Parameter         | Description                                         | Required? |
+| ----------------- | --------------------------------------------------- | --------- |
+| `collection_name` | Name of the collection that contains the partition. | Yes       |
+| `partition_tag`   | Tag of the partition to delete.                     | yes       |
 
 #### Response
 
-| Status code    | Description |
-|-----------------|---|
-| 204     | Deleted |
-| 400     | The request is incorrect. Refer to the error message for details. |
-| 404     | The requested resource does not exist. |
+| Status code | Description                                                       |
+| ----------- | ----------------------------------------------------------------- |
+| 204         | Deleted                                                           |
+| 400         | The request is incorrect. Refer to the error message for details. |
+| 404         | The requested resource does not exist.                            |
 
 #### Example
 
@@ -904,35 +931,34 @@ $ curl -X DELETE "http://127.0.0.1:19121/collections/test_collection/partitions
 
 The deletion is successful if no information is returned.
 
-
 ### `/collections/{collection_name}/segments` (GET)
 
 Gets all segments in a collection starting from `offset` and ends with `page_size`.
 
 #### Request
 
-| Request Component     | Value  |
-|-----------------|-----------|
-| Name     | `/collections/{collection_name}/segments`  |
-| Header  | `accept: application/json`  |
-| Body    |   N/A |
-| Method    |   GET |
+| Request Component | Value                                     |
+| ----------------- | ----------------------------------------- |
+| Name              | `/collections/{collection_name}/segments` |
+| Header            | `accept: application/json`                |
+| Body              | N/A                                       |
+| Method            | GET                                       |
 
 ##### Query Parameters
 
-| Parameter  | Description  |  Required? |
-|-----------------|---|------|
-| `collection_name`              |        Name of the collection.                  |   Yes     |
-| `offset`     |  Row offset from which the data page starts. The default is 0.    | No   |
-| `page_size`  |  Size of the data page. The default is 10.   |  No  |
+| Parameter         | Description                                                   | Required? |
+| ----------------- | ------------------------------------------------------------- | --------- |
+| `collection_name` | Name of the collection.                                       | Yes       |
+| `offset`          | Row offset from which the data page starts. The default is 0. | No        |
+| `page_size`       | Size of the data page. The default is 10.                     | No        |
 
 #### Response
 
-| Status code    | Description |
-|-----------------|---|
-| 200     | The request is successful.|
-| 400     | The request is incorrect. Refer to the error message for details. |
-| 404     | The required resource does not exist. |
+| Status code | Description                                                       |
+| ----------- | ----------------------------------------------------------------- |
+| 200         | The request is successful.                                        |
+| 400         | The request is incorrect. Refer to the error message for details. |
+| 404         | The required resource does not exist.                             |
 
 #### Example
 
@@ -946,18 +972,18 @@ $ curl -X GET "http://127.0.0.1:19121/collections/test_collection/segments?offse
 
 ```json
 {
-   "code":0,
-   "message":"OK",
-   "count":2,
-   "segments":[
-      {
-         "count":10000,
-         "index":"IVFFLAT",
-         "partition_tag":"_default",
-         "segment_name":"1583727470444700000",
-         "size":5284922
-      }
-   ]
+  "code": 0,
+  "message": "OK",
+  "count": 2,
+  "segments": [
+    {
+      "count": 10000,
+      "index": "IVFFLAT",
+      "partition_tag": "_default",
+      "segment_name": "1583727470444700000",
+      "size": 5284922
+    }
+  ]
 }
 ```
 
@@ -967,29 +993,29 @@ Gets all vectors of segment in a collection starting from `offset` and ends with
 
 #### Request
 
-| Request Component     | Value  |
-|-----------------|-----------|
-| Name     | `/collections/{collection_name}/segments`  |
-| Header  | `accept: application/json`  |
-| Body    |   N/A |
-| Method    |   GET |
+| Request Component | Value                                     |
+| ----------------- | ----------------------------------------- |
+| Name              | `/collections/{collection_name}/segments` |
+| Header            | `accept: application/json`                |
+| Body              | N/A                                       |
+| Method            | GET                                       |
 
 ##### Query Parameters
 
-| Parameter  | Description  |  Required? |
-|-----------------|---|------|
-| `collection_name`              |        Name of the collection.                  |   Yes     |
-| `segment_name`              |        Name of the segment.                  |   Yes     |
-| `offset`     |  Row offset from which the data page starts. The default is 0.    | No   |
-| `page_size`  |  Size of the data page. The default is 10.   |  No  |
+| Parameter         | Description                                                   | Required? |
+| ----------------- | ------------------------------------------------------------- | --------- |
+| `collection_name` | Name of the collection.                                       | Yes       |
+| `segment_name`    | Name of the segment.                                          | Yes       |
+| `offset`          | Row offset from which the data page starts. The default is 0. | No        |
+| `page_size`       | Size of the data page. The default is 10.                     | No        |
 
 #### Response
 
-| Status code    | Description |
-|-----------------|---|
-| 200     | The request is successful.|
-| 400     | The request is incorrect. Refer to the error message for details. |
-| 404     | The required resource does not exist. |
+| Status code | Description                                                       |
+| ----------- | ----------------------------------------------------------------- |
+| 200         | The request is successful.                                        |
+| 400         | The request is incorrect. Refer to the error message for details. |
+| 404         | The required resource does not exist.                             |
 
 #### Example
 
@@ -1003,15 +1029,15 @@ $ curl -X GET "http://127.0.0.1:19121/collections/test_collection/segments/15837
 
 ```json
 {
-   "code":0,
-   "message":"OK",
-   "count":2,
-   "vectors": [
-      {
-         "vector": [0.1], 
-         "id": "1583727470435045000"
-      }
-   ]
+  "code": 0,
+  "message": "OK",
+  "count": 2,
+  "vectors": [
+    {
+      "vector": [0.1],
+      "id": "1583727470435045000"
+    }
+  ]
 }
 ```
 
@@ -1021,29 +1047,29 @@ Gets all vector ids of segment in a collection starting from `offset` and ends w
 
 #### Request
 
-| Request Component     | Value  |
-|-----------------|-----------|
-| Name     | `/collections/{collection_name}/segments`  |
-| Header  | `accept: application/json`  |
-| Body    |   N/A |
-| Method    |   GET |
+| Request Component | Value                                     |
+| ----------------- | ----------------------------------------- |
+| Name              | `/collections/{collection_name}/segments` |
+| Header            | `accept: application/json`                |
+| Body              | N/A                                       |
+| Method            | GET                                       |
 
 ##### Query Parameters
 
-| Parameter  | Description  |  Required? |
-|-----------------|---|------|
-| `collection_name`              |        Name of the collection.                  |   Yes     |
-| `segment_name`              |        Name of the segment.                  |   Yes     |
-| `offset`     |  Row offset from which the data page starts. The default is 0.    | No   |
-| `page_size`  |  Size of the data page. The default is 10.   |  No  |
+| Parameter         | Description                                                   | Required? |
+| ----------------- | ------------------------------------------------------------- | --------- |
+| `collection_name` | Name of the collection.                                       | Yes       |
+| `segment_name`    | Name of the segment.                                          | Yes       |
+| `offset`          | Row offset from which the data page starts. The default is 0. | No        |
+| `page_size`       | Size of the data page. The default is 10.                     | No        |
 
 #### Response
 
-| Status code    | Description |
-|-----------------|---|
-| 200     | The request is successful.|
-| 400     | The request is incorrect. Refer to the error message for details. |
-| 404     | The required resource does not exist. |
+| Status code | Description                                                       |
+| ----------- | ----------------------------------------------------------------- |
+| 200         | The request is successful.                                        |
+| 400         | The request is incorrect. Refer to the error message for details. |
+| 404         | The required resource does not exist.                             |
 
 #### Example
 
@@ -1057,12 +1083,11 @@ $ curl -X GET "http://127.0.0.1:19121/collections/test_collection/segments/15837
 
 ```json
 {
-   "ids": ["1583727470435045000"],
-   "count": 10000
+  "ids": ["1583727470435045000"],
+  "count": 10000
 }
 ```
 
-
 ### `/collections/{collection_name}/vectors` (PUT)
 
 1. Searches vectors in a collection.
@@ -1091,30 +1116,29 @@ $ curl -X GET "http://127.0.0.1:19121/collections/test_collection/segments/15837
 
 ##### Body Parameters
 
-| Parameter  | Description  |  Required? |
-|-----------------|---|------|
-| `topk`     |  The top k most similar results of each query vector.   | Yes   |
-| `tags`    |  Tags of partitions that you need to search. You do not have to specify this value if the collection is not partitioned or you wish to search the whole collection.   |  No |
-| `file_ids`    |  IDs of the vector files. You do not have to specify this value if you do not use Milvus in distributed scenarios. Also, if you assign a value to `file_ids`, the value of `tags` is ignored.    |   No  |
-| `vectors`  |  Vectors to query.  |  Yes  |
-| `params`  |  Extra params for search. Please refer to [Search param](#index_param) to get more detail information.  |  Yes  |
+| Parameter  | Description                                                                                                                                                                                  | Required? |
+| ---------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------- |
+| `topk`     | The top k most similar results of each query vector.                                                                                                                                         | Yes       |
+| `tags`     | Tags of partitions that you need to search. You do not have to specify this value if the collection is not partitioned or you wish to search the whole collection.                           | No        |
+| `file_ids` | IDs of the vector files. You do not have to specify this value if you do not use Milvus in distributed scenarios. Also, if you assign a value to `file_ids`, the value of `tags` is ignored. | No        |
+| `vectors`  | Vectors to query.                                                                                                                                                                            | Yes       |
+| `params`   | Extra params for search. Please refer to [Search param](#index_param) to get more detail information.                                                                                        | Yes       |
 
 > Note: Type of items of vectors depends on the metric used by the collection. If the collection uses `L2` or `IP`, you must use `float`. If the collection uses `HAMMING`, `JACCARD`, or `TANIMOTO`, you must use `uint8`.
 
-
 ##### Query Parameters
 
-| Parameter  | Description  |  Required? |
-|-----------------|---|------|
-| `collection_name` |  Name of the collection.      |   Yes     |
+| Parameter         | Description             | Required? |
+| ----------------- | ----------------------- | --------- |
+| `collection_name` | Name of the collection. | Yes       |
 
 #### Response
 
-| Status code    | Description |
-|-----------------|---|
-| 200     | The request is successful.|
-| 400     | The request is incorrect. Refer to the error message for details. |
-| 404     | The required resource does not exist. |
+| Status code | Description                                                       |
+| ----------- | ----------------------------------------------------------------- |
+| 200         | The request is successful.                                        |
+| 400         | The request is incorrect. Refer to the error message for details. |
+| 404         | The required resource does not exist.                             |
 
 #### Example
 
@@ -1127,7 +1151,15 @@ $ curl -X PUT "http://127.0.0.1:19121/collections/test_collection/vectors" -H "a
 ##### Response
 
 ```json
-{"num":1,"results":[[{"id":"1578989029645098000","distance":"0.000000"},{"id":"1578989029645098001","distance":"0.010000"}]]}
+{
+  "num": 1,
+  "results": [
+    [
+      { "id": "1578989029645098000", "distance": "0.000000" },
+      { "id": "1578989029645098001", "distance": "0.010000" }
+    ]
+  ]
+}
 ```
 
 2. Delete vectors
@@ -1150,24 +1182,23 @@ $ curl -X PUT "http://127.0.0.1:19121/collections/test_collection/vectors" -H "a
 
 ##### Body Parameters
 
-| Parameter  | Description  |  Required? |
-|-----------------|---|------|
-| ids | IDs of vectors. | Yes |
-
+| Parameter | Description     | Required? |
+| --------- | --------------- | --------- |
+| ids       | IDs of vectors. | Yes       |
 
 ##### Query Parameters
 
-| Parameter  | Description  |  Required? |
-|-----------------|---|------|
-| `collection_name` |  Name of the collection.      |   Yes     |
+| Parameter         | Description             | Required? |
+| ----------------- | ----------------------- | --------- |
+| `collection_name` | Name of the collection. | Yes       |
 
 #### Response
 
-| Status code    | Description |
-|-----------------|---|
-| 200     | The request is successful.|
-| 400     | The request is incorrect. Refer to the error message for details. |
-| 404     | The required resource does not exist. |
+| Status code | Description                                                       |
+| ----------- | ----------------------------------------------------------------- |
+| 200         | The request is successful.                                        |
+| 400         | The request is incorrect. Refer to the error message for details. |
+| 404         | The required resource does not exist.                             |
 
 #### Example
 
@@ -1180,7 +1211,7 @@ $ curl -X PUT "http://127.0.0.1:19121/collections/test_collection/vectors" -H "a
 ##### Response
 
 ```json
-{"code": 0, "message": "success"}
+{ "code": 0, "message": "success" }
 ```
 
 ### `/collections/{collection_name}/vectors` (POST)
@@ -1207,27 +1238,27 @@ Inserts vectors to a collection.
 
 ##### Body Parameters
 
-| Parameter  | Description  |  Required? |
-|-----------------|---|------|
-| `partition_tag`     |  Tag of the partition to insert vectors to.   | No   |
-| `vectors`  |  Vectors to insert to the collection.  |  Yes  |
-| `ids`    |  IDs of the vectors to insert to the collection. If you assign IDs to the vectors, you must provide IDs for all vectors in the collection. If you do not specify this parameter, Milvus automatically assigns IDs to the vectors. |  No |
+| Parameter       | Description                                                                                                                                                                                                                      | Required? |
+| --------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------- |
+| `partition_tag` | Tag of the partition to insert vectors to.                                                                                                                                                                                       | No        |
+| `vectors`       | Vectors to insert to the collection.                                                                                                                                                                                             | Yes       |
+| `ids`           | IDs of the vectors to insert to the collection. If you assign IDs to the vectors, you must provide IDs for all vectors in the collection. If you do not specify this parameter, Milvus automatically assigns IDs to the vectors. | No        |
 
 > Note: Type of items of `vectors` depends on the metric used by the collection. If the collection uses `L2` or `IP`, you must use `float`. If the collection uses `HAMMING`, `JACCARD`, or `TANIMOTO`, you must use `uint8`.
 
 ##### Query Parameters
 
-| Parameter  | Description  |  Required? |
-|-----------------|---|------|
-| `collection_name` |  Name of the collection.      |   Yes     |
+| Parameter         | Description             | Required? |
+| ----------------- | ----------------------- | --------- |
+| `collection_name` | Name of the collection. | Yes       |
 
 #### Response
 
-| Status code    | Description |
-|-----------------|---|
-| 201     | Created |
-| 400     | The request is incorrect. Refer to the error message for details. |
-| 404     | The required resource does not exist. |
+| Status code | Description                                                       |
+| ----------- | ----------------------------------------------------------------- |
+| 201         | Created                                                           |
+| 400         | The request is incorrect. Refer to the error message for details. |
+| 404         | The required resource does not exist.                             |
 
 #### Example
 
@@ -1240,7 +1271,14 @@ $ curl -X POST "http://127.0.0.1:19121/collections/test_collection/vectors" -H "
 ##### Response
 
 ```json
-{"ids":["1578989029645098000","1578989029645098001","1578989029645098002","1578989029645098003"]}
+{
+  "ids": [
+    "1578989029645098000",
+    "1578989029645098001",
+    "1578989029645098002",
+    "1578989029645098003"
+  ]
+}
 ```
 
 ### `/collections/{collection_name}/vectors?id={vector_id}` (GET)
@@ -1249,28 +1287,27 @@ Obtain a vector to by ID.
 
 #### Request
 
-| Request Component     | Value  |
-|-----------------|-----------|
-| Name     | `/collections/{collection_name}/vectors`  |
-| Header  | `accept: application/json`  |
-| Body    |   N/A |
-| Method    |   GET |
+| Request Component | Value                                    |
+| ----------------- | ---------------------------------------- |
+| Name              | `/collections/{collection_name}/vectors` |
+| Header            | `accept: application/json`               |
+| Body              | N/A                                      |
+| Method            | GET                                      |
 
 #### Query Parameters
 
-| Parameter  | Description  |  Required? |
-|-----------------|---|------|
-| `collection_name` |  Name of the collection.      |   Yes     |
-| `vector_id` | Vector id.   | Yes   |
-
+| Parameter         | Description             | Required? |
+| ----------------- | ----------------------- | --------- |
+| `collection_name` | Name of the collection. | Yes       |
+| `vector_id`       | Vector id.              | Yes       |
 
 #### Response
 
-| Status code    | Description |
-|-----------------|---|
-| 201     | Created |
-| 400     | The request is incorrect. Refer to the error message for details. |
-| 404     | The required resource does not exist. |
+| Status code | Description                                                       |
+| ----------- | ----------------------------------------------------------------- |
+| 201         | Created                                                           |
+| 400         | The request is incorrect. Refer to the error message for details. |
+| 404         | The required resource does not exist.                             |
 
 #### Example
 
@@ -1284,28 +1321,27 @@ $ curl -X POST "http://127.0.0.1:19121/collections/test_collection/vectors?id=15
 
 ```json
 {
-   "vectors": [
-      {
-         "id": "1578989029645098000",
-         "vector": [0.1]
-      }
-   ]
+  "vectors": [
+    {
+      "id": "1578989029645098000",
+      "vector": [0.1]
+    }
+  ]
 }
 ```
 
-
 ### `/collections/{collection_name}/vectors` (OPTIONS)
 
 Use this API for Cross-Origin Resource Sharing (CORS).
 
 #### Request
 
-| Request Component     | Value  |
-|-----------------|---|
-| Name     | `/collections/{collection_name}/vectors`  |
-| Header  | N/A |
-| Body    |   N/A |
-| Method    |   OPTIONS |
+| Request Component | Value                                    |
+| ----------------- | ---------------------------------------- |
+| Name              | `/collections/{collection_name}/vectors` |
+| Header            | N/A                                      |
+| Body              | N/A                                      |
+| Method            | OPTIONS                                  |
 
 #### Example
 
@@ -1321,25 +1357,25 @@ Gets information about the Milvus server.
 
 #### Request
 
-| Request Component     | Value  |
-|-----------------|---|
-| Name     | `/system/{msg}`  |
-| Header  | `accept: application/json`  |
-| Body    |   N/A |
-| Method    |   GET |
+| Request Component | Value                      |
+| ----------------- | -------------------------- |
+| Name              | `/system/{msg}`            |
+| Header            | `accept: application/json` |
+| Body              | N/A                        |
+| Method            | GET                        |
 
 ##### Query Parameters
 
-| Parameter  | Description  |  Required? |
-|-----------------|---|------|
-| `msg` |  Type of the message to return. You can use `status` or `version`.     |   Yes     |
+| Parameter | Description                                                       | Required? |
+| --------- | ----------------------------------------------------------------- | --------- |
+| `msg`     | Type of the message to return. You can use `status` or `version`. | Yes       |
 
 #### Response
 
-| Status code    | Description |
-|-----------------|---|
-| 200     | The request is successful.|
-| 400     | The request is incorrect. Refer to the error message for details. |
+| Status code | Description                                                       |
+| ----------- | ----------------------------------------------------------------- |
+| 200         | The request is successful.                                        |
+| 400         | The request is incorrect. Refer to the error message for details. |
 
 #### Example
 
@@ -1352,14 +1388,14 @@ $ curl -X GET "http://127.0.0.1:19121/system/version" -H "accept: application/js
 ##### Response
 
 ```json
-{"reply":"0.7.0"}
+{ "reply": "0.7.0" }
 ```
 
 ### `system/{op}` (PUT)
 
-1. Flush collections
+#### Flush a collection
 
-#### Request
+##### Request
 
 
@@ -1375,30 +1411,30 @@ $ curl -X GET "http://127.0.0.1:19121/system/version" -H "accept: application/js
 

 
Request ComponentValue
MethodPUT

 

 
-#### Response
+##### Response
 
-| Status code    | Description |
-|-----------------|---|
-| 200     | The request is successful.|
-| 400     | The request is incorrect. Refer to the error message for details. |
+| Status code | Description                                                       |
+| ----------- | ----------------------------------------------------------------- |
+| 200         | The request is successful.                                        |
+| 400         | The request is incorrect. Refer to the error message for details. |
 
-#### Example
+##### Example
 
-##### Request
+###### Request
 
 ```shell
 $ curl -X PUT "http://127.0.0.1:19121/system/task" -H "accept: application/json" -d "{\"flush\": {\"collection_names\": [\"test_collection\"]}}"
 ```
 
-##### Response
+###### Response
 
 ```json
-{"code": 0, "message": "success"}
+{ "code": 0, "message": "success" }
 ```
 
-2. Compact collection
+#### Compact segments in a collection
 
-#### Request
+##### Request
 
 
@@ -1414,30 +1450,30 @@ $ curl -X PUT "http://127.0.0.1:19121/system/task" -H "accept: application/json"
 

 
Request ComponentValue
MethodPUT

 

 
-#### Response
+##### Response
 
-| Status code    | Description |
-|-----------------|---|
-| 200     | The request is successful.|
-| 400     | The request is incorrect. Refer to the error message for details. |
+| Status code | Description                                                       |
+| ----------- | ----------------------------------------------------------------- |
+| 200         | The request is successful.                                        |
+| 400         | The request is incorrect. Refer to the error message for details. |
 
-#### Example
+##### Example
 
-##### Request
+###### Request
 
 ```shell
 $ curl -X PUT "http://127.0.0.1:19121/system/task" -H "accept: application/json" -d "{\"compact\": {\"collection_name\": \"test_collection\"}}"
 ```
 
-##### Response
+###### Response
 
 ```json
-{"code": 0, "message": "success"}
+{ "code": 0, "message": "success" }
 ```
 
-3. Load collection to memory
+#### Load a collection to memory
 
-#### Request
+##### Request
 
 
@@ -1453,30 +1489,30 @@ $ curl -X PUT "http://127.0.0.1:19121/system/task" -H "accept: application/json"
 

 
Request ComponentValue
MethodPUT

 

 
-#### Response
+##### Response
 
-| Status code    | Description |
-|-----------------|---|
-| 200     | The request is successful.|
-| 400     | The request is incorrect. Refer to the error message for details. |
+| Status code | Description                                                       |
+| ----------- | ----------------------------------------------------------------- |
+| 200         | The request is successful.                                        |
+| 400         | The request is incorrect. Refer to the error message for details. |
 
-#### Example
+##### Example
 
-##### Request
+###### Request
 
 ```shell
 $ curl -X PUT "http://127.0.0.1:19121/system/task" -H "accept: application/json" -d "{\"load\": {\"collection_name\": \"test_collection\"}}"
 ```
 
-##### Response
+###### Response
 
 ```json
-{"code": 0, "message": "success"}
+{ "code": 0, "message": "success" }
 ```
 
-## Index param
+## Index parameters
 
-For each index type, it has specific index param and searach param.
+For each index type, the RESTful API has specific index parameters and search parameters.
 
 
@@ -1507,45 +1543,43 @@ For each index type, it has specific index param and searach param.
 

 
Index typeCreate index paramSearch param

 

 
-You can find more details about parameters above [here](https://github.com/yamasite/docs/blob/v0.7.0/site/zh-CN/guides/index.md)
-
+For detailed information about the parameters above, refer to [Index Types](https://milvus.io/docs/v0.7.0/reference/index.md)
 
 ## Error Codes
 
 The RESTful API returns error messages as JSON text. Each type of error message has a specific error code.
 
-| Type    | Code  |
-|----------|------|
-SUCCESS | 0 |
-UNEXPECTED_ERROR | 1 |
-CONNECT_FAILED | 2 |
-PERMISSION_DENIED | 3 |
-TABLE_NOT_EXISTS | 4 |
-ILLEGAL_ARGUMENT | 5 |
-ILLEGAL_RANGE | 6 |
-ILLEGAL_DIMENSION | 7 |
-ILLEGAL_INDEX_TYPE | 8 |
-ILLEGAL_TABLE_NAME | 9 |
-ILLEGAL_TOPK | 10 |
-ILLEGAL_ROWRECORD | 11 |
-ILLEGAL_VECTOR_ID | 12 |
-ILLEGAL_SEARCH_RESULT | 13 |
-FILE_NOT_FOUND | 14 |
-META_FAILED | 15 |
-CACHE_FAILED | 16 |
-CANNOT_CREATE_FOLDER | 17 |
-CANNOT_CREATE_FILE | 18 |
-CANNOT_DELETE_FOLDER | 19 |
-CANNOT_DELETE_FILE | 20 |
-BUILD_INDEX_ERROR | 21 |
-ILLEGAL_NLIST | 22 |
-ILLEGAL_METRIC_TYPE | 23 |
-OUT_OF_MEMORY | 24 |
-PATH_PARAM_LOSS | 31 |
-UNKNOWN_PATH | 32 |
-QUERY_PARAM_LOSS | 33 |
-BODY_FIELD_LOSS | 34 |
-ILLEGAL_BODY | 35 |
-BODY_PARSE_FAIL | 36 |
-ILLEGAL_QUERY_PARAM | 37 |
-
+| Type                  | Code |
+| --------------------- | ---- |
+| SUCCESS               | 0    |
+| UNEXPECTED_ERROR      | 1    |
+| CONNECT_FAILED        | 2    |
+| PERMISSION_DENIED     | 3    |
+| TABLE_NOT_EXISTS      | 4    |
+| ILLEGAL_ARGUMENT      | 5    |
+| ILLEGAL_RANGE         | 6    |
+| ILLEGAL_DIMENSION     | 7    |
+| ILLEGAL_INDEX_TYPE    | 8    |
+| ILLEGAL_TABLE_NAME    | 9    |
+| ILLEGAL_TOPK          | 10   |
+| ILLEGAL_ROWRECORD     | 11   |
+| ILLEGAL_VECTOR_ID     | 12   |
+| ILLEGAL_SEARCH_RESULT | 13   |
+| FILE_NOT_FOUND        | 14   |
+| META_FAILED           | 15   |
+| CACHE_FAILED          | 16   |
+| CANNOT_CREATE_FOLDER  | 17   |
+| CANNOT_CREATE_FILE    | 18   |
+| CANNOT_DELETE_FOLDER  | 19   |
+| CANNOT_DELETE_FILE    | 20   |
+| BUILD_INDEX_ERROR     | 21   |
+| ILLEGAL_NLIST         | 22   |
+| ILLEGAL_METRIC_TYPE   | 23   |
+| OUT_OF_MEMORY         | 24   |
+| PATH_PARAM_LOSS       | 31   |
+| UNKNOWN_PATH          | 32   |
+| QUERY_PARAM_LOSS      | 33   |
+| BODY_FIELD_LOSS       | 34   |
+| ILLEGAL_BODY          | 35   |
+| BODY_PARSE_FAIL       | 36   |
+| ILLEGAL_QUERY_PARAM   | 37   |

From 66a14e16e88b150bf84dc522fa01eb6013698496 Mon Sep 17 00:00:00 2001
From: JinHai-CN 
Date: Tue, 10 Mar 2020 01:03:32 +0800
Subject: [PATCH 47/58] Update CHANGELOG.md

Signed-off-by: JinHai-CN 
---
 CHANGELOG.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 8e46af40b2..bc790e8a61 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -14,7 +14,7 @@ Please mark all change in change log and use the issue from GitHub
 -   \#990 Check gpu resources setting when assign repeated value
 -   \#995 table count set to 0 if no tables found
 -   \#1010 Improve error message when offset or page_size is equal 0
--   \#1022 check if partition name is legal
+-   \#1022 Check if partition name is valid
 -   \#1028 check if table exists when show partitions
 -   \#1029 check if table exists when try to delete partition
 -   \#1066 optimize http insert and search speed

From d466f3d44753422a4dc1cf4988d3ec09a34b54fc Mon Sep 17 00:00:00 2001
From: JinHai-CN 
Date: Tue, 10 Mar 2020 01:05:07 +0800
Subject: [PATCH 48/58] Update CHANGELOG.md

Signed-off-by: JinHai-CN 
---
 CHANGELOG.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index da7c9bd64c..08b53c5965 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -79,7 +79,7 @@ Please mark all change in change log and use the issue from GitHub
 -   \#738 Use Openblas / lapack from apt install
 -   \#758 Enhance config description
 -   \#791 Remove Arrow
--   \#834 add cpu mode for built-in Faiss
+-   \#834 Add cpu mode for built-in Faiss
 -   \#848 Add ready-to-use config files to the Milvus repo for enhanced user experince
 -   \#860 Remove redundant checks in CacheMgr's constructor
 -   \#908 Move "primary_path" and "secondary_path" to storage config

From ee925c13a6a870adfbfef01ae42bed2119e29b96 Mon Sep 17 00:00:00 2001
From: Zhiru Zhu 
Date: Tue, 10 Mar 2020 09:35:34 +0800
Subject: [PATCH 49/58] update

Signed-off-by: Zhiru Zhu 
---
 core/src/codecs/default/DefaultDeletedDocsFormat.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/core/src/codecs/default/DefaultDeletedDocsFormat.cpp b/core/src/codecs/default/DefaultDeletedDocsFormat.cpp
index 190600af05..0b0039cd4f 100644
--- a/core/src/codecs/default/DefaultDeletedDocsFormat.cpp
+++ b/core/src/codecs/default/DefaultDeletedDocsFormat.cpp
@@ -43,7 +43,7 @@ DefaultDeletedDocsFormat::read(const store::DirectoryPtr& directory_ptr, segment
 
     int del_fd = open(del_file_path.c_str(), O_RDONLY, 00664);
     if (del_fd == -1) {
-        std::string err_msg = "Failed to open file: " + del_file_path;
+        std::string err_msg = "Failed to open file: " + del_file_path + ", error: " + std::strerror(errno);
         ENGINE_LOG_ERROR << err_msg;
         throw Exception(SERVER_CANNOT_CREATE_FILE, err_msg);
     }
@@ -91,7 +91,7 @@ DefaultDeletedDocsFormat::write(const store::DirectoryPtr& directory_ptr, const
     // Write to the temp file, in order to avoid possible race condition with search (concurrent read and write)
     int del_fd = open(temp_path.c_str(), O_RDWR | O_CREAT, 00664);
     if (del_fd == -1) {
-        std::string err_msg = "Failed to open file: " + temp_path;
+        std::string err_msg = "Failed to open file: " + temp_path + ", error: " + std::strerror(errno);
         ENGINE_LOG_ERROR << err_msg;
         throw Exception(SERVER_CANNOT_CREATE_FILE, err_msg);
     }

From 6e8c617c6834075506d1ea16e72c88d40d51cb60 Mon Sep 17 00:00:00 2001
From: groot 
Date: Tue, 10 Mar 2020 12:25:38 +0800
Subject: [PATCH 50/58] typo

Signed-off-by: groot 
---
 core/src/utils/ValidationUtil.cpp |  6 +++---
 sdk/include/MilvusApi.h           | 18 +++++++++---------
 2 files changed, 12 insertions(+), 12 deletions(-)

diff --git a/core/src/utils/ValidationUtil.cpp b/core/src/utils/ValidationUtil.cpp
index 616375f08e..50a6536fd2 100644
--- a/core/src/utils/ValidationUtil.cpp
+++ b/core/src/utils/ValidationUtil.cpp
@@ -174,14 +174,14 @@ ValidationUtil::ValidateIndexParams(const milvus::json& index_params, const engi
         case (int32_t)engine::EngineType::FAISS_IVFSQ8:
         case (int32_t)engine::EngineType::FAISS_IVFSQ8H:
         case (int32_t)engine::EngineType::FAISS_BIN_IVFFLAT: {
-            auto status = CheckParameterRange(index_params, knowhere::IndexParams::nlist, 0, 999999, false);
+            auto status = CheckParameterRange(index_params, knowhere::IndexParams::nlist, 1, 999999);
             if (!status.ok()) {
                 return status;
             }
             break;
         }
         case (int32_t)engine::EngineType::FAISS_PQ: {
-            auto status = CheckParameterRange(index_params, knowhere::IndexParams::nlist, 0, 999999, false);
+            auto status = CheckParameterRange(index_params, knowhere::IndexParams::nlist, 1, 999999);
             if (!status.ok()) {
                 return status;
             }
@@ -254,7 +254,7 @@ ValidationUtil::ValidateSearchParams(const milvus::json& search_params, const en
             break;
         }
         case (int32_t)engine::EngineType::HNSW: {
-            auto status = CheckParameterRange(search_params, knowhere::IndexParams::ef, topk, 1000);
+            auto status = CheckParameterRange(search_params, knowhere::IndexParams::ef, topk, 4096);
             if (!status.ok()) {
                 return status;
             }
diff --git a/sdk/include/MilvusApi.h b/sdk/include/MilvusApi.h
index 743248744c..50f012ff62 100644
--- a/sdk/include/MilvusApi.h
+++ b/sdk/include/MilvusApi.h
@@ -84,19 +84,19 @@ using TopKQueryResult = std::vector;  ///< Topk query result
  * @brief Index parameters
  * Note: extra_params is extra parameters list, it must be json format
  *       For different index type, parameter list is different accordingly, for example:
- *       FLAT/IVFLAT/SQ8:  "{nlist: '16384'}"
+ *       FLAT/IVFLAT/SQ8:  {nlist: 16384}
  *           ///< nlist range:[1, 999999]
- *       IVFPQ:  "{nlist: '16384', m: "12"}"
+ *       IVFPQ:  {nlist: 16384, m: 12}
  *           ///< nlist range:[1, 999999]
  *           ///< m is decided by dim and have a couple of results.
- *       NSG:  "{search_length: '45', out_degree:'50', candidate_pool_size:'300', "knng":'100'}"
+ *       NSG:  {search_length: 45, out_degree:50, candidate_pool_size:300, knng:100}
  *           ///< search_length range:[10, 300]
  *           ///< out_degree range:[5, 300]
  *           ///< candidate_pool_size range:[50, 1000]
  *           ///< knng range:[5, 300]
- *       HNSW  "{M: '16', efConstruction:'500'}"
+ *       HNSW  {M: 16, efConstruction:300}
  *           ///< M range:[5, 48]
- *           ///< efConstruction range:[topk, 4096]
+ *           ///< efConstruction range:[100, 500]
  */
 struct IndexParam {
     std::string collection_name;        ///< Collection name for create index
@@ -386,12 +386,12 @@ class Connection {
      * @param extra_params, extra search parameters according to different index type, must be json format.
      * Note: extra_params is extra parameters list, it must be json format, for example:
      *       For different index type, parameter list is different accordingly
-     *       FLAT/IVFLAT/SQ8/IVFPQ:  "{nprobe: '32'}"
+     *       FLAT/IVFLAT/SQ8/IVFPQ:  {nprobe: 32}
      *           ///< nprobe range:[1,999999]
-     *       NSG:  "{search_length:'100'}
+     *       NSG:  {search_length:100}
      *           ///< search_length range:[10, 300]
-     *       HNSW  "{ef: '64'}
-     *           ///< ef range:[k, 4096]
+     *       HNSW  {ef: 64}
+     *           ///< ef range:[topk, 4096]
      * @param topk_query_result, result array.
      *
      * @return Indicate if query is successful.

From e868437fd56ef608e5a0a6aff463eb7e6e5266b8 Mon Sep 17 00:00:00 2001
From: Yhz 
Date: Tue, 10 Mar 2020 14:05:42 +0800
Subject: [PATCH 51/58] fix link error in http doc

Signed-off-by: Yhz 
---
 core/src/server/web_impl/README.md | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/core/src/server/web_impl/README.md b/core/src/server/web_impl/README.md
index 6100894df9..8d28f44eed 100644
--- a/core/src/server/web_impl/README.md
+++ b/core/src/server/web_impl/README.md
@@ -517,7 +517,7 @@ $ curl -X GET "http://127.0.0.1:19121/collections/test_collection" -H "accept: a
   "metric_type": "L2",
   "count": 0,
   "index": "FLAT",
-  "index_params": {}
+  "index_params": {"nprobe":  16384}
 }
 ```
 
@@ -539,7 +539,7 @@ $ curl -X GET "http://127.0.0.1:19121/collections/test_collection?info=stat" -H
       "segments_stat": [
         {
           "count": 1000,
-          "index": "IVFFLAT",
+          "index": "FLAT",
           "segment_name": "1583727170217439000",
           "size": 5284922
         }
@@ -681,7 +681,7 @@ Updates the index type and nlist of a collection.
 | Parameter    | Description                                                                                                                                                                                              | Required? |
 | ------------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------- |
 | `index_type` | The type of indexing method to query the collection. Please refer to [Index Types](https://www.milvus.io/docs/reference/index.md) for detailed introduction of supported indexes. The default is "FLAT". | No        |
-| `params`     | The extra params of indexing method to query the collection. Please refer to - [Index params](#index_param) for detailed introduction of supported indexes.                                              | No        |
+| `params`     | The extra params of indexing method to query the collection. Please refer to `Index params`(see below) for detailed introduction of supported indexes.                                              | No        |
 
 ##### Query Parameters
 
@@ -1122,7 +1122,7 @@ $ curl -X GET "http://127.0.0.1:19121/collections/test_collection/segments/15837
 | `tags`     | Tags of partitions that you need to search. You do not have to specify this value if the collection is not partitioned or you wish to search the whole collection.                           | No        |
 | `file_ids` | IDs of the vector files. You do not have to specify this value if you do not use Milvus in distributed scenarios. Also, if you assign a value to `file_ids`, the value of `tags` is ignored. | No        |
 | `vectors`  | Vectors to query.                                                                                                                                                                            | Yes       |
-| `params`   | Extra params for search. Please refer to [Search param](#index_param) to get more detail information.                                                                                        | Yes       |
+| `params`   | Extra params for search. Please refer to `Search param`(see below) to get more detail information.                                                                                        | Yes       |
 
 > Note: Type of items of vectors depends on the metric used by the collection. If the collection uses `L2` or `IP`, you must use `float`. If the collection uses `HAMMING`, `JACCARD`, or `TANIMOTO`, you must use `uint8`.
 
@@ -1543,7 +1543,7 @@ For each index type, the RESTful API has specific index parameters and search pa
 
 
 
-For detailed information about the parameters above, refer to [Index Types](https://milvus.io/docs/v0.7.0/reference/index.md)
+For detailed information about the parameters above, refer to [Index Types](https://milvus-io.github.io/milvus-sdk-python/pythondoc/v0.2.8/param.html)
 
 ## Error Codes
 

From 84689fdac5860307e4d6556acac9635c77c51809 Mon Sep 17 00:00:00 2001
From: Yhz 
Date: Tue, 10 Mar 2020 14:09:19 +0800
Subject: [PATCH 52/58] [skip ci] update changlog

Signed-off-by: Yhz 
---
 CHANGELOG.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 1cefe1fb6e..2023622910 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -57,6 +57,7 @@ Please mark all change in change log and use the issue from GitHub
 -   \#1577 Row count incorrect if delete vectors then create index
 -   \#1580 Old segment folder not removed after merge/compact if create_index is called before adding data
 -   \#1590 Server down caused by failure to write file during concurrent mixed operations
+-   \#1601 External link bug in HTTP doc
 
 ## Feature
 -   \#216 Add CLI to get server info

From 1b5d299f58c24bd33481f1f154a7083739024338 Mon Sep 17 00:00:00 2001
From: Yhz 
Date: Tue, 10 Mar 2020 14:12:48 +0800
Subject: [PATCH 53/58] fix show tables results typo

Signed-off-by: Yhz 
---
 core/src/server/web_impl/README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/core/src/server/web_impl/README.md b/core/src/server/web_impl/README.md
index 8d28f44eed..4825446acd 100644
--- a/core/src/server/web_impl/README.md
+++ b/core/src/server/web_impl/README.md
@@ -391,7 +391,7 @@ $ curl -X GET "http://127.0.0.1:19121/collections?offset=0&page_size=1" -H "acce
       "metric_type": "L2",
       "count": 0,
       "index": "FLAT",
-      "nlist": 16384
+      "index_params": {"nlist":  4096}
     }
   ],
   "count": 58

From 487bbfa7581c8b102559254bd4eb0c66790ed601 Mon Sep 17 00:00:00 2001
From: Yhz 
Date: Tue, 10 Mar 2020 14:35:16 +0800
Subject: [PATCH 54/58] change index type link address

Signed-off-by: Yhz 
---
 core/src/server/web_impl/README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/core/src/server/web_impl/README.md b/core/src/server/web_impl/README.md
index 4825446acd..8b5d786abb 100644
--- a/core/src/server/web_impl/README.md
+++ b/core/src/server/web_impl/README.md
@@ -1543,7 +1543,7 @@ For each index type, the RESTful API has specific index parameters and search pa
 
 
 
-For detailed information about the parameters above, refer to [Index Types](https://milvus-io.github.io/milvus-sdk-python/pythondoc/v0.2.8/param.html)
+For detailed information about the parameters above, refer to [Index Types](https://milvus.io/docs/v0.7.0/reference/index.md)
 
 ## Error Codes
 

From 596d41b111c34d305ad01f9925d21877217a00dc Mon Sep 17 00:00:00 2001
From: Yhz 
Date: Tue, 10 Mar 2020 14:45:11 +0800
Subject: [PATCH 55/58] fix link error in http doc (fix #1601)

Signed-off-by: Yhz 
---
 core/src/server/web_impl/README.md | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/core/src/server/web_impl/README.md b/core/src/server/web_impl/README.md
index 8b5d786abb..3abe5a462c 100644
--- a/core/src/server/web_impl/README.md
+++ b/core/src/server/web_impl/README.md
@@ -681,7 +681,7 @@ Updates the index type and nlist of a collection.
 | Parameter    | Description                                                                                                                                                                                              | Required? |
 | ------------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------- |
 | `index_type` | The type of indexing method to query the collection. Please refer to [Index Types](https://www.milvus.io/docs/reference/index.md) for detailed introduction of supported indexes. The default is "FLAT". | No        |
-| `params`     | The extra params of indexing method to query the collection. Please refer to `Index params`(see below) for detailed introduction of supported indexes.                                              | No        |
+| `params`     | The extra params of indexing method to query the collection. Please refer to [Index and search parameters](#Index-and-search-parameters) for detailed introduction of supported indexes.                                              | No        |
 
 ##### Query Parameters
 
@@ -1122,7 +1122,7 @@ $ curl -X GET "http://127.0.0.1:19121/collections/test_collection/segments/15837
 | `tags`     | Tags of partitions that you need to search. You do not have to specify this value if the collection is not partitioned or you wish to search the whole collection.                           | No        |
 | `file_ids` | IDs of the vector files. You do not have to specify this value if you do not use Milvus in distributed scenarios. Also, if you assign a value to `file_ids`, the value of `tags` is ignored. | No        |
 | `vectors`  | Vectors to query.                                                                                                                                                                            | Yes       |
-| `params`   | Extra params for search. Please refer to `Search param`(see below) to get more detail information.                                                                                        | Yes       |
+| `params`   | Extra params for search. Please refer to [Index and search parameters](#Index-and-search-parameters) to get more detail information.                                                                                        | Yes       |
 
 > Note: Type of items of vectors depends on the metric used by the collection. If the collection uses `L2` or `IP`, you must use `float`. If the collection uses `HAMMING`, `JACCARD`, or `TANIMOTO`, you must use `uint8`.
 
@@ -1510,7 +1510,7 @@ $ curl -X PUT "http://127.0.0.1:19121/system/task" -H "accept: application/json"
 { "code": 0, "message": "success" }
 ```
 
-## Index parameters
+## Index and search parameters
 
 For each index type, the RESTful API has specific index parameters and search parameters.
 

From f722119188f97830c6f32e355ad0a0e27d3317c4 Mon Sep 17 00:00:00 2001
From: Yhz 
Date: Tue, 10 Mar 2020 15:59:08 +0800
Subject: [PATCH 56/58] add options methods for system

Signed-off-by: Yhz 
---
 core/src/server/web_impl/controller/WebController.hpp | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/core/src/server/web_impl/controller/WebController.hpp b/core/src/server/web_impl/controller/WebController.hpp
index 55a51168a3..f326578405 100644
--- a/core/src/server/web_impl/controller/WebController.hpp
+++ b/core/src/server/web_impl/controller/WebController.hpp
@@ -617,10 +617,7 @@ class WebController : public oatpp::web::server::api::ApiController {
     }
 
     ADD_CORS(VectorsOp)
-    /*************
-     * Search
-     * Delete by ID
-     * */
+
     ENDPOINT("PUT", "/collections/{collection_name}/vectors", VectorsOp,
              PATH(String, collection_name), BODY_STRING(String, body)) {
         TimeRecorder tr(std::string(WEB_LOG_PREFIX) + "PUT \'/collections/" + collection_name->std_str() + "/vectors\'");
@@ -648,6 +645,12 @@ class WebController : public oatpp::web::server::api::ApiController {
         return response;
     }
 
+    ADD_CORS(SystemOptions)
+
+    ENDPOINT("OPTIONS", "/system/{info}", SystemOptions) {
+        return createResponse(Status::CODE_204, "No Content");
+    }
+
     ADD_CORS(SystemInfo)
 
     ENDPOINT("GET", "/system/{info}", SystemInfo, PATH(String, info), QUERIES(const QueryParams&, query_params)) {

From 4625bf37103c27dfbd121474c5b787558e453d40 Mon Sep 17 00:00:00 2001
From: Zhiru Zhu 
Date: Tue, 10 Mar 2020 16:40:17 +0800
Subject: [PATCH 57/58] update

Signed-off-by: Zhiru Zhu 
---
 CHANGELOG.md                           |  1 +
 core/src/scheduler/task/SearchTask.cpp | 13 ++++++++-----
 2 files changed, 9 insertions(+), 5 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 1cefe1fb6e..d906efab83 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -57,6 +57,7 @@ Please mark all change in change log and use the issue from GitHub
 -   \#1577 Row count incorrect if delete vectors then create index
 -   \#1580 Old segment folder not removed after merge/compact if create_index is called before adding data
 -   \#1590 Server down caused by failure to write file during concurrent mixed operations
+-   \#1598 Server down during mixed operations
 
 ## Feature
 -   \#216 Add CLI to get server info
diff --git a/core/src/scheduler/task/SearchTask.cpp b/core/src/scheduler/task/SearchTask.cpp
index 49b225eea8..cd8268cfa3 100644
--- a/core/src/scheduler/task/SearchTask.cpp
+++ b/core/src/scheduler/task/SearchTask.cpp
@@ -264,13 +264,16 @@ XSearchTask::Execute() {
 
             // step 3: pick up topk result
             auto spec_k = file_->row_count_ < topk ? file_->row_count_ : topk;
-            if (search_job->GetResultIds().front() == -1 && search_job->GetResultIds().size() > spec_k) {
-                // initialized results set
-                search_job->GetResultIds().resize(spec_k);
-                search_job->GetResultDistances().resize(spec_k);
-            }
+
             {
                 std::unique_lock lock(search_job->mutex());
+
+                if (search_job->GetResultIds().front() == -1 && search_job->GetResultIds().size() > spec_k) {
+                    // initialized results set
+                    search_job->GetResultIds().resize(spec_k);
+                    search_job->GetResultDistances().resize(spec_k);
+                }
+
                 XSearchTask::MergeTopkToResultSet(output_ids, output_distance, spec_k, nq, topk, ascending_reduce,
                                                   search_job->GetResultIds(), search_job->GetResultDistances());
             }

From 9d4b7fda62069dabacd8bca75fa9aa4549967954 Mon Sep 17 00:00:00 2001
From: Zhiru Zhu 
Date: Tue, 10 Mar 2020 16:40:53 +0800
Subject: [PATCH 58/58] update

Signed-off-by: Zhiru Zhu 
---
 core/src/scheduler/task/SearchTask.cpp | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/core/src/scheduler/task/SearchTask.cpp b/core/src/scheduler/task/SearchTask.cpp
index cd8268cfa3..5e71b803e1 100644
--- a/core/src/scheduler/task/SearchTask.cpp
+++ b/core/src/scheduler/task/SearchTask.cpp
@@ -9,7 +9,10 @@
 // is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
 // or implied. See the License for the specific language governing permissions and limitations under the License.
 
+#include "scheduler/task/SearchTask.h"
+
 #include 
+
 #include 
 #include 
 #include 
@@ -21,7 +24,6 @@
 #include "metrics/Metrics.h"
 #include "scheduler/SchedInst.h"
 #include "scheduler/job/SearchJob.h"
-#include "scheduler/task/SearchTask.h"
 #include "segment/SegmentReader.h"
 #include "utils/Log.h"
 #include "utils/TimeRecorder.h"