admin/milvus
1
0
Fork 0
mirror of https://gitee.com/milvus-io/milvus.git synced 2026-02-04 11:18:44 +08:00
Code Issues Packages Projects Releases Wiki Activity

Add WebServer unittest (#3321)

Browse Source 
* Add web server interface

Signed-off-by: fishpenguin <kun.yu@zilliz.com>

* Add unittest/server

Signed-off-by: fishpenguin <kun.yu@zilliz.com>

* Add web server ut

Signed-off-by: fishpenguin <kun.yu@zilliz.com>
This commit is contained in:
yukun 2020-08-19 18:52:36 +08:00 committed by GitHub
parent 729c6ebdc5
commit 68a2918c30
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
13 changed files with 963 additions and 56 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
core/src/CMakeLists.txt
View File

@ -62,10 +62,10 @@ set(link_lib
milvus_engine
config
metrics
log
oatpp
query
utils
log
)

2
core/src/grpc/milvus.proto
View File

@ -294,7 +294,7 @@ message Command {
* @index_name: a name for index provided by user, unique within this field
* @extra_params: index parameters in json format
* for vector field:
* extra_params["index_type"] = one of the values: IDMAP, IVFLAT, IVFSQ8, NSGMIX, IVFSQ8H,
* extra_params["index_type"] = one of the values: FLAT, IVF_LAT, IVF_SQ8, NSGMIX, IVFSQ8H,
* PQ, HNSW, HNSW_SQ8NM, ANNOY
* extra_params["metric_type"] = one of the values: L2, IP, HAMMING, JACCARD, TANIMOTO
* SUBSTRUCTURE, SUPERSTRUCTURE

16
core/src/server/web_impl/Constants.cpp
View File

@ -16,16 +16,16 @@ namespace server {
namespace web {
const char* NAME_ENGINE_TYPE_FLAT = "FLAT";
const char* NAME_ENGINE_TYPE_IVFFLAT = "IVFFLAT";
const char* NAME_ENGINE_TYPE_IVFSQ8 = "IVFSQ8";
const char* NAME_ENGINE_TYPE_IVFSQ8H = "IVFSQ8H";
const char* NAME_ENGINE_TYPE_IVFFLAT = "IVF_FLAT";
const char* NAME_ENGINE_TYPE_IVFSQ8 = "IVF_SQ8";
const char* NAME_ENGINE_TYPE_IVFSQ8H = "IVF_SQ8H";
const char* NAME_ENGINE_TYPE_RNSG = "RNSG";
const char* NAME_ENGINE_TYPE_IVFPQ = "IVFPQ";
const char* NAME_ENGINE_TYPE_IVFPQ = "IVF_PQ";
const char* NAME_ENGINE_TYPE_HNSW = "HNSW";
const char* NAME_ENGINE_TYPE_ANNOY = "ANNOY";
const char* NAME_ENGINE_TYPE_RHNSWFLAT = "RHNSWFLAT";
const char* NAME_ENGINE_TYPE_RHNSWPQ = "RHNSWPQ";
const char* NAME_ENGINE_TYPE_RHNSWSQ8 = "RHNSWSQ8";
const char* NAME_ENGINE_TYPE_RHNSWFLAT = "RHNSW_FLAT";
const char* NAME_ENGINE_TYPE_RHNSWPQ = "RHNSW_PQ";
const char* NAME_ENGINE_TYPE_RHNSWSQ8 = "RHNSW_SQ8";
const char* NAME_METRIC_TYPE_L2 = "L2";
const char* NAME_METRIC_TYPE_IP = "IP";
@ -47,6 +47,8 @@ const int64_t VALUE_INDEX_NLIST_DEFAULT = 16384;
const int64_t VALUE_CONFIG_CPU_CACHE_CAPACITY_DEFAULT = 4;
const bool VALUE_CONFIG_CACHE_INSERT_DATA_DEFAULT = false;
const char* NAME_ID = "__id";
/////////////////////////////////////////////////////
} // namespace web

1
core/src/server/web_impl/Constants.h
View File

@ -46,6 +46,7 @@ extern const char* VALUE_INDEX_INDEX_TYPE_DEFAULT;
extern const int64_t VALUE_INDEX_NLIST_DEFAULT;
extern const int64_t VALUE_CONFIG_CPU_CACHE_CAPACITY_DEFAULT;
extern const bool VALUE_CONFIG_CACHE_INSERT_DATA_DEFAULT;
extern const char* NAME_ID;
/////////////////////////////////////////////////////

5
core/src/server/web_impl/README.md
View File

@ -676,9 +676,10 @@ Updates the index type and nlist of a collection.
<tr><td>Header </td><td><pre><code>accept: application/json</code></pre> </td></tr>
<tr><td>Body</td><td><pre><code>
{
"index_type": string,
"index_type": "IVF_FLAT",
"metric_type": "IP",
"params": {
......
"nlist": 1024
}
}
</code></pre> </td></tr>

7
core/src/server/web_impl/Types.h
View File

@ -80,6 +80,13 @@ static std::map<std::string, engine::DataType> str2type = {{"int32", engine::Dat
{"vector_float", engine::DataType::VECTOR_FLOAT},
{"vector_binary", engine::DataType::VECTOR_BINARY}};
static std::map<engine::DataType, std::string> type2str = {{engine::DataType::INT32, "int32"},
{engine::DataType::INT64, "int64"},
{engine::DataType::FLOAT, "float"},
{engine::DataType::DOUBLE, "double"},
{engine::DataType::VECTOR_FLOAT, "vector_float"},
{engine::DataType::VECTOR_BINARY, "vector_binary"}};
} // namespace web
} // namespace server
} // namespace milvus

41
core/src/server/web_impl/controller/WebController.hpp
View File

@ -33,8 +33,7 @@ namespace milvus::server::web {
#define WEB_LOG_PREFIX "[Web] "
#define ADD_DEFAULT_CORS(endpoint) \
ADD_CORS(endpoint, "*", "OPTIONS, GET, POST, PUT, DELETE")
#define ADD_DEFAULT_CORS(endpoint) ADD_CORS(endpoint, "*", "OPTIONS, GET, POST, PUT, DELETE")
class WebController : public oatpp::web::server::api::ApiController {
public:
@ -280,7 +279,6 @@ class WebController : public oatpp::web::server::api::ApiController {
// "count": 58
// })";
response = createResponse(Status::CODE_200, result);
return response;
}
@ -364,16 +362,16 @@ class WebController : public oatpp::web::server::api::ApiController {
ADD_DEFAULT_CORS(IndexOptions)
ENDPOINT("OPTIONS", "/collections/{collection_name}/indexes", IndexOptions) {
ENDPOINT("OPTIONS", "/collections/{collection_name}/fields/{field_name}/indexes", IndexOptions) {
return createResponse(Status::CODE_204, "No Content");
}
ADD_DEFAULT_CORS(CreateIndex)
ENDPOINT("POST", "/collections/{collection_name}/fields/{field_name}/indexes/{index_name}", CreateIndex,
PATH(String, collection_name), PATH(String, field_name), PATH(String, index_name),
BODY_STRING(String, body)) {
TimeRecorder tr(std::string(WEB_LOG_PREFIX) + "POST \'/tables/" + collection_name->std_str() + "/indexes\'");
ENDPOINT("POST", "/collections/{collection_name}/fields/{field_name}/indexes", CreateIndex,
PATH(String, collection_name), PATH(String, field_name), BODY_STRING(String, body)) {
TimeRecorder tr(std::string(WEB_LOG_PREFIX) + "POST \'/collections/" + collection_name->std_str() +
"/indexes\'");
tr.RecordSection("Received request.");
auto handler = WebRequestHandler();
@ -432,10 +430,10 @@ class WebController : public oatpp::web::server::api::ApiController {
ADD_DEFAULT_CORS(DropIndex)
ENDPOINT("DELETE", "/collections/{collection_name}/fields/{field_name}/indexes/{index_name}", DropIndex,
PATH(String, collection_name), PATH(String, field_name), PATH(String, index_name)) {
ENDPOINT("DELETE", "/collections/{collection_name}/fields/{field_name}/indexes", DropIndex,
PATH(String, collection_name), PATH(String, field_name)) {
TimeRecorder tr(std::string(WEB_LOG_PREFIX) + "DELETE \'/collections/" + collection_name->std_str() +
"/indexes\'");
"/fields/" + field_name->std_str() + "/indexes\'");
tr.RecordSection("Received request.");
auto handler = WebRequestHandler();
@ -559,12 +557,12 @@ class WebController : public oatpp::web::server::api::ApiController {
ADD_DEFAULT_CORS(GetEntities)
ENDPOINT("GET", "/collections/{collection_name}/partitions/{partition_tag}/entities", GetEntities,
PATH(String, collection_name), PATH(String, partition_tag), QUERIES(QueryParams, query_params),
BODY_STRING(String, body)) {
ENDPOINT("GET", "/collections/{collection_name}/entities", GetEntities, PATH(String, collection_name),
QUERIES(QueryParams, query_params)) {
auto handler = WebRequestHandler();
String response;
String response;
auto status_dto = handler.GetEntity(collection_name, query_params, response);
switch (*(status_dto->code)) {
case StatusCode::SUCCESS:
@ -622,21 +620,21 @@ class WebController : public oatpp::web::server::api::ApiController {
}
}
ADD_DEFAULT_CORS(VectorsOptions)
ADD_CORS(EntityOptions)
ENDPOINT("OPTIONS", "/collections/{collection_name}/entities", VectorsOptions) {
ENDPOINT("OPTIONS", "/collections/{collection_name}/entities", EntityOptions) {
return createResponse(Status::CODE_204, "No Content");
}
ADD_DEFAULT_CORS(InsertEntity)
ADD_DEFAULT_CORS(Insert)
ENDPOINT("POST", "/hybrid_collections/{collection_name}/entities", InsertEntity, PATH(String, collection_name),
ENDPOINT("POST", "/collections/{collection_name}/entities", Insert, PATH(String, collection_name),
BODY_STRING(String, body)) {
TimeRecorder tr(std::string(WEB_LOG_PREFIX) + "POST \'/hybrid_collections/" + collection_name->std_str() +
TimeRecorder tr(std::string(WEB_LOG_PREFIX) + "POST \'/collections/" + collection_name->std_str() +
"/entities\'");
tr.RecordSection("Received request.");
auto ids_dto = VectorIdsDto::createShared();
auto ids_dto = EntityIdsDto::createShared();
WebRequestHandler handler = WebRequestHandler();
std::shared_ptr<OutgoingResponse> response;
@ -751,4 +749,3 @@ class WebController : public oatpp::web::server::api::ApiController {
};
} // namespace milvus::server::web

6
core/src/server/web_impl/dto/VectorDto.hpp
View File

@ -18,14 +18,14 @@ namespace milvus::server::web {
#include OATPP_CODEGEN_BEGIN(DTO)
class VectorIdsDto : public ODTO {
DTO_INIT(VectorIdsDto, DTO)
class EntityIdsDto : public ODTO {
DTO_INIT(EntityIdsDto, DTO)
DTO_FIELD(List<String>, ids);
};
#include OATPP_CODEGEN_END(DTO)
using VectorIdsDtoT = oatpp::Object<VectorIdsDto>;
using EntityIdsDtoT = oatpp::Object<EntityIdsDto>;
} // namespace milvus::server::web

167
core/src/server/web_impl/handler/WebRequestHandler.cpp
View File

@ -97,6 +97,43 @@ CopyStructuredData(const nlohmann::json& json, std::vector<uint8_t>& raw) {
memcpy(raw.data(), values.data(), size * sizeof(T));
}
void
CopyRowVectorFromJson(const nlohmann::json& json, std::vector<uint8_t>& vectors_data, bool bin) {
// if (!json.is_array()) {
// return Status(ILLEGAL_BODY, "field \"vectors\" must be a array");
// }
std::vector<float> float_vector;
if (!bin) {
for (auto& data : json) {
float_vector.emplace_back(data.get<float>());
}
auto size = float_vector.size() * sizeof(float);
vectors_data.resize(size);
memcpy(vectors_data.data(), float_vector.data(), size);
} else {
for (auto& data : json) {
vectors_data.emplace_back(data.get<uint8_t>());
}
}
}
template <typename T>
void
CopyRowStructuredData(const nlohmann::json& entity_json, const std::string& field_name, const int64_t offset,
const int64_t row_num, std::unordered_map<std::string, std::vector<uint8_t>>& chunk_data) {
T value = entity_json.get<T>();
std::vector<uint8_t> temp_data(sizeof(T), 0);
memcpy(temp_data.data(), &value, sizeof(T));
if (chunk_data.find(field_name) == chunk_data.end()) {
std::vector<uint8_t> T_data(row_num * sizeof(T), 0);
memcpy(T_data.data(), temp_data.data(), sizeof(T));
chunk_data.insert({field_name, T_data});
} else {
int64_t T_offset = offset * sizeof(T);
memcpy(chunk_data.at(field_name).data() + T_offset, temp_data.data(), sizeof(T));
}
}
using FloatJson = nlohmann::basic_json<std::map, std::vector, std::string, bool, std::int64_t, std::uint64_t, float>;
/////////////////////////////////// Private methods ///////////////////////////////////////
@ -236,12 +273,15 @@ WebRequestHandler::GetCollectionMetaInfo(const std::string& collection_name, nlo
json_out["collection_name"] = schema.collection_name_;
for (const auto& field : schema.fields_) {
if (field.first == engine::FIELD_UID) {
continue;
}
nlohmann::json field_json;
field_json["field_name"] = field.first;
field_json["field_type"] = field.second.field_type_;
field_json["field_type"] = type2str.at(field.second.field_type_);
field_json["index_params"] = field.second.index_params_;
field_json["extra_params"] = field.second.field_params_;
json_out["field"].push_back(field_json);
json_out["fields"].push_back(field_json);
}
return Status::OK();
}
@ -548,9 +588,16 @@ WebRequestHandler::ProcessLeafQueryJson(const nlohmann::json& json, milvus::quer
}
engine::VectorsData vector_data;
for (auto& vector_records : vector_param_it.value()["values"]) {
// TODO: Binary vector???
for (auto& data : vector_records) {
vector_query->query_vector.float_data.emplace_back(data.get<float>());
if (field_type_.find(vector_name) != field_type_.end()) {
if (field_type_.at(vector_name) == engine::DataType::VECTOR_FLOAT) {
for (auto& data : vector_records) {
vector_query->query_vector.float_data.emplace_back(data.get<float>());
}
} else if (field_type_.at(vector_name) == engine::DataType::VECTOR_BINARY) {
for (auto& data : vector_records) {
vector_query->query_vector.binary_data.emplace_back(data.get<int8_t>());
}
}
}
}
query_ptr->index_fields.insert(vector_name);
@ -653,6 +700,9 @@ WebRequestHandler::Search(const std::string& collection_name, const nlohmann::js
if (!status.ok()) {
return Status{UNEXPECTED_ERROR, "DescribeHybridCollection failed"};
}
for (const auto& field : collection_schema.fields_) {
field_type_.insert({field.first, field.second.field_type_});
}
milvus::json extra_params;
if (json.contains("fields")) {
@ -1136,6 +1186,12 @@ WebRequestHandler::CreateCollection(const milvus::server::web::OString& body) {
}
milvus::json json_params;
if (json_str.contains(engine::PARAM_SEGMENT_ROW_COUNT)) {
json_params[engine::PARAM_SEGMENT_ROW_COUNT] = json_str[engine::PARAM_SEGMENT_ROW_COUNT];
}
if (json_str.contains(engine::PARAM_UID_AUTOGEN)) {
json_params[engine::PARAM_UID_AUTOGEN] = json_str[engine::PARAM_UID_AUTOGEN];
}
auto status = req_handler_.CreateCollection(context_ptr_, collection_name, fields, json_params);
@ -1198,8 +1254,8 @@ WebRequestHandler::ShowCollections(const OQueryParams& query_params, OString& re
result_json["collections"] = collections_json;
}
AddStatusToJson(result_json, status.code(), status.message());
result = result_json.dump().c_str();
ASSIGN_RETURN_STATUS_DTO(status)
}
@ -1474,36 +1530,107 @@ WebRequestHandler::GetSegmentInfo(const OString& collection_name, const OString&
*/
StatusDtoT
WebRequestHandler::InsertEntity(const OString& collection_name, const milvus::server::web::OString& body,
VectorIdsDtoT& ids_dto) {
EntityIdsDtoT& ids_dto) {
if (nullptr == body.get() || body->getSize() == 0) {
RETURN_STATUS_DTO(BODY_FIELD_LOSS, "Request payload is required.")
}
auto body_json = nlohmann::json::parse(body->c_str());
std::string partition_name = body_json["partition_tag"];
int32_t row_num = body_json["row_num"];
std::string partition_name;
if (body_json.contains("partition_tag")) {
partition_name = body_json["partition_tag"];
}
CollectionSchema collection_schema;
std::unordered_map<std::string, engine::DataType> field_types;
auto status = req_handler_.GetCollectionInfo(context_ptr_, collection_name->std_str(), collection_schema);
if (!status.ok()) {
auto msg = "Collection " + collection_name->std_str() + " not exist";
RETURN_STATUS_DTO(COLLECTION_NOT_EXISTS, msg.c_str());
}
for (const auto& field : collection_schema.fields_) {
field_types.insert({field.first, field.second.field_type_});
}
auto entities = body_json["entity"];
if (!entities.is_array()) {
RETURN_STATUS_DTO(ILLEGAL_BODY, "An entity must be an array");
std::unordered_map<std::string, std::vector<uint8_t>> chunk_data;
int64_t row_num;
auto entities_json = body_json["entities"];
if (!entities_json.is_array()) {
RETURN_STATUS_DTO(ILLEGAL_ARGUMENT, "Entities is not an array");
}
row_num = entities_json.size();
int64_t offset = 0;
std::vector<uint8_t> ids;
for (auto& one_entity : entities_json) {
for (auto& entity : one_entity.items()) {
std::string field_name = entity.key();
if (field_name == NAME_ID) {
if (ids.empty()) {
ids.resize(row_num * sizeof(int64_t));
}
int64_t id = entity.value().get<int64_t>();
int64_t id_offset = offset * sizeof(int64_t);
memcpy(ids.data() + id_offset, &id, sizeof(int64_t));
continue;
}
std::vector<uint8_t> temp_data;
switch (field_types.at(field_name)) {
case engine::DataType::INT32: {
CopyRowStructuredData<int32_t>(entity.value(), field_name, offset, row_num, chunk_data);
break;
}
case engine::DataType::INT64: {
CopyRowStructuredData<int64_t>(entity.value(), field_name, offset, row_num, chunk_data);
break;
}
case engine::DataType::FLOAT: {
CopyRowStructuredData<float>(entity.value(), field_name, offset, row_num, chunk_data);
break;
}
case engine::DataType::DOUBLE: {
CopyRowStructuredData<double>(entity.value(), field_name, offset, row_num, chunk_data);
break;
}
case engine::DataType::VECTOR_FLOAT:
case engine::DataType::VECTOR_BINARY: {
bool is_bin = !(field_types.at(field_name) == engine::DataType::VECTOR_FLOAT);
CopyRowVectorFromJson(entity.value(), temp_data, is_bin);
auto size = temp_data.size();
if (chunk_data.find(field_name) == chunk_data.end()) {
std::vector<uint8_t> vector_data(row_num * size, 0);
memcpy(vector_data.data(), temp_data.data(), size);
chunk_data.insert({field_name, vector_data});
} else {
int64_t vector_offset = offset * size;
memcpy(chunk_data.at(field_name).data() + vector_offset, temp_data.data(), size);
}
break;
}
default: {}
}
}
offset++;
}
std::unordered_map<std::string, std::vector<uint8_t>> chunk_data;
if (!ids.empty()) {
chunk_data.insert({engine::FIELD_UID, ids});
}
for (auto& entity : entities) {
std::string field_name = entity["field_name"];
auto field_value = entity["field_value"];
auto size = field_value.size();
if (size != row_num) {
RETURN_STATUS_DTO(ILLEGAL_ROWRECORD, "Field row count inconsist");
#if 0
for (auto& entity : body_json["entities"].items()) {
std::string field_name = entity.key();
auto field_value = entity.value();
if (!field_value.is_array()) {
RETURN_STATUS_DTO(ILLEGAL_ROWRECORD, "Field value is not an array");
}
if (field_name == NAME_ID) {
std::vector<uint8_t> temp_data(field_value.size() * sizeof(int64_t), 0);
CopyStructuredData<int64_t>(field_value, temp_data);
chunk_data.insert({engine::FIELD_UID, temp_data});
continue;
}
row_num = field_value.size();
std::vector<uint8_t> temp_data;
switch (field_types.at(field_name)) {
@ -1536,6 +1663,7 @@ WebRequestHandler::InsertEntity(const OString& collection_name, const milvus::se
chunk_data.insert(std::make_pair(field_name, temp_data));
}
#endif
status = req_handler_.Insert(context_ptr_, collection_name->c_str(), partition_name, row_num, chunk_data);
if (!status.ok()) {
@ -1547,6 +1675,7 @@ WebRequestHandler::InsertEntity(const OString& collection_name, const milvus::se
if (pair != chunk_data.end()) {
int64_t count = pair->second.size() / 8;
int64_t* pdata = reinterpret_cast<int64_t*>(pair->second.data());
ids_dto->ids = ids_dto->ids.createShared();
for (int64_t i = 0; i < count; ++i) {
ids_dto->ids->push_back(std::to_string(pdata[i]).c_str());
}

2
core/src/server/web_impl/handler/WebRequestHandler.h
View File

@ -213,7 +213,7 @@ class WebRequestHandler {
* Vector
*/
StatusDtoT
InsertEntity(const OString& collection_name, const OString& body, VectorIdsDtoT& ids_dto);
InsertEntity(const OString& collection_name, const OString& body, EntityIdsDtoT& ids_dto);
StatusDtoT
GetEntity(const OString& collection_name, const OQueryParams& query_params, OString& response);

2
core/unittest/CMakeLists.txt
View File

@ -38,6 +38,7 @@ set( SCHEDULER_FILES ${SCHEDULER_MAIN_FILES}
${SCHEDULER_SELECTOR_FILES}
${SCHEDULER_RESOURCE_FILES}
${SCHEDULER_TASK_FILES}
)
set( ENTRY_FILE ${CMAKE_CURRENT_SOURCE_DIR}/main.cpp )
@ -80,6 +81,7 @@ if ( MILVUS_WITH_AWS )
endif ()
add_subdirectory(db)
add_subdirectory(server)
#add_subdirectory(metrics)
#add_subdirectory(scheduler)
#add_subdirectory(thirdparty)

44
core/unittest/server/CMakeLists.txt Normal file
View File

@ -0,0 +1,44 @@
#-------------------------------------------------------------------------------
# Copyright (C) 2019-2020 Zilliz. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software distributed under the License
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
# or implied. See the License for the specific language governing permissions and limitations under the License.
#-------------------------------------------------------------------------------
set( TEST_FILES
${CMAKE_CURRENT_SOURCE_DIR}/test_web.cpp
)
add_executable( test_server
${TEST_FILES}
${SCHEDULER_FILES}
# ${grpc_server_files}
# ${grpc_service_files}
# ${web_server_files}
# ${server_delivery_files}
# ${server_files}
# ${server_init_files}
)
get_target_property( var server INCLUDE_DIRECTORIES)
#set_target_properties( server PROPERTIES INTERFACE_INCLUDE_DIRECTORIES $<BUILD_INTERFACE:${var}>)
target_link_libraries( test_server
${UNITTEST_LIBS}
server
milvus_engine
metrics
config
stdc++
utils
tracing
query
log
)
install( TARGETS test_server DESTINATION unittest )

724
core/unittest/server/test_web.cpp Normal file
View File

@ -0,0 +1,724 @@
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
#include <unistd.h>
#include <random>
#include <thread>
#include <fiu-control.h>
#include <fiu-local.h>
#include <gtest/gtest.h>
#include <src/server/delivery/ReqScheduler.h>
#include <boost/filesystem.hpp>
#include <oatpp/core/macro/component.hpp>
#include <oatpp/network/client/SimpleTCPConnectionProvider.hpp>
#include <oatpp/web/client/ApiClient.hpp>
#include <oatpp/web/client/HttpRequestExecutor.hpp>
#include "config/ConfigMgr.h"
#include "db/snapshot/EventExecutor.h"
#include "db/snapshot/OperationExecutor.h"
#include "scheduler/ResourceFactory.h"
#include "scheduler/SchedInst.h"
#include "server/DBWrapper.h"
#include "server/web_impl/Types.h"
#include "server/web_impl/WebServer.h"
#include "server/web_impl/dto/CollectionDto.hpp"
#include "server/web_impl/dto/StatusDto.hpp"
#include "server/web_impl/dto/VectorDto.hpp"
#include "server/web_impl/handler/WebRequestHandler.h"
#include "server/web_impl/utils/Util.h"
#include "src/version.h"
#include "utils/CommonUtil.h"
#include "utils/StringHelpFunctions.h"
INITIALIZE_EASYLOGGINGPP
static const char* COLLECTION_NAME = "test_milvus_web_collection";
static int64_t DIM = 128;
using OStatus = oatpp::web::protocol::http::Status;
using OString = milvus::server::web::OString;
using OQueryParams = milvus::server::web::OQueryParams;
using OChunkedBuffer = oatpp::data::stream::ChunkedBuffer;
using OOutputStream = oatpp::data::stream::BufferOutputStream;
using OFloat32 = milvus::server::web::OFloat32;
using OInt64 = milvus::server::web::OInt64;
template <class T>
using OList = milvus::server::web::OList<T>;
using StatusCode = milvus::server::web::StatusCode;
namespace {
nlohmann::json
RandomRawRecordJson(int64_t dim) {
nlohmann::json json;
std::default_random_engine e;
std::uniform_real_distribution<float> u(0, 1);
for (size_t i = 0; i < dim; i++) {
json.push_back(u(e));
}
return json;
}
nlohmann::json
RandomRecordsJson(int64_t dim, int64_t num) {
nlohmann::json json;
for (size_t i = 0; i < num; i++) {
json.push_back(RandomRawRecordJson(dim));
}
return json;
}
nlohmann::json
RandomRawBinRecordJson(int64_t dim) {
nlohmann::json json;
std::default_random_engine e;
std::uniform_real_distribution<float> u(0, 255);
for (size_t i = 0; i < dim / 8; i++) {
json.push_back(static_cast<uint8_t>(u(e)));
}
return json;
}
nlohmann::json
RandomBinRecordsJson(int64_t dim, int64_t num) {
nlohmann::json json;
for (size_t i = 0; i < num; i++) {
json.push_back(RandomRawBinRecordJson(dim));
}
return json;
}
nlohmann::json
RandomAttrRecordsJson(int64_t row_num) {
nlohmann::json json;
std::default_random_engine e;
std::uniform_int_distribution<unsigned> u(0, 1000);
for (size_t i = 0; i < row_num; i++) {
json.push_back(u(e));
}
return json;
}
std::string
RandomName() {
unsigned seed = std::chrono::system_clock::now().time_since_epoch().count();
std::default_random_engine e(seed);
std::uniform_int_distribution<unsigned> u(0, 1000000);
size_t name_len = u(e) % 16 + 3;
char* name = new char[name_len + 1];
name[name_len] = '\0';
for (size_t i = 0; i < name_len; i++) {
unsigned random_i = u(e);
char remainder = static_cast<char>(random_i % 26);
name[i] = (random_i % 2 == 0) ? 'A' + remainder : 'a' + remainder;
}
std::string random_name(name);
delete[] name;
return random_name;
}
} // namespace
///////////////////////////////////////////////////////////////////////////////////////
namespace {
static const char* CONTROLLER_TEST_VALID_CONFIG_STR =
"# Default values are used when you make no changes to the following parameters.\n"
"\n"
"version: 0.5\n"
"\n"
"cluster:\n"
" enable: false\n"
" role: rw\n"
"\n"
"general:\n"
" timezone: UTC+8\n"
" meta_uri: sqlite://:@:/\n"
"\n"
"network:\n"
" bind.address: 0.0.0.0\n"
" bind.port: 19530\n"
" http.enable: true\n"
" http.port: 29999\n"
"\n"
"storage:\n"
" path: /tmp/milvus_web_controller_test\n"
" auto_flush_interval: 1\n"
"\n"
"wal:\n"
" enable: false\n"
" recovery_error_ignore: false\n"
" buffer_size: 256MB\n"
" path: /tmp/milvus_web_controller_test/wal\n"
"\n"
"cache:\n"
" cache_size: 4GB\n"
" insert_buffer_size: 1GB\n"
" preload_collection:\n"
"\n"
"gpu:\n"
" enable: true\n"
" cache_size: 1GB\n"
" gpu_search_threshold: 1000\n"
" search_devices:\n"
" - gpu0\n"
" build_index_devices:\n"
" - gpu0\n"
"\n"
"logs:\n"
" level: debug\n"
" trace.enable: true\n"
" path: /tmp/milvus_web_controller_test/logs\n"
" max_log_file_size: 1024MB\n"
" log_rotate_num: 0\n"
"\n"
"metric:\n"
" enable: false\n"
" address: 127.0.0.1\n"
" port: 9091\n"
"\n";
} // namespace
static const char* CONTROLLER_TEST_COLLECTION_NAME = "controller_unit_test";
static const char* CONTROLLER_TEST_CONFIG_DIR = "/tmp/milvus_web_controller_test/";
static const char* CONTROLLER_TEST_CONFIG_WAL_DIR = "/tmp/milvus_web_controller_test/wal";
static const char* CONTROLLER_TEST_CONFIG_FILE = "config.yaml";
class TestClient : public oatpp::web::client::ApiClient {
public:
#include OATPP_CODEGEN_BEGIN(ApiClient)
API_CLIENT_INIT(TestClient)
API_CALL("GET", "/", root)
API_CALL("GET", "/state", getState)
API_CALL("GET", "/devices", getDevices)
API_CALL("GET", "/config/advanced", getAdvanced)
API_CALL("OPTIONS", "/config/advanced", optionsAdvanced)
API_CALL("PUT", "/config/advanced", setAdvanced, BODY_DTO(milvus::server::web::AdvancedConfigDtoT, body))
//#ifdef MILVUS_GPU_VERSION
//
// API_CALL("OPTIONS", "config/gpu_resources", optionsGpuConfig)
//
// API_CALL("GET", "/config/gpu_resources", getGPUConfig)
//
// API_CALL("PUT", "/config/gpu_resources", setGPUConfig,
// BODY_DTO(milvus::server::web::GPUConfigDto::ObjectWrapper, body))
//
//#endif
API_CALL("OPTIONS", "/collections", optionsCollections)
API_CALL("POST", "/collections", createCollection, BODY_STRING(String, body_str))
API_CALL("GET", "/collections", showCollections, QUERY(String, offset), QUERY(String, page_size))
API_CALL("OPTIONS", "/collections/{collection_name}", optionsCollection,
PATH(String, collection_name, "collection_name"))
API_CALL("GET", "/collections/{collection_name}", getCollection, PATH(String, collection_name, "collection_name"),
QUERY(String, info))
API_CALL("DELETE", "/collections/{collection_name}", dropCollection,
PATH(String, collection_name, "collection_name"))
API_CALL("OPTIONS", "/collections/{collection_name}/fields/{field_name}/indexes", optionsIndexes,
PATH(String, collection_name, "collection_name"))
API_CALL("POST", "/collections/{collection_name}/fields/{field_name}/indexes", createIndex,
PATH(String, collection_name, "collection_name"), BODY_STRING(OString, body))
API_CALL("DELETE", "/collections/{collection_name}/fields/{field_name}/indexes", dropIndex,
PATH(String, collection_name, "collection_name"))
API_CALL("OPTIONS", "/collections/{collection_name}/partitions", optionsPartitions,
PATH(String, collection_name, "collection_name"))
API_CALL("POST", "/collections/{collection_name}/partitions", createPartition,
PATH(String, collection_name, "collection_name"),
BODY_DTO(milvus::server::web::PartitionRequestDtoT, body))
API_CALL("GET", "/collections/{collection_name}/partitions", showPartitions,
PATH(String, collection_name, "collection_name"), QUERY(String, offset), QUERY(String, page_size),
BODY_STRING(String, body))
API_CALL("DELETE", "/collections/{collection_name}/partitions", dropPartition,
PATH(String, collection_name, "collection_name"), BODY_STRING(String, body))
API_CALL("GET", "/collections/{collection_name}/segments", showSegments,
PATH(String, collection_name, "collection_name"), QUERY(String, offset), QUERY(String, page_size),
QUERY(String, partition_tag))
API_CALL("GET", "/collections/{collection_name}/segments/{segment_name}/{info}", getSegmentInfo,
PATH(String, collection_name, "collection_name"), PATH(String, segment_name, "segment_name"),
PATH(String, info, "info"), QUERY(String, offset), QUERY(String, page_size))
API_CALL("OPTIONS", "/collections/{collection_name}/entities", optionsEntity,
PATH(String, collection_name, "collection_name"))
API_CALL("GET", "/collections/{collection_name}/entities", getEntity,
PATH(String, collection_name, "collection_name"), QUERY(String, ids))
API_CALL("POST", "/collections/{collection_name}/entities", insert,
PATH(String, collection_name, "collection_name"), BODY_STRING(String, body))
API_CALL("PUT", "/collections/{collection_name}/entities", entityOp,
PATH(String, collection_name, "collection_name"), BODY_STRING(String, body))
API_CALL("GET", "/system/{msg}", cmd, PATH(String, cmd_str, "msg"), QUERY(String, action), QUERY(String, target))
API_CALL("PUT", "/system/{op}", op, PATH(String, cmd_str, "op"), BODY_STRING(String, body))
#include OATPP_CODEGEN_END(ApiClient)
};
using TestClientP = std::shared_ptr<TestClient>;
using TestConnP = std::shared_ptr<oatpp::web::client::RequestExecutor::ConnectionHandle>;
class WebControllerTest : public ::testing::Test {
public:
static void
SetUpTestCase() {
mkdir(CONTROLLER_TEST_CONFIG_DIR, S_IRWXU);
// Load basic config
std::string config_path = std::string(CONTROLLER_TEST_CONFIG_DIR).append(CONTROLLER_TEST_CONFIG_FILE);
std::fstream fs(config_path.c_str(), std::ios_base::out);
fs << CONTROLLER_TEST_VALID_CONFIG_STR;
fs.flush();
fs.close();
milvus::ConfigMgr::GetInstance().Init();
milvus::ConfigMgr::GetInstance().Load(config_path);
auto res_mgr = milvus::scheduler::ResMgrInst::GetInstance();
res_mgr->Clear();
res_mgr->Add(milvus::scheduler::ResourceFactory::Create("disk", "DISK", 0, false));
res_mgr->Add(milvus::scheduler::ResourceFactory::Create("cpu", "CPU", 0));
auto default_conn = milvus::scheduler::Connection("IO", 500.0);
// auto PCIE = milvus::scheduler::Connection("IO", 11000.0);
res_mgr->Connect("disk", "cpu", default_conn);
res_mgr->Start();
milvus::scheduler::SchedInst::GetInstance()->Start();
milvus::scheduler::JobMgrInst::GetInstance()->Start();
milvus::scheduler::CPUBuilderInst::GetInstance()->Start();
milvus::engine::DBOptions opt;
// boost::filesystem::remove_all(CONTROLLER_TEST_CONFIG_DIR);
milvus::server::DBWrapper::GetInstance().StartService();
milvus::server::web::WebServer::GetInstance().Start();
sleep(3);
}
void
SetUp() override {
mkdir(CONTROLLER_TEST_CONFIG_DIR, S_IRWXU);
// Load basic config
std::string config_path = std::string(CONTROLLER_TEST_CONFIG_DIR).append(CONTROLLER_TEST_CONFIG_FILE);
std::fstream fs(config_path.c_str(), std::ios_base::out);
fs << CONTROLLER_TEST_VALID_CONFIG_STR;
fs.flush();
fs.close();
// milvus::ConfigMgr::GetInstance().Init();
milvus::ConfigMgr::GetInstance().Load(config_path);
OATPP_COMPONENT(std::shared_ptr<oatpp::network::ClientConnectionProvider>, clientConnectionProvider);
OATPP_COMPONENT(std::shared_ptr<oatpp::data::mapping::ObjectMapper>, objectMapper);
object_mapper = objectMapper;
auto requestExecutor = oatpp::web::client::HttpRequestExecutor::createShared(clientConnectionProvider);
client_ptr = TestClient::createShared(requestExecutor, objectMapper);
connection_ptr = client_ptr->getConnection();
}
static void
TearDownTestCase() {
milvus::server::web::WebServer::GetInstance().Stop();
milvus::server::DBWrapper::GetInstance().StopService();
milvus::scheduler::JobMgrInst::GetInstance()->Stop();
milvus::scheduler::SchedInst::GetInstance()->Stop();
milvus::scheduler::CPUBuilderInst::GetInstance()->Stop();
milvus::scheduler::ResMgrInst::GetInstance()->Stop();
milvus::scheduler::ResMgrInst::GetInstance()->Clear();
boost::filesystem::remove_all(CONTROLLER_TEST_CONFIG_DIR);
}
void
TearDown() override{};
protected:
std::shared_ptr<oatpp::data::mapping::ObjectMapper> object_mapper;
TestConnP connection_ptr;
TestClientP client_ptr;
};
void
CreateCollection(const TestClientP& client_ptr, const TestConnP& connection_ptr, const std::string& collection_name,
nlohmann::json& mapping_json, bool auto_id = true) {
std::string mapping_str = R"({
"collection_name": "test_collection",
"fields": [
{
"field_name": "field_vec",
"field_type": "VECTOR_FLOAT",
"index_params": {"name": "index_1", "index_type": "IVFFLAT", "nlist": 4096},
"extra_params": {"dim": 128}
},
{
"field_name": "int64",
"field_type": "int64",
"index_params": {},
"extra_params": {}
}
],
"segment_row_count": 100000
})";
mapping_json = nlohmann::json::parse(mapping_str);
mapping_json["collection_name"] = collection_name;
mapping_json["auto_id"] = auto_id;
auto response = client_ptr->getCollection(collection_name.c_str(), "", connection_ptr);
if (OStatus::CODE_200.code == response->getStatusCode()) {
return;
}
client_ptr->createCollection(mapping_json.dump().c_str(), connection_ptr);
}
void
GenEntities(const int64_t nb, const int64_t dim, nlohmann::json& insert_json, bool is_bin = false,
bool auto_id = true) {
nlohmann::json entities_json;
std::default_random_engine e;
std::uniform_int_distribution<unsigned> u(0, 1000);
for (int64_t i = 0; i < nb; i++) {
nlohmann::json one_json;
one_json["int64"] = u(e);
if (is_bin) {
one_json["field_vec"] = RandomRawBinRecordJson(dim);
} else {
one_json["field_vec"] = RandomRawRecordJson(dim);
}
if (!auto_id) {
one_json["__id"] = i;
}
entities_json.push_back(one_json);
}
insert_json["entities"] = entities_json;
}
milvus::Status
FlushCollection(const TestClientP& client_ptr, const TestConnP& connection_ptr, const OString& collection_name) {
nlohmann::json flush_json;
flush_json["flush"]["collection_names"] = {collection_name->std_str()};
auto response = client_ptr->op("task", flush_json.dump().c_str(), connection_ptr);
if (OStatus::CODE_200.code != response->getStatusCode()) {
return milvus::Status(milvus::SERVER_UNEXPECTED_ERROR, response->readBodyToString()->std_str());
}
return milvus::Status::OK();
}
/////////////////////////////////////////////////////////////////////////////////
TEST_F(WebControllerTest, OPTIONS) {
auto response = client_ptr->root(connection_ptr);
ASSERT_EQ(OStatus::CODE_200.code, response->getStatusCode());
response = client_ptr->getState(connection_ptr);
ASSERT_EQ(OStatus::CODE_200.code, response->getStatusCode());
response = client_ptr->optionsAdvanced(connection_ptr);
ASSERT_EQ(OStatus::CODE_204.code, response->getStatusCode());
response = client_ptr->optionsPartitions("collection_name", connection_ptr);
ASSERT_EQ(OStatus::CODE_204.code, response->getStatusCode());
response = client_ptr->optionsCollection("collection", connection_ptr);
ASSERT_EQ(OStatus::CODE_204.code, response->getStatusCode());
response = client_ptr->optionsCollections(connection_ptr);
ASSERT_EQ(OStatus::CODE_204.code, response->getStatusCode());
response = client_ptr->optionsEntity("collection", connection_ptr);
ASSERT_EQ(OStatus::CODE_204.code, response->getStatusCode());
}
TEST_F(WebControllerTest, CREATE_COLLECTION) {
std::string mapping_str = R"({
"collection_name": "test_collection",
"fields": [
{
"field_name": "field_vec",
"field_type": "VECTOR_FLOAT",
"index_params": {"name": "index_1", "index_type": "IVFFLAT", "nlist": 4096},
"extra_params": {"dim": 128}
},
{
"field_name": "int64",
"field_type": "int64",
"index_params": {},
"extra_params": {}
}
]
})";
auto response = client_ptr->createCollection(mapping_str.c_str(), connection_ptr);
ASSERT_EQ(OStatus::CODE_201.code, response->getStatusCode());
// auto error_dto = response->readBodyToDto<milvus::server::web::StatusDtoT>(object_mapper.get());
}
TEST_F(WebControllerTest, GET_COLLECTION_INFO) {
auto collection_name = "test_web" + RandomName();
nlohmann::json mapping_json;
CreateCollection(client_ptr, connection_ptr, collection_name, mapping_json);
OQueryParams params;
auto response = client_ptr->getCollection(collection_name.c_str(), "", connection_ptr);
ASSERT_EQ(OStatus::CODE_200.code, response->getStatusCode());
auto json_response = nlohmann::json::parse(response->readBodyToString()->c_str());
ASSERT_EQ(collection_name, json_response["collection_name"]);
ASSERT_EQ(2, json_response["fields"].size());
// invalid collection name
collection_name = "57474dgdfhdfhdh dgd";
response = client_ptr->getCollection(collection_name.c_str(), "", connection_ptr);
ASSERT_EQ(OStatus::CODE_400.code, response->getStatusCode());
auto status_sto = response->readBodyToDto<milvus::server::web::StatusDtoT>(object_mapper.get());
ASSERT_EQ(milvus::server::web::StatusCode::ILLEGAL_COLLECTION_NAME, status_sto->code);
}
TEST_F(WebControllerTest, SHOW_COLLECTIONS) {
auto response = client_ptr->showCollections("1", "1", connection_ptr);
ASSERT_EQ(OStatus::CODE_200.code, response->getStatusCode());
auto json_response = nlohmann::json::parse(response->readBodyToString()->std_str());
ASSERT_GE(json_response["count"].get<int64_t>(), 0);
// test query collection empty
response = client_ptr->showCollections("0", "0", connection_ptr);
ASSERT_EQ(OStatus::CODE_200.code, response->getStatusCode());
response = client_ptr->showCollections("-1", "0", connection_ptr);
ASSERT_EQ(OStatus::CODE_400.code, response->getStatusCode());
response = client_ptr->showCollections("0", "-10", connection_ptr);
ASSERT_EQ(OStatus::CODE_400.code, response->getStatusCode());
// test wrong param
response = client_ptr->showCollections("0.1", "1", connection_ptr);
ASSERT_EQ(OStatus::CODE_400.code, response->getStatusCode());
response = client_ptr->showCollections("1", "1.1", connection_ptr);
ASSERT_EQ(OStatus::CODE_400.code, response->getStatusCode());
response =
client_ptr->showCollections("0", "9000000000000000000000000000000000000000000000000000000", connection_ptr);
ASSERT_EQ(OStatus::CODE_400.code, response->getStatusCode());
}
TEST_F(WebControllerTest, DROP_COLLECTION) {
auto collection_name = "collection_drop_test" + RandomName();
nlohmann::json mapping_json;
CreateCollection(client_ptr, connection_ptr, collection_name, mapping_json);
sleep(1);
auto response = client_ptr->dropCollection(collection_name.c_str(), connection_ptr);
ASSERT_EQ(OStatus::CODE_204.code, response->getStatusCode());
collection_name = "collection_drop_test_not_exists_" + RandomName();
response = client_ptr->dropCollection(collection_name.c_str(), connection_ptr);
ASSERT_EQ(OStatus::CODE_404.code, response->getStatusCode());
auto error_dto = response->readBodyToDto<milvus::server::web::StatusDtoT>(object_mapper.get());
ASSERT_EQ(milvus::server::web::StatusCode::COLLECTION_NOT_EXISTS, error_dto->code);
}
TEST_F(WebControllerTest, INSERT) {
auto collection_name = "test_insert_collection_test" + RandomName();
nlohmann::json mapping_json;
CreateCollection(client_ptr, connection_ptr, collection_name, mapping_json);
const int64_t dim = DIM;
const int64_t nb = 1000;
nlohmann::json insert_json;
GenEntities(nb, dim, insert_json);
auto response = client_ptr->insert(collection_name.c_str(), insert_json.dump().c_str(), connection_ptr);
ASSERT_EQ(OStatus::CODE_201.code, response->getStatusCode());
auto result_dto = response->readBodyToDto<milvus::server::web::EntityIdsDtoT>(object_mapper.get());
ASSERT_EQ(nb, result_dto->ids->size());
auto not_exist_col = collection_name + "ooowrweindexsgs";
response = client_ptr->insert(not_exist_col.c_str(), insert_json.dump().c_str(), connection_ptr);
ASSERT_EQ(OStatus::CODE_404.code, response->getStatusCode());
response = client_ptr->dropCollection(collection_name.c_str(), connection_ptr);
ASSERT_EQ(OStatus::CODE_204.code, response->getStatusCode());
}
TEST_F(WebControllerTest, INSERT_BIN) {
auto collection_name = "test_insert_bin_collection_test" + RandomName();
std::string mapping_str = R"({
"collection_name": "test_collection",
"fields": [
{
"field_name": "field_vec",
"field_type": "VECTOR_BINARY",
"index_params": {"name": "index_1", "index_type": "IVFFLAT", "nlist": 4096},
"extra_params": {"dim": 128}
},
{
"field_name": "int64",
"field_type": "int64",
"index_params": {},
"extra_params": {}
}
],
"segment_row_count": 100000,
"auto_id": true
})";
nlohmann::json mapping_json = nlohmann::json::parse(mapping_str);
mapping_json["collection_name"] = collection_name;
auto response = client_ptr->createCollection(mapping_json.dump().c_str(), connection_ptr);
const int64_t dim = DIM;
const int64_t nb = 20;
nlohmann::json insert_json;
GenEntities(nb, dim, insert_json, true);
response = client_ptr->insert(collection_name.c_str(), insert_json.dump().c_str(), connection_ptr);
ASSERT_EQ(OStatus::CODE_201.code, response->getStatusCode()) << response->readBodyToString()->std_str();
auto status = FlushCollection(client_ptr, connection_ptr, collection_name.c_str());
ASSERT_TRUE(status.ok()) << status.message();
auto result_dto = response->readBodyToDto<milvus::server::web::EntityIdsDtoT>(object_mapper.get());
ASSERT_EQ(nb, result_dto->ids->size());
response = client_ptr->dropCollection(collection_name.c_str(), connection_ptr);
ASSERT_EQ(OStatus::CODE_204.code, response->getStatusCode());
}
TEST_F(WebControllerTest, INSERT_IDS) {
auto collection_name = "test_insert_collection_test" + RandomName();
nlohmann::json mapping_json;
CreateCollection(client_ptr, connection_ptr, collection_name, mapping_json, false);
const int64_t dim = DIM;
const int64_t nb = 20;
nlohmann::json insert_json;
GenEntities(nb, dim, insert_json, false, false);
auto response = client_ptr->insert(collection_name.c_str(), insert_json.dump().c_str(), connection_ptr);
ASSERT_EQ(OStatus::CODE_201.code, response->getStatusCode()) << response->readBodyToString()->std_str();
auto result_dto = response->readBodyToDto<milvus::server::web::EntityIdsDtoT>(object_mapper.get());
ASSERT_EQ(20, result_dto->ids->size());
response = client_ptr->dropCollection(collection_name.c_str(), connection_ptr);
ASSERT_EQ(OStatus::CODE_204.code, response->getStatusCode());
}
TEST_F(WebControllerTest, INDEX) {
// auto collection_name = "test_insert_collection_test" + RandomName();
// nlohmann::json mapping_json;
// CreateCollection(client_ptr, connection_ptr, collection_name, mapping_json);
//
// // test index with imcomplete param
// nlohmann::json index_json;
// auto response = client_ptr->createIndex(collection_name.c_str(), index_json.dump().c_str(), connection_ptr);
// ASSERT_EQ(OStatus::CODE_400.code, response->getStatusCode());
//
// // index_json["index_type"] = milvus::server::web::IndexMap.at(milvus::engine::FAISS_IDMAP);
//
// // missing index `params`
// response = client_ptr->createIndex(collection_name.c_str(), index_json.dump().c_str(), connection_ptr);
// ASSERT_EQ(OStatus::CODE_400.code, response->getStatusCode());
//
// index_json["params"] = nlohmann::json::parse("{\"nlist\": 10}");
// response = client_ptr->createIndex(collection_name.c_str(), index_json.dump().c_str(), connection_ptr);
// ASSERT_EQ(OStatus::CODE_201.code, response->getStatusCode());
//
// // drop index
// response = client_ptr->dropIndex(collection_name.c_str(), connection_ptr);
// ASSERT_EQ(OStatus::CODE_204.code, response->getStatusCode());
//
// // create index without existing collection
// response = client_ptr->createIndex((collection_name + "fgafafafafafUUUUUUa124254").c_str(),
// index_json.dump().c_str(), connection_ptr);
// ASSERT_EQ(OStatus::CODE_404.code, response->getStatusCode());
//
// // invalid index type
// index_json["index_type"] = "J46";
// response = client_ptr->createIndex(collection_name.c_str(), index_json.dump().c_str(), connection_ptr);
// ASSERT_EQ(OStatus::CODE_400.code, response->getStatusCode());
// auto result_dto = response->readBodyToDto<milvus::server::web::StatusDtoT>(object_mapper.get());
// ASSERT_EQ((int64_t)milvus::server::web::StatusCode::ILLEGAL_INDEX_TYPE, result_dto->code);
//
// // drop index
// response = client_ptr->dropIndex(collection_name.c_str(), connection_ptr);
// ASSERT_EQ(OStatus::CODE_204.code, response->getStatusCode());
// insert data and create index
// auto status = InsertData(client_ptr, connection_ptr, collection_name, 64, 200);
// ASSERT_TRUE(status.ok()) << status.message();
//
// index_json["index_type"] = milvus::server::web::IndexMap.at(milvus::engine::EngineType::FAISS_IVFFLAT);
// response = client_ptr->createIndex(collection_name, index_json.dump().c_str(), connection_ptr);
// ASSERT_EQ(OStatus::CODE_201.code, response->getStatusCode());
//
// // get index
// response = client_ptr->getIndex(collection_name, connection_ptr);
// ASSERT_EQ(OStatus::CODE_200.code, response->getStatusCode());
// auto result_index_json = nlohmann::json::parse(response->readBodyToString()->c_str());
// ASSERT_TRUE(result_index_json.contains("index_type"));
// ASSERT_EQ("IVFFLAT", result_index_json["index_type"].get<std::string>());
// ASSERT_TRUE(result_index_json.contains("params"));
//
// // check index params
// auto params_json = result_index_json["params"];
// ASSERT_TRUE(params_json.contains("nlist"));
// auto nlist_json = params_json["nlist"];
// ASSERT_TRUE(nlist_json.is_number());
// ASSERT_EQ(10, nlist_json.get<int64_t>());
//
// // get index of collection which not exists
// response = client_ptr->getIndex(collection_name + "dfaedXXXdfdfet4t343aa4", connection_ptr);
// ASSERT_EQ(OStatus::CODE_404.code, response->getStatusCode());
// auto error_dto = response->readBodyToDto<milvus::server::web::StatusDto>(object_mapper.get());
// ASSERT_EQ(milvus::server::web::StatusCode::COLLECTION_NOT_EXISTS, error_dto->code->getValue());
}