From 6862db78d42a27df7056a19f7c414c5a3ba0f27c Mon Sep 17 00:00:00 2001 From: BossZou <40255591+BossZou@users.noreply.github.com> Date: Mon, 2 Mar 2020 14:48:42 +0800 Subject: [PATCH] Add crud APIs and segments APIs into http server (#1462) * add new http api Signed-off-by: Yhz * [DOC] modify comments Signed-off-by: Yhz * allow get whole tables & partitions Signed-off-by: Yhz * add test case Signed-off-by: Yhz * update changlog (fix #1461) Signed-off-by: Yhz * format Signed-off-by: Yhz * fix cpu build issue Signed-off-by: Yhz * fix unittest fail on cpu version Signed-off-by: Yhz * fix get system config bug Signed-off-by: Yhz * fix crash when show tables Signed-off-by: Yhz * fix check server restart fail Signed-off-by: Yhz * remove some comments Signed-off-by: Yhz * add count field into segment ids Signed-off-by: Yhz * optimize code fixing codegacy Signed-off-by: Yhz * add default response string value Signed-off-by: Yhz --- CHANGELOG.md | 1 + core/src/server/Config.cpp | 4 +- core/src/server/web_impl/Types.h | 13 +- .../web_impl/controller/WebController.hpp | 132 +- .../web_impl/handler/WebRequestHandler.cpp | 1207 ++++++++++++----- .../web_impl/handler/WebRequestHandler.h | 119 +- core/unittest/server/test_web.cpp | 886 ++++++++---- tests/milvus_python_test/test_table_info.py | 2 +- 8 files changed, 1760 insertions(+), 604 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8700e91c48..b478f7fa14 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -46,6 +46,7 @@ Please mark all change in change log and use the issue from GitHub - \#1204 - Add api to get table data information - \#1250 - Support CPU profiling - \#1302 - Get all record IDs in a segment by given a segment id +- \#1461 - Add crud APIs and segments APIs into http module ## Improvement - \#738 - Use Openblas / lapack from apt install diff --git a/core/src/server/Config.cpp b/core/src/server/Config.cpp index 5b472209ad..ed67c458b3 100644 --- a/core/src/server/Config.cpp +++ b/core/src/server/Config.cpp @@ -405,8 +405,8 @@ Config::SetConfigCli(const std::string& parent_key, const std::string& child_key if (status.ok()) { status = UpdateFileConfigFromMem(parent_key, child_key); - if (status.ok() && (parent_key == CONFIG_SERVER || parent_key == CONFIG_DB || parent_key == CONFIG_STORAGE || - parent_key == CONFIG_METRIC || parent_key == CONFIG_TRACING)) { + if (status.ok() && + !(parent_key == CONFIG_CACHE || parent_key == CONFIG_ENGINE || parent_key == CONFIG_GPU_RESOURCE)) { restart_required_ = true; } } diff --git a/core/src/server/web_impl/Types.h b/core/src/server/web_impl/Types.h index 2076bf55f7..4c0b462e02 100644 --- a/core/src/server/web_impl/Types.h +++ b/core/src/server/web_impl/Types.h @@ -63,11 +63,14 @@ enum StatusCode : int { // HTTP error code PATH_PARAM_LOSS = 31, - QUERY_PARAM_LOSS = 32, - BODY_FIELD_LOSS = 33, - ILLEGAL_BODY = 34, - BODY_PARSE_FAIL = 35, - ILLEGAL_QUERY_PARAM = 36, + UNKNOWN_PATH = 32, + QUERY_PARAM_LOSS = 33, + BODY_FIELD_LOSS = 34, + ILLEGAL_BODY = 35, + BODY_PARSE_FAIL = 36, + ILLEGAL_QUERY_PARAM = 37, + + MAX = ILLEGAL_QUERY_PARAM }; static const std::unordered_map IndexMap = { diff --git a/core/src/server/web_impl/controller/WebController.hpp b/core/src/server/web_impl/controller/WebController.hpp index 0fc0e85948..56137dbc35 100644 --- a/core/src/server/web_impl/controller/WebController.hpp +++ b/core/src/server/web_impl/controller/WebController.hpp @@ -67,7 +67,6 @@ class WebController : public oatpp::web::server::api::ApiController { return createDtoResponse(Status::CODE_200, StatusDto::createShared()); } - ADD_CORS(GetDevices) ENDPOINT("GET", "/devices", GetDevices) { @@ -245,15 +244,13 @@ class WebController : public oatpp::web::server::api::ApiController { WebRequestHandler handler = WebRequestHandler(); - auto response_dto = TableListFieldsDto::createShared(); - auto offset = query_params.get("offset"); - auto page_size = query_params.get("page_size"); + String result; + auto status_dto = handler.ShowTables(query_params, result); std::shared_ptr response; - auto status_dto = handler.ShowTables(offset, page_size, response_dto); switch (status_dto->code->getValue()) { case StatusCode::SUCCESS: - response = createDtoResponse(Status::CODE_200, response_dto); + response = createResponse(Status::CODE_200, result); break; default: response = createDtoResponse(Status::CODE_400, status_dto); @@ -281,13 +278,13 @@ class WebController : public oatpp::web::server::api::ApiController { WebRequestHandler handler = WebRequestHandler(); - auto fields_dto = TableFieldsDto::createShared(); - auto status_dto = handler.GetTable(table_name, query_params, fields_dto); + String response_str; + auto status_dto = handler.GetTable(table_name, query_params, response_str); std::shared_ptr response; switch (status_dto->code->getValue()) { case StatusCode::SUCCESS: - response = createDtoResponse(Status::CODE_200, fields_dto); + response = createResponse(Status::CODE_200, response_str); break; case StatusCode::TABLE_NOT_EXISTS: response = createDtoResponse(Status::CODE_404, status_dto); @@ -472,7 +469,7 @@ class WebController : public oatpp::web::server::api::ApiController { auto handler = WebRequestHandler(); std::shared_ptr response; - auto status_dto = handler.ShowPartitions(offset, page_size, table_name, partition_list_dto); + auto status_dto = handler.ShowPartitions(table_name, query_params, partition_list_dto); switch (status_dto->code->getValue()) { case StatusCode::SUCCESS: response = createDtoResponse(Status::CODE_200, partition_list_dto); @@ -489,24 +486,18 @@ class WebController : public oatpp::web::server::api::ApiController { return response; } - ADD_CORS(PartitionOptions) - - ENDPOINT("OPTIONS", "/tables/{table_name}/partitions/{partition_tag}", PartitionOptions) { - return createResponse(Status::CODE_204, "No Content"); - } - ADD_CORS(DropPartition) - ENDPOINT("DELETE", "/tables/{table_name}/partitions/{partition_tag}", DropPartition, - PATH(String, table_name), PATH(String, partition_tag)) { + ENDPOINT("DELETE", "/tables/{table_name}/partitions", DropPartition, + PATH(String, table_name), BODY_STRING(String, body)) { TimeRecorder tr(std::string(WEB_LOG_PREFIX) + - "DELETE \'/tables/" + table_name->std_str() + "/partitions/" + partition_tag->std_str() + "\'"); + "DELETE \'/tables/" + table_name->std_str() + "/partitions\'"); tr.RecordSection("Received request."); auto handler = WebRequestHandler(); std::shared_ptr response; - auto status_dto = handler.DropPartition(table_name, partition_tag); + auto status_dto = handler.DropPartition(table_name, body); switch (status_dto->code->getValue()) { case StatusCode::SUCCESS: response = createDtoResponse(Status::CODE_204, status_dto); @@ -524,16 +515,82 @@ class WebController : public oatpp::web::server::api::ApiController { return response; } + ADD_CORS(ShowSegments) + + ENDPOINT("GET", "/tables/{table_name}/segments", ShowSegments, + PATH(String, table_name), QUERIES(const QueryParams&, query_params)) { + auto offset = query_params.get("offset"); + auto page_size = query_params.get("page_size"); + + auto handler = WebRequestHandler(); + String response; + auto status_dto = handler.ShowSegments(table_name, query_params, response); + + switch (status_dto->code->getValue()) { + case StatusCode::SUCCESS:{ + return createResponse(Status::CODE_200, response); + } + default:{ + return createDtoResponse(Status::CODE_400, status_dto); + } + } + } + + ADD_CORS(GetSegmentInfo) + /** + * + * GetSegmentVector + */ + ENDPOINT("GET", "/tables/{table_name}/segments/{segment_name}/{info}", GetSegmentInfo, + PATH(String, table_name), PATH(String, segment_name), PATH(String, info), QUERIES(const QueryParams&, query_params)) { + auto offset = query_params.get("offset"); + auto page_size = query_params.get("page_size"); + + auto handler = WebRequestHandler(); + String response; + auto status_dto = handler.GetSegmentInfo(table_name, segment_name, info, query_params, response); + + switch (status_dto->code->getValue()) { + case StatusCode::SUCCESS:{ + return createResponse(Status::CODE_200, response); + } + default:{ + return createDtoResponse(Status::CODE_400, status_dto); + } + } + } + ADD_CORS(VectorsOptions) ENDPOINT("OPTIONS", "/tables/{table_name}/vectors", VectorsOptions) { return createResponse(Status::CODE_204, "No Content"); } + ADD_CORS(GetVectors) + /** + * + * GetVectorByID ?id= + */ + ENDPOINT("GET", "/tables/{table_name}/vectors", GetVectors, + PATH(String, table_name), QUERIES(const QueryParams&, query_params)) { + auto handler = WebRequestHandler(); + String response; + auto status_dto = handler.GetVector(table_name, query_params, response); + + switch (status_dto->code->getValue()) { + case StatusCode::SUCCESS:{ + return createResponse(Status::CODE_200, response); + } + default:{ + return createDtoResponse(Status::CODE_400, status_dto); + } + } + } + ADD_CORS(Insert) ENDPOINT("POST", "/tables/{table_name}/vectors", Insert, - PATH(String, table_name), BODY_DTO(InsertRequestDto::ObjectWrapper, body)) { + PATH(String, table_name), BODY_STRING(String, body)) { TimeRecorder tr(std::string(WEB_LOG_PREFIX) + "POST \'/tables/" + table_name->std_str() + "/vectors\'"); tr.RecordSection("Received request."); @@ -559,21 +616,24 @@ class WebController : public oatpp::web::server::api::ApiController { return response; } - ADD_CORS(Search) - - ENDPOINT("PUT", "/tables/{table_name}/vectors", Search, - PATH(String, table_name), BODY_DTO(SearchRequestDto::ObjectWrapper, body)) { + ADD_CORS(VectorsOp) + /************* + * Search + * Delete by ID + * */ + ENDPOINT("PUT", "/tables/{table_name}/vectors", VectorsOp, + PATH(String, table_name), BODY_STRING(String, body)) { TimeRecorder tr(std::string(WEB_LOG_PREFIX) + "PUT \'/tables/" + table_name->std_str() + "/vectors\'"); tr.RecordSection("Received request."); - auto results_dto = TopkResultsDto::createShared(); WebRequestHandler handler = WebRequestHandler(); + OString result; std::shared_ptr response; - auto status_dto = handler.Search(table_name, body, results_dto); + auto status_dto = handler.VectorsOp(table_name, body, result); switch (status_dto->code->getValue()) { case StatusCode::SUCCESS: - response = createDtoResponse(Status::CODE_200, results_dto); + response = createResponse(Status::CODE_200, result); break; case StatusCode::TABLE_NOT_EXISTS: response = createDtoResponse(Status::CODE_404, status_dto); @@ -590,18 +650,17 @@ class WebController : public oatpp::web::server::api::ApiController { ADD_CORS(SystemInfo) - ENDPOINT("GET", "/system/{msg}", SystemInfo, PATH(String, msg), QUERIES(const QueryParams&, query_params)) { - TimeRecorder tr(std::string(WEB_LOG_PREFIX) + "GET \'/system/" + msg->std_str() + "\'"); + ENDPOINT("GET", "/system/{info}", SystemInfo, PATH(String, info), QUERIES(const QueryParams&, query_params)) { + TimeRecorder tr(std::string(WEB_LOG_PREFIX) + "GET \'/system/" + info->std_str() + "\'"); tr.RecordSection("Received request."); - auto info_dto = CommandDto::createShared(); WebRequestHandler handler = WebRequestHandler(); - - auto status_dto = handler.SystemInfo(msg, info_dto); + OString result = ""; + auto status_dto = handler.SystemInfo(info, query_params, result); std::shared_ptr response; switch (status_dto->code->getValue()) { case StatusCode::SUCCESS: - response = createDtoResponse(Status::CODE_200, info_dto); + response = createResponse(Status::CODE_200, result); break; default: response = createDtoResponse(Status::CODE_400, status_dto); @@ -615,6 +674,11 @@ class WebController : public oatpp::web::server::api::ApiController { ADD_CORS(SystemOp) + /** + * Load + * Compact + * Flush + */ ENDPOINT("PUT", "/system/{Op}", SystemOp, PATH(String, Op), BODY_STRING(String, body_str)) { TimeRecorder tr(std::string(WEB_LOG_PREFIX) + "PUT \'/system/" + Op->std_str() + "\'"); tr.RecordSection("Received request."); diff --git a/core/src/server/web_impl/handler/WebRequestHandler.cpp b/core/src/server/web_impl/handler/WebRequestHandler.cpp index 271f4bc933..9a78e44d3f 100644 --- a/core/src/server/web_impl/handler/WebRequestHandler.cpp +++ b/core/src/server/web_impl/handler/WebRequestHandler.cpp @@ -11,6 +11,7 @@ #include "server/web_impl/handler/WebRequestHandler.h" +#include #include #include #include @@ -25,7 +26,6 @@ #include "server/web_impl/utils/Util.h" #include "thirdparty/nlohmann/json.hpp" #include "utils/StringHelpFunctions.h" -#include "utils/TimeRecorder.h" #include "utils/ValidationUtil.h" namespace milvus { @@ -67,17 +67,95 @@ WebErrorMap(ErrorCode code) { {DB_NOT_FOUND, StatusCode::TABLE_NOT_EXISTS}, {DB_META_TRANSACTION_FAILED, StatusCode::META_FAILED}, }; - - if (code_map.find(code) != code_map.end()) { + if (code < StatusCode::MAX) { + return StatusCode(code); + } else if (code_map.find(code) != code_map.end()) { return code_map.at(code); } else { return StatusCode::UNEXPECTED_ERROR; } } +/////////////////////////////////// Private methods /////////////////////////////////////// +void +WebRequestHandler::AddStatusToJson(nlohmann::json& json, int64_t code, const std::string& msg) { + json["code"] = (int64_t)code; + json["message"] = msg; +} + +Status +WebRequestHandler::ParseSegmentStat(const milvus::server::SegmentStat& seg_stat, nlohmann::json& json) { + json["segment_name"] = seg_stat.name_; + json["index"] = seg_stat.index_name_; + json["count"] = seg_stat.row_num_; + json["size"] = seg_stat.data_size_; + + return Status::OK(); +} + +Status +WebRequestHandler::ParsePartitionStat(const milvus::server::PartitionStat& par_stat, nlohmann::json& json) { + json["partition_tag"] = par_stat.tag_; + json["count"] = par_stat.total_row_num_; + + std::vector seg_stat_json; + for (auto& seg : par_stat.segments_stat_) { + nlohmann::json seg_json; + ParseSegmentStat(seg, seg_json); + seg_stat_json.push_back(seg_json); + } + json["segments_stat"] = seg_stat_json; + + return Status::OK(); +} + +Status +WebRequestHandler::IsBinaryTable(const std::string& table_name, bool& bin) { + TableSchema schema; + auto status = request_handler_.DescribeTable(context_ptr_, table_name, schema); + if (status.ok()) { + auto metric = engine::MetricType(schema.metric_type_); + bin = engine::MetricType::HAMMING == metric || engine::MetricType::JACCARD == metric || + engine::MetricType::TANIMOTO == metric; + } + + return status; +} + +Status +WebRequestHandler::CopyRecordsFromJson(const nlohmann::json& json, engine::VectorsData& vectors, bool bin) { + if (!json.is_array()) { + return Status(ILLEGAL_BODY, "field \"vectors\" must be a array"); + } + + vectors.vector_count_ = json.size(); + + if (!bin) { + for (auto& vec : json) { + if (!vec.is_array()) { + return Status(ILLEGAL_BODY, "A vector in field \"vectors\" must be a float array"); + } + for (auto& data : vec) { + vectors.float_data_.emplace_back(data.get()); + } + } + } else { + for (auto& vec : json) { + if (!vec.is_array()) { + return Status(ILLEGAL_BODY, "A vector in field \"vectors\" must be a float array"); + } + for (auto& data : vec) { + vectors.binary_data_.emplace_back(data.get()); + } + } + } + + return Status::OK(); +} + ///////////////////////// WebRequestHandler methods /////////////////////////////////////// Status -WebRequestHandler::GetTableInfo(const std::string& table_name, TableFieldsDto::ObjectWrapper& table_fields) { +WebRequestHandler::GetTableMetaInfo(const std::string& table_name, nlohmann::json& json_out) { TableSchema schema; auto status = request_handler_.DescribeTable(context_ptr_, table_name, schema); if (!status.ok()) { @@ -96,13 +174,86 @@ WebRequestHandler::GetTableInfo(const std::string& table_name, TableFieldsDto::O return status; } - table_fields->table_name = schema.table_name_.c_str(); - table_fields->dimension = schema.dimension_; - table_fields->index_file_size = schema.index_file_size_; - table_fields->index = IndexMap.at(engine::EngineType(index_param.index_type_)).c_str(); - table_fields->nlist = index_param.nlist_; - table_fields->metric_type = MetricMap.at(engine::MetricType(schema.metric_type_)).c_str(); - table_fields->count = count; + json_out["table_name"] = schema.table_name_; + json_out["dimension"] = schema.dimension_; + json_out["index_file_size"] = schema.index_file_size_; + json_out["index"] = IndexMap.at(engine::EngineType(index_param.index_type_)); + json_out["nlist"] = index_param.nlist_; + json_out["metric_type"] = MetricMap.at(engine::MetricType(schema.metric_type_)); + json_out["count"] = count; + + return Status::OK(); +} + +Status +WebRequestHandler::GetTableStat(const std::string& table_name, nlohmann::json& json_out) { + struct TableInfo table_info; + auto status = request_handler_.ShowTableInfo(context_ptr_, table_name, table_info); + + if (status.ok()) { + json_out["count"] = table_info.total_row_num_; + + std::vector par_stat_json; + for (auto& par : table_info.partitions_stat_) { + nlohmann::json par_json; + ParsePartitionStat(par, par_json); + par_stat_json.push_back(par_json); + } + json_out["partitions_stat"] = par_stat_json; + } + + return status; +} + +Status +WebRequestHandler::GetSegmentVectors(const std::string& table_name, const std::string& segment_name, int64_t page_size, + int64_t offset, nlohmann::json& json_out) { + std::vector vector_ids; + auto status = request_handler_.GetVectorIDs(context_ptr_, table_name, segment_name, vector_ids); + if (!status.ok()) { + return status; + } + + auto ids_begin = std::min(vector_ids.size(), (size_t)offset); + auto ids_end = std::min(vector_ids.size(), (size_t)(offset + page_size)); + + auto ids = std::vector(vector_ids.begin() + ids_begin, vector_ids.begin() + ids_end); + nlohmann::json vectors_json; + status = GetVectorsByIDs(table_name, ids, vectors_json); + + nlohmann::json result_json; + if (vectors_json.empty()) { + json_out["vectors"] = std::vector(); + } else { + json_out["vectors"] = vectors_json; + } + json_out["count"] = vector_ids.size(); + + AddStatusToJson(json_out, status.code(), status.message()); + + return Status::OK(); +} + +Status +WebRequestHandler::GetSegmentIds(const std::string& table_name, const std::string& segment_name, int64_t page_size, + int64_t offset, nlohmann::json& json_out) { + std::vector vector_ids; + auto status = request_handler_.GetVectorIDs(context_ptr_, table_name, segment_name, vector_ids); + if (status.ok()) { + auto ids_begin = std::min(vector_ids.size(), (size_t)offset); + auto ids_end = std::min(vector_ids.size(), (size_t)(offset + page_size)); + + if (ids_begin >= ids_end) { + json_out["ids"] = std::vector(); + } else { + for (size_t i = ids_begin; i < ids_end; i++) { + json_out["ids"].push_back(std::to_string(vector_ids.at(i))); + } + } + json_out["count"] = vector_ids.size(); + } + + return status; } Status @@ -110,8 +261,339 @@ WebRequestHandler::CommandLine(const std::string& cmd, std::string& reply) { return request_handler_.Cmd(context_ptr_, cmd, reply); } -/////////////////////////////////////////// Router methods //////////////////////////////////////////// +Status +WebRequestHandler::Cmd(const std::string& cmd, std::string& result_str) { + std::string reply; + auto status = CommandLine(cmd, reply); + if (status.ok()) { + nlohmann::json result; + AddStatusToJson(result, status.code(), status.message()); + result["reply"] = reply; + result_str = result.dump(); + } + + return status; +} + +Status +WebRequestHandler::PreLoadTable(const nlohmann::json& json, std::string& result_str) { + if (!json.contains("table_name")) { + return Status(BODY_FIELD_LOSS, "Field \"load\" must contains table_name"); + } + + auto table_name = json["table_name"]; + auto status = request_handler_.PreloadTable(context_ptr_, table_name.get()); + if (status.ok()) { + nlohmann::json result; + AddStatusToJson(result, status.code(), status.message()); + result_str = result.dump(); + } + + return status; +} + +Status +WebRequestHandler::Flush(const nlohmann::json& json, std::string& result_str) { + if (!json.contains("table_names")) { + return Status(BODY_FIELD_LOSS, "Field \"flush\" must contains table_names"); + } + + auto table_names = json["table_names"]; + if (!table_names.is_array()) { + return Status(BODY_FIELD_LOSS, "Field \"table_names\" must be and array"); + } + + std::vector names; + for (auto& name : table_names) { + names.emplace_back(name.get()); + } + + auto status = request_handler_.Flush(context_ptr_, names); + if (status.ok()) { + nlohmann::json result; + AddStatusToJson(result, status.code(), status.message()); + // result["code"] = status.code(); + // result["message"] = status.message(); + result_str = result.dump(); + } + + return status; +} + +Status +WebRequestHandler::Compact(const nlohmann::json& json, std::string& result_str) { + if (!json.contains("table_name")) { + return Status(BODY_FIELD_LOSS, "Field \"compact\" must contains table_names"); + } + + auto table_name = json["table_name"]; + if (!table_name.is_string()) { + return Status(BODY_FIELD_LOSS, "Field \"table_names\" must be a string"); + } + + auto name = table_name.get(); + + auto status = request_handler_.Compact(context_ptr_, name); + + if (status.ok()) { + nlohmann::json result; + result["code"] = status.code(); + result["message"] = status.message(); + result_str = result.dump(); + } + + return status; +} + +Status +WebRequestHandler::GetConfig(std::string& result_str) { + std::string cmd = "get_config *"; + std::string reply; + auto status = CommandLine(cmd, reply); + if (status.ok()) { + nlohmann::json j = nlohmann::json::parse(reply); +#ifdef MILVUS_GPU_VERSION + if (j.contains("gpu_resource_config")) { + std::vector gpus; + if (j["gpu_resource_config"].contains("search_resources")) { + auto gpu_search_res = j["gpu_resource_config"]["search_resources"].get(); + StringHelpFunctions::SplitStringByDelimeter(gpu_search_res, ",", gpus); + j["gpu_resource_config"]["search_resources"] = gpus; + } + if (j["gpu_resource_config"].contains("build_index_resources")) { + auto gpu_build_res = j["gpu_resource_config"]["build_index_resources"].get(); + gpus.clear(); + StringHelpFunctions::SplitStringByDelimeter(gpu_build_res, ",", gpus); + j["gpu_resource_config"]["build_index_resources"] = gpus; + } + } +#endif + // check if server require start + Config& config = Config::GetInstance(); + bool required = false; + config.GetServerRestartRequired(required); + j["restart_required"] = required; + result_str = j.dump(); + } + + return Status::OK(); +} + +Status +WebRequestHandler::SetConfig(const nlohmann::json& json, std::string& result_str) { + if (!json.is_object()) { + return Status(ILLEGAL_BODY, "Payload must be a map"); + } + + std::vector cmds; + for (auto& el : json.items()) { + auto evalue = el.value(); + if (!evalue.is_object()) { + return Status(ILLEGAL_BODY, "Invalid payload format, the root value must be json map"); + } + + for (auto& iel : el.value().items()) { + auto ievalue = iel.value(); + if (!(ievalue.is_string() || ievalue.is_number() || ievalue.is_boolean())) { + return Status(ILLEGAL_BODY, "Config value must be one of string, numeric or boolean"); + } + std::ostringstream ss; + if (ievalue.is_string()) { + std::string vle = ievalue; + ss << "set_config " << el.key() << "." << iel.key() << " " << vle; + } else { + ss << "set_config " << el.key() << "." << iel.key() << " " << ievalue; + } + cmds.emplace_back(ss.str()); + } + } + + std::string msg; + + for (auto& c : cmds) { + std::string reply; + auto status = CommandLine(c, reply); + if (!status.ok()) { + return status; + } + msg += c + " successfully;"; + } + + bool required = false; + Config& config = Config::GetInstance(); + config.GetServerRestartRequired(required); + + nlohmann::json result; + result["code"] = StatusCode::SUCCESS; + result["message"] = msg; + result["restart_required"] = required; + + result_str = result.dump(); + + return Status::OK(); +} + +Status +WebRequestHandler::Search(const std::string& table_name, const nlohmann::json& json, std::string& result_str) { + if (!json.contains("topk")) { + return Status(BODY_FIELD_LOSS, "Field \'topk\' is required"); + } + int64_t topk = json["topk"]; + + if (!json.contains("nprobe")) { + return Status(BODY_FIELD_LOSS, "Field \'nprobe\' is required"); + } + int64_t nprobe = json["nprobe"]; + + std::vector partition_tags; + if (json.contains("partition_tags")) { + auto tags = json["partition_tags"]; + if (!tags.is_null() && !tags.is_array()) { + return Status(BODY_PARSE_FAIL, "Field \"partition_tags\" must be a array"); + } + + for (auto& tag : tags) { + partition_tags.emplace_back(tag.get()); + } + } + + TopKQueryResult result; + if (json.contains("vector_id")) { + auto vec_id = json["vector_id"].get(); + auto status = + request_handler_.SearchByID(context_ptr_, table_name, vec_id, topk, nprobe, partition_tags, result); + if (!status.ok()) { + return status; + } + } else { + std::vector file_id_vec; + if (json.contains("file_ids")) { + auto ids = json["file_ids"]; + if (!ids.is_null() && !ids.is_array()) { + return Status(BODY_PARSE_FAIL, "Field \"file_ids\" must be a array"); + } + for (auto& id : ids) { + file_id_vec.emplace_back(id.get()); + } + } + + bool bin_flag = false; + auto status = IsBinaryTable(table_name, bin_flag); + if (!status.ok()) { + return status; + } + + if (!json.contains("vectors")) { + return Status(BODY_FIELD_LOSS, "Field \"vectors\" is required"); + } + + engine::VectorsData vectors_data; + status = CopyRecordsFromJson(json["vectors"], vectors_data, bin_flag); + if (!status.ok()) { + return status; + } + + status = request_handler_.Search(context_ptr_, table_name, vectors_data, topk, nprobe, partition_tags, + file_id_vec, result); + if (!status.ok()) { + return status; + } + } + + nlohmann::json result_json; + result_json["num"] = result.row_num_; + if (result.row_num_ == 0) { + result_json["result"] = std::vector(); + result_str = result_json.dump(); + return Status::OK(); + } + + auto step = result.id_list_.size() / result.row_num_; + nlohmann::json search_result_json; + for (size_t i = 0; i < result.row_num_; i++) { + nlohmann::json raw_result_json; + for (size_t j = 0; j < step; j++) { + nlohmann::json one_result_json; + one_result_json["id"] = std::to_string(result.id_list_.at(i * step + j)); + one_result_json["distance"] = std::to_string(result.distance_list_.at(i * step + j)); + raw_result_json.emplace_back(one_result_json); + } + search_result_json.emplace_back(raw_result_json); + } + result_json["result"] = search_result_json; + result_str = result_json.dump(); + + return Status::OK(); +} + +Status +WebRequestHandler::DeleteByIDs(const std::string& table_name, const nlohmann::json& json, std::string& result_str) { + std::vector vector_ids; + if (!json.contains("ids")) { + return Status(BODY_FIELD_LOSS, "Field \"delete\" must contains \"ids\""); + } + auto ids = json["ids"]; + if (!ids.is_array()) { + return Status(BODY_FIELD_LOSS, "\"ids\" must be an array"); + } + + for (auto& id : ids) { + vector_ids.emplace_back(id.get()); + } + + auto status = request_handler_.DeleteByID(context_ptr_, table_name, vector_ids); + if (status.ok()) { + nlohmann::json result_json; + result_json["code"] = status.code(); + result_json["message"] = status.message(); + result_str = result_json.dump(); + } + + return status; +} + +Status +WebRequestHandler::GetVectorsByIDs(const std::string& table_name, const std::vector& ids, + nlohmann::json& json_out) { + std::vector vector_batch; + for (size_t i = 0; i < ids.size(); i++) { + auto vec_ids = std::vector(ids.begin() + i, ids.begin() + i + 1); + engine::VectorsData vectors_data; + auto status = request_handler_.GetVectorByID(context_ptr_, table_name, ids, vectors_data); + if (!status.ok()) { + return status; + } + vector_batch.push_back(vectors_data); + } + + bool bin; + auto status = IsBinaryTable(table_name, bin); + if (!status.ok()) { + return status; + } + + nlohmann::json vectors_json; + for (size_t i = 0; i < vector_batch.size(); i++) { + nlohmann::json vector_json; + if (bin) { + vector_json["vector"] = vector_batch.at(i).binary_data_; + } else { + vector_json["vector"] = vector_batch.at(i).float_data_; + } + vector_json["id"] = std::to_string(ids[i]); + json_out.push_back(vector_json); + } + + return Status::OK(); +} + +////////////////////////////////// Router methods //////////////////////////////////////////// + +/************ + * Device { + * + */ StatusDto::ObjectWrapper WebRequestHandler::GetDevices(DevicesDto::ObjectWrapper& devices_dto) { auto system_info = SystemInfo::GetInstance(); @@ -122,7 +604,6 @@ WebRequestHandler::GetDevices(DevicesDto::ObjectWrapper& devices_dto) { devices_dto->gpus = devices_dto->gpus->createShared(); #ifdef MILVUS_GPU_VERSION - size_t count = system_info.num_device(); std::vector device_mems = system_info.GPUMemoryTotal(); @@ -135,7 +616,6 @@ WebRequestHandler::GetDevices(DevicesDto::ObjectWrapper& devices_dto) { device_dto->memory = device_mems.at(i) >> 30; devices_dto->gpus->put("GPU" + OString(std::to_string(i).c_str()), device_dto); } - #endif ASSIGN_RETURN_STATUS_DTO(Status::OK()); @@ -241,7 +721,6 @@ WebRequestHandler::SetAdvancedConfig(const AdvancedConfigDto::ObjectWrapper& adv } #ifdef MILVUS_GPU_VERSION - StatusDto::ObjectWrapper WebRequestHandler::GetGpuConfig(GPUConfigDto::ObjectWrapper& gpu_config_dto) { std::string reply; @@ -295,10 +774,6 @@ WebRequestHandler::GetGpuConfig(GPUConfigDto::ObjectWrapper& gpu_config_dto) { ASSIGN_RETURN_STATUS_DTO(Status::OK()); } -#endif - -#ifdef MILVUS_GPU_VERSION - StatusDto::ObjectWrapper WebRequestHandler::SetGpuConfig(const GPUConfigDto::ObjectWrapper& gpu_config_dto) { // Step 1: Check config param @@ -382,9 +857,12 @@ WebRequestHandler::SetGpuConfig(const GPUConfigDto::ObjectWrapper& gpu_config_dt ASSIGN_RETURN_STATUS_DTO(Status::OK()); } - #endif +/************* + * + * Table { + */ StatusDto::ObjectWrapper WebRequestHandler::CreateTable(const TableRequestDto::ObjectWrapper& table_schema) { if (nullptr == table_schema->table_name.get()) { @@ -415,25 +893,10 @@ WebRequestHandler::CreateTable(const TableRequestDto::ObjectWrapper& table_schem } StatusDto::ObjectWrapper -WebRequestHandler::GetTable(const OString& table_name, const OQueryParams& query_params, - TableFieldsDto::ObjectWrapper& fields_dto) { - if (nullptr == table_name.get()) { - RETURN_STATUS_DTO(PATH_PARAM_LOSS, "Path param \'table_name\' is required!"); - } - - // TODO: query string field `fields` npt used here - auto status = GetTableInfo(table_name->std_str(), fields_dto); - - ASSIGN_RETURN_STATUS_DTO(status); -} - -StatusDto::ObjectWrapper -WebRequestHandler::ShowTables(const OString& offset, const OString& page_size, - TableListFieldsDto::ObjectWrapper& response_dto) { +WebRequestHandler::ShowTables(const OQueryParams& query_params, OString& result) { int64_t offset_value = 0; - int64_t page_size_value = 10; - - if (nullptr != offset.get()) { + auto offset = query_params.get("offset"); + if (nullptr != offset.get() && offset->getSize() > 0) { std::string offset_str = offset->std_str(); if (!ValidationUtil::ValidateStringIsNumber(offset_str).ok()) { RETURN_STATUS_DTO(ILLEGAL_QUERY_PARAM, @@ -442,7 +905,9 @@ WebRequestHandler::ShowTables(const OString& offset, const OString& page_size, offset_value = std::stol(offset_str); } - if (nullptr != page_size.get()) { + int64_t page_size_value = 10; + auto page_size = query_params.get("page_size"); + if (nullptr != page_size.get() && page_size->getSize() > 0) { std::string page_size_str = page_size->std_str(); if (!ValidationUtil::ValidateStringIsNumber(page_size_str).ok()) { RETURN_STATUS_DTO(ILLEGAL_QUERY_PARAM, @@ -455,34 +920,79 @@ WebRequestHandler::ShowTables(const OString& offset, const OString& page_size, RETURN_STATUS_DTO(ILLEGAL_QUERY_PARAM, "Query param 'offset' or 'page_size' should equal or bigger than 0"); } + bool all_required = false; + auto required = query_params.get("all_required"); + if (nullptr != required.get()) { + auto required_str = required->std_str(); + if (!ValidationUtil::ValidateStringIsBool(required_str).ok()) { + RETURN_STATUS_DTO(ILLEGAL_QUERY_PARAM, "Query param \'all_required\' must be a bool") + } + all_required = required_str == "True" || required_str == "true"; + } + std::vector tables; auto status = request_handler_.ShowTables(context_ptr_, tables); if (!status.ok()) { ASSIGN_RETURN_STATUS_DTO(status) } - response_dto->tables = response_dto->tables->createShared(); - - if (offset_value >= tables.size()) { - ASSIGN_RETURN_STATUS_DTO(Status::OK()); + if (all_required) { + offset_value = 0; + page_size_value = tables.size(); + } else { + offset_value = std::min((size_t)offset_value, tables.size()); + page_size_value = std::min(tables.size() - offset_value, (size_t)page_size_value); } - response_dto->count = tables.size(); - - int64_t size = page_size_value + offset_value > tables.size() ? tables.size() - offset_value : page_size_value; - for (int64_t i = offset_value; i < size + offset_value; i++) { - auto table_fields_dto = TableFieldsDto::createShared(); - status = GetTableInfo(tables.at(i), table_fields_dto); + nlohmann::json tables_json; + for (int64_t i = offset_value; i < page_size_value + offset_value; i++) { + nlohmann::json table_json; + status = GetTableMetaInfo(tables.at(i), table_json); if (!status.ok()) { - break; + ASSIGN_RETURN_STATUS_DTO(status) } - - response_dto->tables->pushBack(table_fields_dto); + tables_json.push_back(table_json); } + nlohmann::json result_json; + result_json["count"] = tables.size(); + if (tables_json.empty()) { + result_json["tables"] = std::vector(); + } else { + result_json["tables"] = tables_json; + } + + result = result_json.dump().c_str(); + ASSIGN_RETURN_STATUS_DTO(status) } +StatusDto::ObjectWrapper +WebRequestHandler::GetTable(const OString& table_name, const OQueryParams& query_params, OString& result) { + if (nullptr == table_name.get()) { + RETURN_STATUS_DTO(PATH_PARAM_LOSS, "Path param \'table_name\' is required!"); + } + + auto status = Status::OK(); + + auto stat = query_params.get("info"); + if (nullptr != stat.get() && stat->std_str() == "stat") { + nlohmann::json json; + status = GetTableStat(table_name->std_str(), json); + if (status.ok()) { + result = json.dump().c_str(); + } + } else { + nlohmann::json json; + status = GetTableMetaInfo(table_name->std_str(), json); + if (status.ok()) { + result = json.dump().c_str(); + } + } + + ASSIGN_RETURN_STATUS_DTO(status); +} + StatusDto::ObjectWrapper WebRequestHandler::DropTable(const OString& table_name) { auto status = request_handler_.DropTable(context_ptr_, table_name->std_str()); @@ -490,6 +1000,11 @@ WebRequestHandler::DropTable(const OString& table_name) { ASSIGN_RETURN_STATUS_DTO(status) } +/*********** + * + * Index { + */ + StatusDto::ObjectWrapper WebRequestHandler::CreateIndex(const OString& table_name, const IndexRequestDto::ObjectWrapper& index_param) { if (nullptr == index_param->index_type.get()) { @@ -530,6 +1045,10 @@ WebRequestHandler::DropIndex(const OString& table_name) { ASSIGN_RETURN_STATUS_DTO(status) } +/*********** + * + * Partition { + */ StatusDto::ObjectWrapper WebRequestHandler::CreatePartition(const OString& table_name, const PartitionRequestDto::ObjectWrapper& param) { if (nullptr == param->partition_tag.get()) { @@ -543,11 +1062,181 @@ WebRequestHandler::CreatePartition(const OString& table_name, const PartitionReq } StatusDto::ObjectWrapper -WebRequestHandler::ShowPartitions(const OString& offset, const OString& page_size, const OString& table_name, +WebRequestHandler::ShowPartitions(const OString& table_name, const OQueryParams& query_params, PartitionListDto::ObjectWrapper& partition_list_dto) { int64_t offset_value = 0; - int64_t page_size_value = 10; + auto offset = query_params.get("offset"); + if (nullptr != offset.get() && offset->getSize() > 0) { + std::string offset_str = offset->std_str(); + if (!ValidationUtil::ValidateStringIsNumber(offset_str).ok()) { + RETURN_STATUS_DTO(ILLEGAL_QUERY_PARAM, + "Query param \'offset\' is illegal, only non-negative integer supported"); + } + offset_value = std::stol(offset_str); + } + int64_t page_size_value = 10; + auto page_size = query_params.get("page_size"); + if (nullptr != page_size.get() && page_size->getSize() > 0) { + std::string page_size_str = page_size->std_str(); + if (!ValidationUtil::ValidateStringIsNumber(page_size_str).ok()) { + RETURN_STATUS_DTO(ILLEGAL_QUERY_PARAM, + "Query param \'page_size\' is illegal, only non-negative integer supported"); + } + page_size_value = std::stol(page_size_str); + } + + if (offset_value < 0 || page_size_value < 0) { + ASSIGN_RETURN_STATUS_DTO( + Status(SERVER_UNEXPECTED_ERROR, "Query param 'offset' or 'page_size' should equal or bigger than 0")); + } + + bool all_required = false; + auto required = query_params.get("all_required"); + if (nullptr != required.get()) { + auto required_str = required->std_str(); + if (!ValidationUtil::ValidateStringIsBool(required_str).ok()) { + RETURN_STATUS_DTO(ILLEGAL_QUERY_PARAM, "Query param \'all_required\' must be a bool") + } + all_required = required_str == "True" || required_str == "true"; + } + + std::vector partitions; + auto status = request_handler_.ShowPartitions(context_ptr_, table_name->std_str(), partitions); + if (!status.ok()) { + ASSIGN_RETURN_STATUS_DTO(status) + } + + if (all_required) { + offset_value = 0; + page_size_value = partitions.size(); + } else { + offset_value = std::min((size_t)offset_value, partitions.size()); + page_size_value = std::min(partitions.size() - offset_value, (size_t)page_size_value); + } + + partition_list_dto->count = partitions.size(); + partition_list_dto->partitions = partition_list_dto->partitions->createShared(); + + if (offset_value < partitions.size()) { + for (int64_t i = offset_value; i < page_size_value + offset_value; i++) { + auto partition_dto = PartitionFieldsDto::createShared(); + partition_dto->partition_tag = partitions.at(i).tag_.c_str(); + partition_list_dto->partitions->pushBack(partition_dto); + } + } + + ASSIGN_RETURN_STATUS_DTO(status) +} + +StatusDto::ObjectWrapper +WebRequestHandler::DropPartition(const OString& table_name, const OString& body) { + std::string tag; + try { + auto json = nlohmann::json::parse(body->std_str()); + tag = json["partition_tag"].get(); + } catch (nlohmann::detail::parse_error& e) { + RETURN_STATUS_DTO(BODY_PARSE_FAIL, e.what()) + } catch (nlohmann::detail::type_error& e) { + RETURN_STATUS_DTO(BODY_PARSE_FAIL, e.what()) + } + auto status = request_handler_.DropPartition(context_ptr_, table_name->std_str(), tag); + + ASSIGN_RETURN_STATUS_DTO(status) +} + +/*********** + * + * Segment { + */ +StatusDto::ObjectWrapper +WebRequestHandler::ShowSegments(const OString& table_name, const OQueryParams& query_params, OString& response) { + int64_t offset_value = 0; + auto offset = query_params.get("offset"); + if (nullptr != offset.get() && offset->getSize() > 0) { + std::string offset_str = offset->std_str(); + if (!ValidationUtil::ValidateStringIsNumber(offset_str).ok()) { + RETURN_STATUS_DTO(ILLEGAL_QUERY_PARAM, + "Query param \'offset\' is illegal, only non-negative integer supported"); + } + offset_value = std::stol(offset_str); + } + + int64_t page_size_value = 10; + auto page_size = query_params.get("page_size"); + if (nullptr != page_size.get() && page_size->getSize() > 0) { + std::string page_size_str = page_size->std_str(); + if (!ValidationUtil::ValidateStringIsNumber(page_size_str).ok()) { + RETURN_STATUS_DTO(ILLEGAL_QUERY_PARAM, + "Query param \'page_size\' is illegal, only non-negative integer supported"); + } + page_size_value = std::stol(page_size_str); + } + + if (offset_value < 0 || page_size_value < 0) { + RETURN_STATUS_DTO(ILLEGAL_QUERY_PARAM, "Query param 'offset' or 'page_size' should equal or bigger than 0"); + } + + std::string tag; + if (nullptr != query_params.get("partition_tag").get()) { + tag = query_params.get("partition_tag")->std_str(); + } + + struct TableInfo info; + auto status = request_handler_.ShowTableInfo(context_ptr_, table_name->std_str(), info); + if (!status.ok()) { + ASSIGN_RETURN_STATUS_DTO(status) + } + + typedef std::pair Pair; + std::vector segments; + for (auto& par_stat : info.partitions_stat_) { + if (!tag.empty() && tag != par_stat.tag_) { + continue; + } + for (auto& seg_stat : par_stat.segments_stat_) { + auto segment_stat = std::pair(par_stat.tag_, seg_stat); + segments.push_back(segment_stat); + } + } + + int64_t size = segments.size(); + auto iter_begin = std::min(size, offset_value); + auto iter_end = std::min(size, offset_value + page_size_value); + + auto segments_out = std::vector(segments.begin() + iter_begin, segments.begin() + iter_end); + + // sort with segment name + auto compare = [](Pair& a, Pair& b) -> bool { return a.second.name_ >= b.second.name_; }; + std::sort(segments_out.begin(), segments_out.end(), compare); + + nlohmann::json result_json; + if (segments_out.empty()) { + result_json["segments"] = std::vector(); + } else { + nlohmann::json segs_json; + for (auto& s : segments_out) { + nlohmann::json seg_json; + ParseSegmentStat(s.second, seg_json); + seg_json["partition_tag"] = s.first; + segs_json.push_back(seg_json); + } + result_json["segments"] = segs_json; + } + + result_json["count"] = size; + AddStatusToJson(result_json, status.code(), status.message()); + + response = result_json.dump().c_str(); + + ASSIGN_RETURN_STATUS_DTO(status) +} + +StatusDto::ObjectWrapper +WebRequestHandler::GetSegmentInfo(const OString& table_name, const OString& segment_name, const OString& info, + const OQueryParams& query_params, OString& result) { + int64_t offset_value = 0; + auto offset = query_params.get("offset"); if (nullptr != offset.get()) { std::string offset_str = offset->std_str(); if (!ValidationUtil::ValidateStringIsNumber(offset_str).ok()) { @@ -557,6 +1246,8 @@ WebRequestHandler::ShowPartitions(const OString& offset, const OString& page_siz offset_value = std::stol(offset_str); } + int64_t page_size_value = 10; + auto page_size = query_params.get("page_size"); if (nullptr != page_size.get()) { std::string page_size_str = page_size->std_str(); if (!ValidationUtil::ValidateStringIsNumber(page_size_str).ok()) { @@ -571,78 +1262,74 @@ WebRequestHandler::ShowPartitions(const OString& offset, const OString& page_siz Status(SERVER_UNEXPECTED_ERROR, "Query param 'offset' or 'page_size' should equal or bigger than 0")); } - std::vector partitions; - auto status = request_handler_.ShowPartitions(context_ptr_, table_name->std_str(), partitions); - if (!status.ok()) { - ASSIGN_RETURN_STATUS_DTO(status) + std::string re = info->std_str(); + auto status = Status::OK(); + nlohmann::json json; + // Get vectors + if (re == "vectors") { + status = GetSegmentVectors(table_name->std_str(), segment_name->std_str(), page_size_value, offset_value, json); + // Get vector ids + } else if (re == "ids") { + status = GetSegmentIds(table_name->std_str(), segment_name->std_str(), page_size_value, offset_value, json); } - partition_list_dto->count = partitions.size(); - partition_list_dto->partitions = partition_list_dto->partitions->createShared(); - - if (offset_value < partitions.size()) { - int64_t size = - offset_value + page_size_value > partitions.size() ? partitions.size() - offset_value : page_size_value; - for (int64_t i = offset_value; i < size + offset_value; i++) { - auto partition_dto = PartitionFieldsDto::createShared(); - partition_dto->partition_tag = partitions.at(i).tag_.c_str(); - partition_list_dto->partitions->pushBack(partition_dto); - } - } - - ASSIGN_RETURN_STATUS_DTO(status) -} - -StatusDto::ObjectWrapper -WebRequestHandler::DropPartition(const OString& table_name, const OString& tag) { - auto status = request_handler_.DropPartition(context_ptr_, table_name->std_str(), tag->std_str()); - - ASSIGN_RETURN_STATUS_DTO(status) -} - -StatusDto::ObjectWrapper -WebRequestHandler::Insert(const OString& table_name, const InsertRequestDto::ObjectWrapper& request, - VectorIdsDto::ObjectWrapper& ids_dto) { - TableSchema schema; - auto status = request_handler_.DescribeTable(context_ptr_, table_name->std_str(), schema); - if (!status.ok()) { - ASSIGN_RETURN_STATUS_DTO(status) - } - - auto metric = engine::MetricType(schema.metric_type_); - engine::VectorsData vectors; - bool bin_flag = engine::MetricType::HAMMING == metric || engine::MetricType::JACCARD == metric || - engine::MetricType::TANIMOTO == metric; - - if (!bin_flag) { - if (nullptr == request->records.get()) { - RETURN_STATUS_DTO(BODY_FIELD_LOSS, "Field \'records\' is required to fill vectors"); - } - vectors.vector_count_ = request->records->count(); - status = CopyRowRecords(request->records, vectors.float_data_); + if (status.ok()) { + result = json.dump().c_str(); } else { - if (nullptr == request->records_bin.get()) { - RETURN_STATUS_DTO(BODY_FIELD_LOSS, "Field \'records_bin\' is required to fill vectors"); - } - vectors.vector_count_ = request->records_bin->count(); - status = CopyBinRowRecords(request->records_bin, vectors.binary_data_); + result = "NULL"; } + ASSIGN_RETURN_STATUS_DTO(status) +} + +/********** + * + * Vector { + */ +StatusDto::ObjectWrapper +WebRequestHandler::Insert(const OString& table_name, const OString& body, VectorIdsDto::ObjectWrapper& ids_dto) { + if (nullptr == body.get() || body->getSize() == 0) { + RETURN_STATUS_DTO(BODY_FIELD_LOSS, "Request payload is required.") + } + + // step 1: copy vectors + bool bin_flag; + auto status = IsBinaryTable(table_name->std_str(), bin_flag); + if (!status.ok()) { + ASSIGN_RETURN_STATUS_DTO(status) + } + + auto body_json = nlohmann::json::parse(body->std_str()); + if (!body_json.contains("vectors")) { + RETURN_STATUS_DTO(BODY_FIELD_LOSS, "Field \'vectors\' is required"); + } + engine::VectorsData vectors; + CopyRecordsFromJson(body_json["vectors"], vectors, bin_flag); if (!status.ok()) { ASSIGN_RETURN_STATUS_DTO(status) } // step 2: copy id array - if (nullptr != request->ids.get()) { + if (body_json.contains("ids")) { + auto& ids_json = body_json["ids"]; + if (!ids_json.is_array()) { + RETURN_STATUS_DTO(ILLEGAL_BODY, "Field \"ids\" must be a array"); + } auto& id_array = vectors.id_array_; - id_array.resize(request->ids->count()); - - size_t i = 0; - request->ids->forEach([&id_array, &i](const OInt64& item) { id_array[i++] = item->getValue(); }); + id_array.clear(); + for (auto& id : ids_json) { + id_array.emplace_back(id.get()); + } } - status = request_handler_.Insert(context_ptr_, table_name->std_str(), vectors, request->tag->std_str()); + // step 3: copy partition tag + std::string tag; + if (body_json.contains("partition_tag")) { + tag = body_json["partition_tag"]; + } + // step 4: construct result + status = request_handler_.Insert(context_ptr_, table_name->std_str(), vectors, tag); if (status.ok()) { ids_dto->ids = ids_dto->ids->createShared(); for (auto& id : vectors.id_array_) { @@ -654,213 +1341,93 @@ WebRequestHandler::Insert(const OString& table_name, const InsertRequestDto::Obj } StatusDto::ObjectWrapper -WebRequestHandler::Search(const OString& table_name, const SearchRequestDto::ObjectWrapper& request, - TopkResultsDto::ObjectWrapper& results_dto) { - if (nullptr == request->topk.get()) { - RETURN_STATUS_DTO(BODY_FIELD_LOSS, "Field \'topk\' is required in request body") +WebRequestHandler::GetVector(const OString& table_name, const OQueryParams& query_params, OString& response) { + auto id_str = query_params.get("id"); + if (id_str.get() == nullptr) { + RETURN_STATUS_DTO(QUERY_PARAM_LOSS, "Need to specify vector id in query string") } - int64_t topk_t = request->topk->getValue(); - - if (nullptr == request->nprobe.get()) { - RETURN_STATUS_DTO(BODY_FIELD_LOSS, "Field \'nprobe\' is required in request body") - } - int64_t nprobe_t = request->nprobe->getValue(); - - std::vector tag_list; - if (nullptr != request->tags.get()) { - request->tags->forEach([&tag_list](const OString& tag) { tag_list.emplace_back(tag->std_str()); }); + if (!ValidationUtil::ValidateStringIsNumber(id_str->std_str()).ok()) { + RETURN_STATUS_DTO(ILLEGAL_QUERY_PARAM, "Query param \'id\' is illegal, only non-negative integer supported"); } - std::vector file_id_list; - if (nullptr != request->file_ids.get()) { - request->file_ids->forEach([&file_id_list](const OString& id) { file_id_list.emplace_back(id->std_str()); }); - } + auto id = std::stol(id_str->c_str()); - TableSchema schema; - auto status = request_handler_.DescribeTable(context_ptr_, table_name->std_str(), schema); - if (!status.ok()) { - ASSIGN_RETURN_STATUS_DTO(status) - } - - auto metric = engine::MetricType(schema.metric_type_); - bool bin_flag = engine::MetricType::HAMMING == metric || engine::MetricType::JACCARD == metric || - engine::MetricType::TANIMOTO == metric; + std::vector ids = {id}; engine::VectorsData vectors; + nlohmann::json vectors_json; + auto status = GetVectorsByIDs(table_name->std_str(), ids, vectors_json); + if (!status.ok()) { + response = "NULL"; + ASSIGN_RETURN_STATUS_DTO(status) + } - if (!bin_flag) { - if (nullptr == request->records.get()) { - RETURN_STATUS_DTO(BODY_FIELD_LOSS, "Field \'records\' is required to fill vectors"); - } - vectors.vector_count_ = request->records->count(); - status = CopyRowRecords(request->records, vectors.float_data_); + nlohmann::json json; + json["code"] = status.code(); + json["message"] = status.message(); + if (vectors_json.empty()) { + json["vectors"] = std::vector(); } else { - if (nullptr == request->records_bin.get()) { - RETURN_STATUS_DTO(BODY_FIELD_LOSS, "Field \'records_bin\' is required to fill vectors"); - } - vectors.vector_count_ = request->records_bin->count(); - status = CopyBinRowRecords(request->records_bin, vectors.binary_data_); + json["vectors"] = vectors_json; } - if (!status.ok()) { - ASSIGN_RETURN_STATUS_DTO(status) - } - - TopKQueryResult result; - auto context_ptr = GenContextPtr("Web Handler"); - status = request_handler_.Search(context_ptr, table_name->std_str(), vectors, topk_t, nprobe_t, tag_list, - file_id_list, result); - if (!status.ok()) { - ASSIGN_RETURN_STATUS_DTO(status) - } - - results_dto->num = result.row_num_; - results_dto->results = results_dto->results->createShared(); - if (0 == result.row_num_) { - ASSIGN_RETURN_STATUS_DTO(status) - } - - auto step = result.id_list_.size() / result.row_num_; - for (size_t i = 0; i < result.row_num_; i++) { - auto row_result_dto = OList::createShared(); - for (size_t j = 0; j < step; j++) { - auto result_dto = ResultDto::createShared(); - result_dto->id = std::to_string(result.id_list_.at(i * step + j)).c_str(); - result_dto->dit = std::to_string(result.distance_list_.at(i * step + j)).c_str(); - row_result_dto->pushBack(result_dto); - } - results_dto->results->pushBack(row_result_dto); - } + response = json.dump().c_str(); ASSIGN_RETURN_STATUS_DTO(status) } StatusDto::ObjectWrapper -WebRequestHandler::SystemInfo(const OString& cmd, CommandDto::ObjectWrapper& cmd_dto) { - std::string info = cmd->std_str(); +WebRequestHandler::VectorsOp(const OString& table_name, const OString& payload, OString& response) { auto status = Status::OK(); - std::string reply_str; + std::string result_str; - if ("config" == info) { - info = "get_config *"; - status = CommandLine(info, reply_str); - if (status.ok()) { - try { - nlohmann::json j = nlohmann::json::parse(reply_str); -#ifdef MILVUS_GPU_VERSION - auto gpu_search_res = j["gpu_resource_config"]["search_resources"].get(); - std::vector gpus; - StringHelpFunctions::SplitStringByDelimeter(gpu_search_res, ",", gpus); - j["gpu_resource_config"]["search_resources"] = gpus; + try { + nlohmann::json payload_json = nlohmann::json::parse(payload->std_str()); - auto gpu_build_res = j["gpu_resource_config"]["build_index_resources"].get(); - gpus.clear(); - StringHelpFunctions::SplitStringByDelimeter(gpu_build_res, ",", gpus); - j["gpu_resource_config"]["build_index_resources"] = gpus; -#endif - // check if server require start - Config& config = Config::GetInstance(); - bool required = false; - config.GetServerRestartRequired(required); - j["restart_required"] = required; - cmd_dto->reply = j.dump().c_str(); - } catch (std::exception& e) { - RETURN_STATUS_DTO(UNEXPECTED_ERROR, e.what()); - } - } - } else { - if ("info" == info) { - info = "get_system_info"; - } - status = CommandLine(info, reply_str); - - if (status.ok()) { - cmd_dto->reply = reply_str.c_str(); + if (payload_json.contains("delete")) { + status = DeleteByIDs(table_name->std_str(), payload_json["delete"], result_str); + } else if (payload_json.contains("search")) { + status = Search(table_name->std_str(), payload_json["search"], result_str); + } else { + status = Status(ILLEGAL_BODY, "Unknown body"); } + } catch (nlohmann::detail::parse_error& e) { + std::string emsg = "json error: code=" + std::to_string(e.id) + ", reason=" + e.what(); + RETURN_STATUS_DTO(BODY_PARSE_FAIL, emsg.c_str()); + } catch (nlohmann::detail::type_error& e) { + std::string emsg = "json error: code=" + std::to_string(e.id) + ", reason=" + e.what(); + RETURN_STATUS_DTO(BODY_PARSE_FAIL, emsg.c_str()); + } catch (std::exception& e) { + RETURN_STATUS_DTO(SERVER_UNEXPECTED_ERROR, e.what()); } - ASSIGN_RETURN_STATUS_DTO(status); + if (status.ok()) { + response = result_str.c_str(); + } else { + response = "NULL"; + } + + ASSIGN_RETURN_STATUS_DTO(status) } +/********** + * + * System { + */ StatusDto::ObjectWrapper -WebRequestHandler::SystemOp(const OString& op, const OString& body_str, OString& response_str) { - Status status = Status::OK(); +WebRequestHandler::SystemInfo(const OString& cmd, const OQueryParams& query_params, OString& response_str) { + std::string info = cmd->std_str(); + + auto status = Status::OK(); + std::string result_str; - if (nullptr == body_str.get() || body_str->getSize() == 0) { - RETURN_STATUS_DTO(BODY_FIELD_LOSS, "Payload is empty."); - } try { - nlohmann::json j = nlohmann::json::parse(body_str->c_str()); - if (op->equals("task")) { - if (j.contains("load")) { - auto table_name = j["load"]["table_name"]; - if (!table_name.is_string()) { - RETURN_STATUS_DTO(ILLEGAL_BODY, "\"table_name\" must be a string") - } - status = request_handler_.PreloadTable(context_ptr_, table_name.get()); - } - if (j.contains("flush")) { - auto table_names = j["flush"]["table_names"]; - if (!table_names.is_array()) { - RETURN_STATUS_DTO(ILLEGAL_BODY, "\"table_names\" must be a array") - } - std::vector names; - for (auto& n : table_names) { - if (!n.is_string()) { - RETURN_STATUS_DTO(ILLEGAL_BODY, "item of \"table_names\" must be a string") - } - names.push_back(n.get()); - } - status = Status(SERVER_UNEXPECTED_ERROR, "Flush() is not implemented"); - // status = request_handler_.Flush(context_ptr_, table_names); - } - if (j.contains("compact")) { - auto table_names = j["compact"]["table_names"]; - if (!table_names.is_array()) { - RETURN_STATUS_DTO(ILLEGAL_BODY, "\"table_name\" must be a array") - } - std::vector names; - for (auto& n : table_names) { - names.push_back(n.get()); - } - status = Status(SERVER_UNEXPECTED_ERROR, "Compact() is not implemented"); - // status = request_handler_.Compact(context_ptr_, table_names); - } - } else if (op->equals("config")) { - if (!j.is_object()) { - RETURN_STATUS_DTO(ILLEGAL_BODY, "Error format") - } - - std::vector cmds; - for (auto& el : j.items()) { - auto evalue = el.value(); - if (!evalue.is_object()) { - RETURN_STATUS_DTO(ILLEGAL_BODY, "Invalid payload format, the root value must be json map"); - } - - for (auto& iel : el.value().items()) { - auto ievalue = iel.value(); - if (!(ievalue.is_string() || ievalue.is_number() || ievalue.is_boolean())) { - RETURN_STATUS_DTO(ILLEGAL_BODY, "Config value must be one of string, numeric or boolean") - } - std::ostringstream ss; - if (ievalue.is_string()) { - std::string vle = ievalue; - ss << "set_config " << el.key() << "." << iel.key() << " " << vle; - } else { - ss << "set_config " << el.key() << "." << iel.key() << " " << ievalue; - } - std::string cmd = ss.str(); - cmds.push_back(cmd); - } - } - - std::string reply; - for (auto& c : cmds) { - status = CommandLine(c, reply); - if (!status.ok()) { - ASSIGN_RETURN_STATUS_DTO(status) - } + if (info == "config") { + status = GetConfig(result_str); + } else { + if ("info" == info) { + info = "get_system_info"; } + status = Cmd(info, result_str); } } catch (nlohmann::detail::parse_error& e) { std::string emsg = "json error: code=" + std::to_string(e.id) + ", reason=" + e.what(); @@ -870,16 +1437,52 @@ WebRequestHandler::SystemOp(const OString& op, const OString& body_str, OString& RETURN_STATUS_DTO(ILLEGAL_BODY, emsg.c_str()); } - nlohmann::json j = nlohmann::json(); - j["code"] = status.code(); - j["message"] = status.message(); - if (op->equals("config")) { - Config& config = Config::GetInstance(); - bool required = false; - config.GetServerRestartRequired(required); - j["restart_required"] = required; + if (status.ok()) { + response_str = result_str.c_str(); + } else { + response_str = "NULL"; + } + + ASSIGN_RETURN_STATUS_DTO(status); +} + +StatusDto::ObjectWrapper +WebRequestHandler::SystemOp(const OString& op, const OString& body_str, OString& response_str) { + if (nullptr == body_str.get() || body_str->getSize() == 0) { + RETURN_STATUS_DTO(BODY_FIELD_LOSS, "Payload is empty."); + } + + Status status = Status::OK(); + std::string result_str; + try { + nlohmann::json j = nlohmann::json::parse(body_str->c_str()); + if (op->equals("task")) { + if (j.contains("load")) { + status = PreLoadTable(j["load"], result_str); + } else if (j.contains("flush")) { + status = Flush(j["flush"], result_str); + } + if (j.contains("compact")) { + status = Compact(j["compact"], result_str); + } + } else if (op->equals("config")) { + SetConfig(j, result_str); + } else { + status = Status(UNKNOWN_PATH, "Unknown path: /system/" + op->std_str()); + } + } catch (nlohmann::detail::parse_error& e) { + std::string emsg = "json error: code=" + std::to_string(e.id) + ", reason=" + e.what(); + RETURN_STATUS_DTO(ILLEGAL_BODY, emsg.c_str()); + } catch (nlohmann::detail::type_error& e) { + std::string emsg = "json error: code=" + std::to_string(e.id) + ", reason=" + e.what(); + RETURN_STATUS_DTO(ILLEGAL_BODY, emsg.c_str()); + } + + if (status.ok()) { + response_str = result_str.c_str(); + } else { + response_str = "NULL"; } - response_str = j.dump().c_str(); ASSIGN_RETURN_STATUS_DTO(status); } diff --git a/core/src/server/web_impl/handler/WebRequestHandler.h b/core/src/server/web_impl/handler/WebRequestHandler.h index 24601e9096..3ee9cab230 100644 --- a/core/src/server/web_impl/handler/WebRequestHandler.h +++ b/core/src/server/web_impl/handler/WebRequestHandler.h @@ -15,12 +15,16 @@ #include #include #include +#include #include #include #include #include +#include "db/Types.h" +#include "server/context/Context.h" +#include "server/delivery/RequestHandler.h" #include "server/web_impl/Types.h" #include "server/web_impl/dto/CmdDto.hpp" #include "server/web_impl/dto/ConfigDto.hpp" @@ -29,10 +33,7 @@ #include "server/web_impl/dto/PartitionDto.hpp" #include "server/web_impl/dto/TableDto.hpp" #include "server/web_impl/dto/VectorDto.hpp" - -#include "db/Types.h" -#include "server/context/Context.h" -#include "server/delivery/RequestHandler.h" +#include "thirdparty/nlohmann/json.hpp" #include "utils/Status.h" namespace milvus { @@ -76,19 +77,74 @@ class WebRequestHandler { return context_ptr; } + private: + void + AddStatusToJson(nlohmann::json& json, int64_t code, const std::string& msg); + + Status + ParseSegmentStat(const SegmentStat& seg_stat, nlohmann::json& json); + + Status + ParsePartitionStat(const PartitionStat& par_stat, nlohmann::json& json); + + Status + IsBinaryTable(const std::string& table_name, bool& bin); + + Status + CopyRecordsFromJson(const nlohmann::json& json, engine::VectorsData& vectors, bool bin); + protected: Status - GetTableInfo(const std::string& table_name, TableFieldsDto::ObjectWrapper& table_fields); + GetTableMetaInfo(const std::string& table_name, nlohmann::json& json_out); + + Status + GetTableStat(const std::string& table_name, nlohmann::json& json_out); + + Status + GetSegmentVectors(const std::string& table_name, const std::string& segment_name, int64_t page_size, int64_t offset, + nlohmann::json& json_out); + + Status + GetSegmentIds(const std::string& table_name, const std::string& segment_name, int64_t page_size, int64_t offset, + nlohmann::json& json_out); Status CommandLine(const std::string& cmd, std::string& reply); + Status + Cmd(const std::string& cmd, std::string& result_str); + + Status + PreLoadTable(const nlohmann::json& json, std::string& result_str); + + Status + Flush(const nlohmann::json& json, std::string& result_str); + + Status + Compact(const nlohmann::json& json, std::string& result_str); + + Status + GetConfig(std::string& result_str); + + Status + SetConfig(const nlohmann::json& json, std::string& result_str); + + Status + Search(const std::string& table_name, const nlohmann::json& json, std::string& result_str); + + Status + DeleteByIDs(const std::string& table_name, const nlohmann::json& json, std::string& result_str); + + Status + GetVectorsByIDs(const std::string& table_name, const std::vector& ids, nlohmann::json& json_out); + public: WebRequestHandler() { context_ptr_ = GenContextPtr("Web Handler"); request_handler_ = RequestHandler(); } + public: StatusDto::ObjectWrapper GetDevices(DevicesDto::ObjectWrapper& devices); @@ -106,18 +162,25 @@ class WebRequestHandler { SetGpuConfig(const GPUConfigDto::ObjectWrapper& gpu_config_dto); #endif + /********** + * + * Table + */ StatusDto::ObjectWrapper CreateTable(const TableRequestDto::ObjectWrapper& table_schema); + StatusDto::ObjectWrapper + ShowTables(const OQueryParams& query_params, OString& result); StatusDto::ObjectWrapper - GetTable(const OString& table_name, const OQueryParams& query_params, TableFieldsDto::ObjectWrapper& schema_dto); - - StatusDto::ObjectWrapper - ShowTables(const OString& offset, const OString& page_size, TableListFieldsDto::ObjectWrapper& table_list_dto); + GetTable(const OString& table_name, const OQueryParams& query_params, OString& result); StatusDto::ObjectWrapper DropTable(const OString& table_name); + /********** + * + * Index + */ StatusDto::ObjectWrapper CreateIndex(const OString& table_name, const IndexRequestDto::ObjectWrapper& index_param); @@ -127,26 +190,50 @@ class WebRequestHandler { StatusDto::ObjectWrapper DropIndex(const OString& table_name); + /*********** + * + * Partition + */ StatusDto::ObjectWrapper CreatePartition(const OString& table_name, const PartitionRequestDto::ObjectWrapper& param); StatusDto::ObjectWrapper - ShowPartitions(const OString& offset, const OString& page_size, const OString& table_name, + ShowPartitions(const OString& table_name, const OQueryParams& query_params, PartitionListDto::ObjectWrapper& partition_list_dto); StatusDto::ObjectWrapper - DropPartition(const OString& table_name, const OString& tag); + DropPartition(const OString& table_name, const OString& body); + + /*********** + * + * Segment + */ + StatusDto::ObjectWrapper + ShowSegments(const OString& table_name, const OQueryParams& query_params, OString& response); StatusDto::ObjectWrapper - Insert(const OString& table_name, const InsertRequestDto::ObjectWrapper& param, - VectorIdsDto::ObjectWrapper& ids_dto); + GetSegmentInfo(const OString& table_name, const OString& segment_name, const OString& info, + const OQueryParams& query_params, OString& result); + + /** + * + * Vector + */ + StatusDto::ObjectWrapper + Insert(const OString& table_name, const OString& body, VectorIdsDto::ObjectWrapper& ids_dto); StatusDto::ObjectWrapper - Search(const OString& table_name, const SearchRequestDto::ObjectWrapper& search_request, - TopkResultsDto::ObjectWrapper& results_dto); + GetVector(const OString& table_name, const OQueryParams& query_params, OString& response); StatusDto::ObjectWrapper - SystemInfo(const OString& cmd, CommandDto::ObjectWrapper& cmd_dto); + VectorsOp(const OString& table_name, const OString& payload, OString& response); + + /** + * + * System + */ + StatusDto::ObjectWrapper + SystemInfo(const OString& cmd, const OQueryParams& query_params, OString& response_str); StatusDto::ObjectWrapper SystemOp(const OString& op, const OString& body_str, OString& response_str); diff --git a/core/unittest/server/test_web.cpp b/core/unittest/server/test_web.cpp index 5ea1960761..779c86e40d 100644 --- a/core/unittest/server/test_web.cpp +++ b/core/unittest/server/test_web.cpp @@ -10,17 +10,16 @@ // or implied. See the License for the specific language governing permissions and limitations under the License. #include + #include #include #include #include -#include #include #include #include #include -#include #include "scheduler/ResourceFactory.h" #include "scheduler/SchedInst.h" @@ -40,11 +39,10 @@ #include "server/web_impl/dto/TableDto.hpp" #include "server/web_impl/dto/VectorDto.hpp" #include "server/web_impl/handler/WebRequestHandler.h" - +#include "unittest/server/utils.h" #include "utils/CommonUtil.h" #include "wrapper/VecIndex.h" -#include "unittest/server/utils.h" static const char* TABLE_NAME = "test_web"; static constexpr int64_t TABLE_DIM = 256; @@ -113,6 +111,52 @@ RandomBinRecordsDto(int64_t dim, int64_t num) { return records_dto; } +nlohmann::json +RandomRawRecordJson(int64_t dim) { + nlohmann::json json; + + std::default_random_engine e; + std::uniform_real_distribution u(0, 1); + for (size_t i = 0; i < dim; i++) { + json.push_back(u(e)); + } + + return json; +} + +nlohmann::json +RandomRecordsJson(int64_t dim, int64_t num) { + nlohmann::json json; + for (size_t i = 0; i < num; i++) { + json.push_back(RandomRawRecordJson(dim)); + } + + return json; +} + +nlohmann::json +RandomRawBinRecordJson(int64_t dim) { + nlohmann::json json; + + std::default_random_engine e; + std::uniform_real_distribution u(0, 255); + for (size_t i = 0; i < dim / 8; i++) { + json.push_back(static_cast(u(e))); + } + + return json; +} + +nlohmann::json +RandomBinRecordsJson(int64_t dim, int64_t num) { + nlohmann::json json; + for (size_t i = 0; i < num; i++) { + json.push_back(RandomRawBinRecordJson(dim)); + } + + return json; +} + std::string RandomName() { unsigned seed = std::chrono::system_clock::now().time_since_epoch().count(); @@ -258,8 +302,8 @@ TEST_F(WebHandlerTest, HAS_TABLE_TEST) { GenTable(table_name->std_str(), 10, 10, "L2"); milvus::server::web::OQueryParams query_params; - auto tables_dto = milvus::server::web::TableFieldsDto::createShared(); - auto status_dto = handler->GetTable(table_name, query_params, tables_dto); + OString response; + auto status_dto = handler->GetTable(table_name, query_params, response); ASSERT_EQ(0, status_dto->code->getValue()); } @@ -270,12 +314,14 @@ TEST_F(WebHandlerTest, GET_TABLE) { GenTable(table_name->std_str(), 10, 10, "L2"); milvus::server::web::OQueryParams query_params; - auto table_dto = milvus::server::web::TableFieldsDto::createShared(); - auto status_dto = handler->GetTable(table_name, query_params, table_dto); + OString result; + auto status_dto = handler->GetTable(table_name, query_params, result); ASSERT_EQ(0, status_dto->code->getValue()); - ASSERT_EQ(10, table_dto->dimension->getValue()); - ASSERT_EQ(10, table_dto->index_file_size->getValue()); - ASSERT_EQ("L2", table_dto->metric_type->std_str()); + + auto result_json = nlohmann::json::parse(result->std_str()); + ASSERT_EQ(10, result_json["dimension"].get()); + ASSERT_EQ(10, result_json["index_file_size"].get()); + ASSERT_EQ("L2", result_json["metric_type"].get()); } TEST_F(WebHandlerTest, INSERT_COUNT) { @@ -284,17 +330,10 @@ TEST_F(WebHandlerTest, INSERT_COUNT) { auto table_name = milvus::server::web::OString(TABLE_NAME) + RandomName().c_str(); GenTable(table_name->std_str(), 16, 10, "L2"); - auto insert_request_dto = milvus::server::web::InsertRequestDto::createShared(); - insert_request_dto->records = insert_request_dto->records->createShared(); - for (size_t i = 0; i < 1000; i++) { - insert_request_dto->records->pushBack(RandomRowRecordDto(16)); - } - insert_request_dto->ids = insert_request_dto->ids->createShared(); - + nlohmann::json body_json; + body_json["vectors"] = RandomRecordsJson(16, 1000); auto ids_dto = milvus::server::web::VectorIdsDto::createShared(); - - auto status_dto = handler->Insert(table_name, insert_request_dto, ids_dto); - + auto status_dto = handler->Insert(table_name, body_json.dump().c_str(), ids_dto); ASSERT_EQ(0, status_dto->code->getValue()); ASSERT_EQ(1000, ids_dto->ids->count()); @@ -302,10 +341,12 @@ TEST_F(WebHandlerTest, INSERT_COUNT) { milvus::server::web::OQueryParams query_params; query_params.put("fields", "num"); - auto tables_dto = milvus::server::web::TableFieldsDto::createShared(); - status_dto = handler->GetTable(table_name, query_params, tables_dto); + OString result; + status_dto = handler->GetTable(table_name, query_params, result); ASSERT_EQ(0, status_dto->code->getValue()); - ASSERT_EQ(1000, tables_dto->count->getValue()); + + auto result_json = nlohmann::json::parse(result->std_str()); + ASSERT_EQ(1000, result_json["count"].get()); } TEST_F(WebHandlerTest, INDEX) { @@ -353,15 +394,23 @@ TEST_F(WebHandlerTest, PARTITION) { ASSERT_EQ(0, status_dto->code->getValue()); auto partitions_dto = milvus::server::web::PartitionListDto::createShared(); - status_dto = handler->ShowPartitions("0", "10", table_name, partitions_dto); + OQueryParams query_params; + query_params.put("offset", "0"); + query_params.put("page_size", "10"); + status_dto = handler->ShowPartitions(table_name, query_params, partitions_dto); + ASSERT_EQ(milvus::server::web::SUCCESS, status_dto->code->getValue()); ASSERT_EQ(2, partitions_dto->partitions->count()); - status_dto = handler->DropPartition(table_name, "test"); + status_dto = handler->DropPartition(table_name, "{\"partition_tag\": \"test\"}"); ASSERT_EQ(0, status_dto->code->getValue()); // Show all partitions - partitions_dto = milvus::server::web::PartitionListDto::createShared(); - status_dto = handler->ShowPartitions("0", "10", table_name, partitions_dto); + status_dto = handler->ShowPartitions(table_name, query_params, partitions_dto); + ASSERT_EQ(milvus::server::web::SUCCESS, status_dto->code->getValue()); + + query_params.put("all_required", "true"); + status_dto = handler->ShowPartitions(table_name, query_params, partitions_dto); + ASSERT_EQ(milvus::server::web::SUCCESS, status_dto->code->getValue()); } TEST_F(WebHandlerTest, SEARCH) { @@ -370,41 +419,76 @@ TEST_F(WebHandlerTest, SEARCH) { auto table_name = milvus::server::web::OString(TABLE_NAME) + RandomName().c_str(); GenTable(table_name->std_str(), TABLE_DIM, 10, "L2"); - auto insert_request_dto = milvus::server::web::InsertRequestDto::createShared(); - insert_request_dto->records = insert_request_dto->records->createShared(); - for (size_t i = 0; i < 1000; i++) { - insert_request_dto->records->pushBack(RandomRowRecordDto(TABLE_DIM)); - } - insert_request_dto->ids = insert_request_dto->ids->createShared(); + nlohmann::json insert_json; + insert_json["vectors"] = RandomRecordsJson(TABLE_DIM, 1000); auto ids_dto = milvus::server::web::VectorIdsDto::createShared(); - auto status_dto = handler->Insert(table_name, insert_request_dto, ids_dto); - ASSERT_EQ(0, status_dto->code->getValue()); + auto status_dto = handler->Insert(table_name, insert_json.dump().c_str(), ids_dto); + ASSERT_EQ(milvus::server::web::SUCCESS, status_dto->code->getValue()); - auto search_request_dto = milvus::server::web::SearchRequestDto::createShared(); - search_request_dto->records = RandomRecordsDto(TABLE_DIM, 10); - search_request_dto->topk = 1; - search_request_dto->nprobe = 1; + nlohmann::json search_pram_json; + search_pram_json["vectors"] = RandomRecordsJson(TABLE_DIM, 10); + search_pram_json["topk"] = 1; + search_pram_json["nprobe"] = 1; - auto results_dto = milvus::server::web::TopkResultsDto::createShared(); + nlohmann::json search_json; + search_json["search"] = search_pram_json; - status_dto = handler->Search(table_name, search_request_dto, results_dto); + OString result = ""; + status_dto = handler->VectorsOp(table_name, search_json.dump().c_str(), result); ASSERT_EQ(0, status_dto->code->getValue()) << status_dto->message->std_str(); } -TEST_F(WebHandlerTest, CMD) { +TEST_F(WebHandlerTest, SYSTEM_INFO) { handler->RegisterRequestHandler(milvus::server::RequestHandler()); - milvus::server::web::OString cmd; - auto cmd_dto = milvus::server::web::CommandDto::createShared(); - cmd = "status"; - auto status_dto = handler->SystemInfo(cmd, cmd_dto); - ASSERT_EQ(0, status_dto->code->getValue()); - ASSERT_EQ("OK", cmd_dto->reply->std_str()); + OQueryParams query_params; + OString result; - cmd = "version"; - status_dto = handler->SystemInfo(cmd, cmd_dto); + auto status_dto = handler->SystemInfo("status", query_params, result); ASSERT_EQ(0, status_dto->code->getValue()); - ASSERT_EQ("0.7.0", cmd_dto->reply->std_str()); +// ASSERT_EQ("OK", cmd_dto->reply->std_str()); + + status_dto = handler->SystemInfo("version", query_params, result); + ASSERT_EQ(0, status_dto->code->getValue()); +// ASSERT_EQ("0.7.0", cmd_dto->reply->std_str()); +} + +TEST_F(WebHandlerTest, FLUSH) { + handler->RegisterRequestHandler(milvus::server::RequestHandler()); + + auto table_name = milvus::server::web::OString(TABLE_NAME) + RandomName().c_str(); + GenTable(table_name->std_str(), 16, 10, "L2"); + + nlohmann::json body_json; + body_json["vectors"] = RandomRecordsJson(16, 1000); + auto ids_dto = milvus::server::web::VectorIdsDto::createShared(); + auto status_dto = handler->Insert(table_name, body_json.dump().c_str(), ids_dto); + ASSERT_EQ(0, status_dto->code->getValue()) << status_dto->message->std_str(); + + nlohmann::json flush_json; + flush_json["flush"]["table_names"] = {table_name->std_str()}; + OString result; + status_dto = handler->SystemOp("task", flush_json.dump().c_str(), result); + ASSERT_EQ(milvus::server::web::SUCCESS, status_dto->code->getValue()); +} + +TEST_F(WebHandlerTest, COMPACT) { + handler->RegisterRequestHandler(milvus::server::RequestHandler()); + + auto table_name = milvus::server::web::OString(TABLE_NAME) + RandomName().c_str(); + GenTable(table_name->std_str(), 16, 10, "L2"); + + nlohmann::json body_json; + body_json["vectors"] = RandomRecordsJson(16, 1000); + auto ids_dto = milvus::server::web::VectorIdsDto::createShared(); + auto status_dto = handler->Insert(table_name, body_json.dump().c_str(), ids_dto); + ASSERT_EQ(0, status_dto->code->getValue()) << status_dto->message->std_str(); + + nlohmann::json compact_json; + compact_json["compact"]["table_name"] = table_name->std_str(); + OString result; + status_dto = handler->SystemOp("task", compact_json.dump().c_str(), result); + ASSERT_EQ(milvus::server::web::SUCCESS, status_dto->code->getValue()); } /////////////////////////////////////////////////////////////////////////////////////// @@ -535,14 +619,12 @@ class TestClient : public oatpp::web::client::ApiClient { BODY_DTO(milvus::server::web::AdvancedConfigDto::ObjectWrapper, body)) #ifdef MILVUS_GPU_VERSION - API_CALL("OPTIONS", "config/gpu_resources", optionsGpuConfig) API_CALL("GET", "/config/gpu_resources", getGPUConfig) API_CALL("PUT", "/config/gpu_resources", setGPUConfig, BODY_DTO(milvus::server::web::GPUConfigDto::ObjectWrapper, body)) - #endif API_CALL("OPTIONS", "/tables", optionsTables) @@ -553,7 +635,7 @@ class TestClient : public oatpp::web::client::ApiClient { API_CALL("OPTIONS", "/tables/{table_name}", optionsTable, PATH(String, table_name, "table_name")) - API_CALL("GET", "/tables/{table_name}", getTable, PATH(String, table_name, "table_name")) + API_CALL("GET", "/tables/{table_name}", getTable, PATH(String, table_name, "table_name"), QUERY(String, info)) API_CALL("DELETE", "/tables/{table_name}", dropTable, PATH(String, table_name, "table_name")) @@ -574,23 +656,30 @@ class TestClient : public oatpp::web::client::ApiClient { API_CALL("GET", "/tables/{table_name}/partitions", showPartitions, PATH(String, table_name, "table_name"), QUERY(String, offset), QUERY(String, page_size)) - API_CALL("OPTIONS", "/tables/{table_name}/partitions/{partition_tag}", optionsParTag, - PATH(String, table_name, "table_name"), PATH(String, partition_tag, "partition_tag")) + API_CALL("DELETE", "/tables/{table_name}/partitions", dropPartition, + PATH(String, table_name, "table_name"), BODY_STRING(String, body)) - API_CALL("DELETE", "/tables/{table_name}/partitions/{partition_tag}", dropPartition, - PATH(String, table_name, "table_name"), PATH(String, partition_tag)) + API_CALL("GET", "/tables/{table_name}/segments", showSegments, PATH(String, table_name, "table_name"), + QUERY(String, offset), QUERY(String, page_size), QUERY(String, partition_tag)) + + API_CALL("GET", "/tables/{table_name}/segments/{segment_name}/{info}", getSegmentInfo, + PATH(String, table_name, "table_name"), PATH(String, segment_name, "segment_name"), + PATH(String, info, "info"), QUERY(String, offset), QUERY(String, page_size)) API_CALL("OPTIONS", "/tables/{table_name}/vectors", optionsVectors, PATH(String, table_name, "table_name")) - API_CALL("POST", "/tables/{table_name}/vectors", insert, PATH(String, table_name, "table_name"), - BODY_DTO(milvus::server::web::InsertRequestDto::ObjectWrapper, body)) + API_CALL("GET", "/tables/{table_name}/vectors", getVectors, + PATH(String, table_name, "table_name"), QUERY(String, id)) - API_CALL("PUT", "/tables/{table_name}/vectors", search, PATH(String, table_name, "table_name"), - BODY_DTO(milvus::server::web::SearchRequestDto::ObjectWrapper, body)) + API_CALL("POST", "/tables/{table_name}/vectors", insert, + PATH(String, table_name, "table_name"), BODY_STRING(String, body)) + + API_CALL("PUT", "/tables/{table_name}/vectors", vectorsOp, + PATH(String, table_name, "table_name"), BODY_STRING(String, body)) API_CALL("GET", "/system/{msg}", cmd, PATH(String, cmd_str, "msg"), QUERY(String, action), QUERY(String, target)) - API_CALL("PUT", "/system/{op}", exec, PATH(String, op, "op"), BODY_STRING(String, body)) + API_CALL("PUT", "/system/{op}", op, PATH(String, cmd_str, "op"), BODY_STRING(String, body)) #include OATPP_CODEGEN_END(ApiClient) }; @@ -652,7 +741,7 @@ class WebControllerTest : public testing::Test { void GenTable(const OString& table_name, int64_t dim, int64_t index_size, const OString& metric) { - auto response = client_ptr->getTable(table_name, conncetion_ptr); + auto response = client_ptr->getTable(table_name, "", conncetion_ptr); if (OStatus::CODE_200.code == response->getStatusCode()) { return; } @@ -664,6 +753,89 @@ class WebControllerTest : public testing::Test { client_ptr->createTable(table_dto, conncetion_ptr); } + milvus::Status + FlushTable(const std::string& table_name) { + nlohmann::json flush_json; + flush_json["flush"]["table_names"] = {table_name}; + auto response = client_ptr->op("task", flush_json.dump().c_str(), conncetion_ptr); + if (OStatus::CODE_200.code != response->getStatusCode()) { + return milvus::Status(milvus::SERVER_UNEXPECTED_ERROR, response->readBodyToString()->std_str()); + } + + return milvus::Status::OK(); + } + + milvus::Status + FlushTable(const OString& table_name) { + nlohmann::json flush_json; + flush_json["flush"]["table_names"] = {table_name->std_str()}; + auto response = client_ptr->op("task", flush_json.dump().c_str(), conncetion_ptr); + if (OStatus::CODE_200.code != response->getStatusCode()) { + return milvus::Status(milvus::SERVER_UNEXPECTED_ERROR, response->readBodyToString()->std_str()); + } + + return milvus::Status::OK(); + } + + milvus::Status + InsertData(const OString& table_name, int64_t dim, int64_t count, std::string tag = "", bool bin = false) { + nlohmann::json insert_json; + + if (bin) + insert_json["vectors"] = RandomBinRecordsJson(dim, count); + else + insert_json["vectors"] = RandomRecordsJson(dim, count); + + if (!tag.empty()) { + insert_json["partition_tag"] = tag; + } + + auto response = client_ptr->insert(table_name, insert_json.dump().c_str(), conncetion_ptr); + if (OStatus::CODE_201.code != response->getStatusCode()) { + return milvus::Status(milvus::SERVER_UNEXPECTED_ERROR, response->readBodyToString()->c_str()); + } + + return FlushTable(table_name); + } + + milvus::Status + InsertData(const OString& table_name, int64_t dim, int64_t count, + const std::vector& ids, std::string tag = "", bool bin = false) { + nlohmann::json insert_json; + + if (bin) + insert_json["vectors"] = RandomBinRecordsJson(dim, count); + else + insert_json["vectors"] = RandomRecordsJson(dim, count); + + if (!ids.empty()) { + insert_json["ids"] = ids; + } + + if (!tag.empty()) { + insert_json["partition_tag"] = tag; + } + + auto response = client_ptr->insert(table_name, insert_json.dump().c_str(), conncetion_ptr); + if (OStatus::CODE_201.code != response->getStatusCode()) { + return milvus::Status(milvus::SERVER_UNEXPECTED_ERROR, response->readBodyToString()->c_str()); + } + + return FlushTable(table_name); + } + + milvus::Status + GenPartition(const OString& table_name, const OString& tag) { + auto par_param = milvus::server::web::PartitionRequestDto::createShared(); + par_param->partition_tag = tag; + auto response = client_ptr->createPartition(table_name, par_param); + if (OStatus::CODE_201.code != response->getStatusCode()) { + return milvus::Status(milvus::SERVER_UNEXPECTED_ERROR, response->readBodyToString()->c_str()); + } + + return milvus::Status::OK(); + } + void SetUp() override { std::string config_path = std::string(CONTROLLER_TEST_CONFIG_DIR).append(CONTROLLER_TEST_CONFIG_FILE); @@ -717,18 +889,13 @@ TEST_F(WebControllerTest, OPTIONS) { ASSERT_EQ(OStatus::CODE_204.code, response->getStatusCode()); #ifdef MILVUS_GPU_VERSION - response = client_ptr->optionsGpuConfig(conncetion_ptr); ASSERT_EQ(OStatus::CODE_204.code, response->getStatusCode()); - #endif response = client_ptr->optionsIndexes("test", conncetion_ptr); ASSERT_EQ(OStatus::CODE_204.code, response->getStatusCode()); - response = client_ptr->optionsParTag("test", "tag", conncetion_ptr); - ASSERT_EQ(OStatus::CODE_204.code, response->getStatusCode()); - response = client_ptr->optionsPartitions("table_name", conncetion_ptr); ASSERT_EQ(OStatus::CODE_204.code, response->getStatusCode()); @@ -772,15 +939,13 @@ TEST_F(WebControllerTest, CREATE_TABLE) { ASSERT_EQ(OStatus::CODE_400.code, response->getStatusCode()); } -TEST_F(WebControllerTest, GET_TABLE) { +TEST_F(WebControllerTest, GET_TABLE_META) { OString table_name = "web_test_create_table" + OString(RandomName().c_str()); GenTable(table_name, 10, 10, "L2"); OQueryParams params; - // fields value is 'num', test count table - params.put("fields", "num"); - auto response = client_ptr->getTable(table_name, conncetion_ptr); + auto response = client_ptr->getTable(table_name, "", conncetion_ptr); ASSERT_EQ(OStatus::CODE_200.code, response->getStatusCode()); auto result_dto = response->readBodyToDto(object_mapper.get()); ASSERT_EQ(table_name->std_str(), result_dto->table_name->std_str()); @@ -791,16 +956,47 @@ TEST_F(WebControllerTest, GET_TABLE) { // invalid table name table_name = "57474dgdfhdfhdh dgd"; - response = client_ptr->getTable(table_name, conncetion_ptr); + response = client_ptr->getTable(table_name, "", conncetion_ptr); ASSERT_EQ(OStatus::CODE_400.code, response->getStatusCode()); auto status_sto = response->readBodyToDto(object_mapper.get()); ASSERT_EQ(milvus::server::web::StatusCode::ILLEGAL_TABLE_NAME, status_sto->code->getValue()); table_name = "test_table_not_found_000000000111010101002020203020aaaaa3030435"; - response = client_ptr->getTable(table_name, conncetion_ptr); + response = client_ptr->getTable(table_name, "", conncetion_ptr); ASSERT_EQ(OStatus::CODE_404.code, response->getStatusCode()); } +TEST_F(WebControllerTest, GET_TABLE_STAT) { + OString table_name = "web_test_get_table_stat" + OString(RandomName().c_str()); + GenTable(table_name, 128, 5, "L2"); + + for (size_t i = 0; i < 5; i++) { + InsertData(table_name, 128, 1000); + } + + auto response = client_ptr->getTable(table_name, "stat", conncetion_ptr); + ASSERT_EQ(OStatus::CODE_200.code, response->getStatusCode()); + auto result_json = nlohmann::json::parse(response->readBodyToString()->c_str()); + ASSERT_TRUE(result_json.contains("count")); + ASSERT_EQ(5 * 1000, result_json["count"].get()); + + ASSERT_TRUE(result_json.contains("partitions_stat")); + + auto partitions_stat_json = result_json["partitions_stat"]; + ASSERT_TRUE(partitions_stat_json.is_array()); + + auto partition0_json = partitions_stat_json[0]; + ASSERT_TRUE(partition0_json.contains("segments_stat")); + ASSERT_TRUE(partition0_json.contains("count")); + ASSERT_TRUE(partition0_json.contains("partition_tag")); + + auto seg0_stat = partition0_json["segments_stat"][0]; + ASSERT_TRUE(seg0_stat.contains("segment_name")); + ASSERT_TRUE(seg0_stat.contains("index")); + ASSERT_TRUE(seg0_stat.contains("count")); + ASSERT_TRUE(seg0_stat.contains("size")); +} + TEST_F(WebControllerTest, SHOW_TABLES) { // test query table limit 1 auto response = client_ptr->showTables("1", "1", conncetion_ptr); @@ -825,7 +1021,7 @@ TEST_F(WebControllerTest, SHOW_TABLES) { response = client_ptr->showTables("1", "1.1", conncetion_ptr); ASSERT_EQ(OStatus::CODE_400.code, response->getStatusCode()); - response = client_ptr->showTables("0", "90000000000000000000000000000000000000000000000000000000", conncetion_ptr); + response = client_ptr->showTables("0", "9000000000000000000000000000000000000000000000000000000", conncetion_ptr); ASSERT_EQ(OStatus::CODE_400.code, response->getStatusCode()); } @@ -849,16 +1045,15 @@ TEST_F(WebControllerTest, INSERT) { const int64_t dim = 64; GenTable(table_name, dim, 100, "L2"); - auto insert_dto = milvus::server::web::InsertRequestDto::createShared(); - insert_dto->ids = insert_dto->ids->createShared(); - insert_dto->records = RandomRecordsDto(dim, 20); + nlohmann::json insert_json; + insert_json["vectors"] = RandomRecordsJson(dim, 20); - auto response = client_ptr->insert(table_name, insert_dto, conncetion_ptr); + auto response = client_ptr->insert(table_name, insert_json.dump().c_str(), conncetion_ptr); ASSERT_EQ(OStatus::CODE_201.code, response->getStatusCode()); auto result_dto = response->readBodyToDto(object_mapper.get()); ASSERT_EQ(20, result_dto->ids->count()); - response = client_ptr->insert(table_name + "ooowrweindexsgs", insert_dto, conncetion_ptr); + response = client_ptr->insert(table_name + "ooowrweindexsgs", insert_json.dump().c_str(), conncetion_ptr); ASSERT_EQ(OStatus::CODE_404.code, response->getStatusCode()); response = client_ptr->dropTable(table_name, conncetion_ptr); @@ -870,12 +1065,15 @@ TEST_F(WebControllerTest, INSERT_BIN) { const int64_t dim = 64; GenTable(table_name, dim, 100, "HAMMING"); - auto insert_dto = milvus::server::web::InsertRequestDto::createShared(); - insert_dto->ids = insert_dto->ids->createShared(); - insert_dto->records_bin = RandomBinRecordsDto(dim, 20); + nlohmann::json insert_json; + insert_json["vectors"] = RandomBinRecordsJson(dim, 20); + + auto response = client_ptr->insert(table_name, insert_json.dump().c_str(), conncetion_ptr); + ASSERT_EQ(OStatus::CODE_201.code, response->getStatusCode()) << response->readBodyToString()->std_str(); + + auto status = FlushTable(table_name); + ASSERT_TRUE(status.ok()) << status.message(); - auto response = client_ptr->insert(table_name, insert_dto, conncetion_ptr); - ASSERT_EQ(OStatus::CODE_201.code, response->getStatusCode()); auto result_dto = response->readBodyToDto(object_mapper.get()); ASSERT_EQ(20, result_dto->ids->count()); @@ -888,16 +1086,17 @@ TEST_F(WebControllerTest, INSERT_IDS) { const int64_t dim = 64; GenTable(table_name, dim, 100, "L2"); - auto insert_dto = milvus::server::web::InsertRequestDto::createShared(); - insert_dto->ids = insert_dto->ids->createShared(); + std::vector ids; for (size_t i = 0; i < 20; i++) { - insert_dto->ids->pushBack(i); + ids.emplace_back(i); } - insert_dto->records = RandomRecordsDto(dim, 20); + nlohmann::json insert_json; + insert_json["vectors"] = RandomRecordsJson(dim, 20); + insert_json["ids"] = ids; - auto response = client_ptr->insert(table_name, insert_dto, conncetion_ptr); - ASSERT_EQ(OStatus::CODE_201.code, response->getStatusCode()); + auto response = client_ptr->insert(table_name, insert_json.dump().c_str(), conncetion_ptr); + ASSERT_EQ(OStatus::CODE_201.code, response->getStatusCode()) << response->readBodyToString()->std_str(); auto result_dto = response->readBodyToDto(object_mapper.get()); ASSERT_EQ(20, result_dto->ids->count()); @@ -905,35 +1104,6 @@ TEST_F(WebControllerTest, INSERT_IDS) { ASSERT_EQ(OStatus::CODE_204.code, response->getStatusCode()); } -TEST_F(WebControllerTest, LOAD_TABLE) { - milvus::server::Config& config = milvus::server::Config::GetInstance(); - auto status = config.SetCacheConfigInsertBufferSize("1"); - ASSERT_TRUE(status.ok()); - - const OString table_name = "test_web_controller_table_load_test" + OString(RandomName().c_str()); - GenTable(table_name, 64, 100, "L2"); - - // Insert 200 vectors into table - auto insert_dto = milvus::server::web::InsertRequestDto::createShared(); - insert_dto->ids = insert_dto->ids->createShared(); - insert_dto->records = RandomRecordsDto(64, 100); - auto response = client_ptr->insert(table_name, insert_dto, conncetion_ptr); - ASSERT_EQ(OStatus::CODE_201.code, response->getStatusCode()); - auto result_dto = response->readBodyToDto(object_mapper.get()); - ASSERT_EQ(100, result_dto->ids->count()); - - sleep(2); - - std::string request_str = "{\"load\": {\"table_name\": \"" + table_name->std_str() + "\"}}"; - response = client_ptr->exec("task", request_str.c_str()); - ASSERT_EQ(OStatus::CODE_200.code, response->getStatusCode()) << response->readBodyToString()->c_str(); - - // test with non-exist table - request_str = "{\"load\": {\"table_name\": \"OOOO124214\"}}"; - response = client_ptr->exec("task", request_str.c_str()); - ASSERT_NE(OStatus::CODE_200.code, response->getStatusCode()); -} - TEST_F(WebControllerTest, INDEX) { auto table_name = "test_insert_table_test" + OString(RandomName().c_str()); GenTable(table_name, 64, 100, "L2"); @@ -986,12 +1156,8 @@ TEST_F(WebControllerTest, INDEX) { response = client_ptr->dropIndex(table_name, conncetion_ptr); ASSERT_EQ(OStatus::CODE_204.code, response->getStatusCode()); - auto insert_dto = milvus::server::web::InsertRequestDto::createShared(); - insert_dto->ids = insert_dto->ids->createShared(); - insert_dto->records = RandomRecordsDto(64, 200); - - response = client_ptr->insert(table_name, insert_dto, conncetion_ptr); - ASSERT_EQ(OStatus::CODE_201.code, response->getStatusCode()); + auto status = InsertData(table_name, 64, 200); + ASSERT_TRUE(status.ok()) << status.message(); index_dto->index_type = milvus::server::web::IndexMap.at(milvus::engine::EngineType::FAISS_IDMAP).c_str(); response = client_ptr->createIndex(table_name, index_dto, conncetion_ptr); @@ -1008,10 +1174,6 @@ TEST_F(WebControllerTest, INDEX) { ASSERT_EQ(OStatus::CODE_404.code, response->getStatusCode()); auto error_dto = response->readBodyToDto(object_mapper.get()); ASSERT_EQ(milvus::server::web::StatusCode::TABLE_NOT_EXISTS, error_dto->code->getValue()); - - // drop index which table is non-existent - response = client_ptr->dropIndex("Table_name_non_existant_000000000000000000", conncetion_ptr); - ASSERT_EQ(OStatus::CODE_404.code, response->getStatusCode()) << response->readBodyToString()->c_str(); } TEST_F(WebControllerTest, PARTITION) { @@ -1041,17 +1203,8 @@ TEST_F(WebControllerTest, PARTITION) { ASSERT_EQ(milvus::server::web::StatusCode::TABLE_NOT_EXISTS, error_dto->code); // insert 200 vectors into table with tag = 'tag01' - OQueryParams query_params; - // add partition tag - auto insert_dto = milvus::server::web::InsertRequestDto::createShared(); - insert_dto->tag = OString("tag01"); - insert_dto->ids = insert_dto->ids->createShared(); - insert_dto->records = insert_dto->records->createShared(); - for (size_t i = 0; i < 200; i++) { - insert_dto->records->pushBack(RandomRowRecordDto(64)); - } - response = client_ptr->insert(table_name, insert_dto, conncetion_ptr); - ASSERT_EQ(OStatus::CODE_201.code, response->getStatusCode()); + auto status = InsertData(table_name, 64, 200, "tag01"); + ASSERT_TRUE(status.ok()) << status.message(); // Show all partitins response = client_ptr->showPartitions(table_name, "0", "10", conncetion_ptr); @@ -1075,75 +1228,171 @@ TEST_F(WebControllerTest, PARTITION) { error_dto = response->readBodyToDto(object_mapper.get()); ASSERT_EQ(milvus::server::web::StatusCode::TABLE_NOT_EXISTS, error_dto->code->getValue()); - response = client_ptr->dropPartition(table_name, "tag01", conncetion_ptr); + response = client_ptr->dropPartition(table_name, "{\"partition_tag\": \"tag01\"}", conncetion_ptr); ASSERT_EQ(OStatus::CODE_204.code, response->getStatusCode()); // drop without existing tables - response = client_ptr->dropPartition(table_name + "565755682353464aaasafdsfagagqq1223", "tag01", conncetion_ptr); + response = client_ptr->dropPartition(table_name + "565755682353464aaasafdsfagagqq1223", + "{\"partition_tag\": \"tag01\"}", conncetion_ptr); ASSERT_EQ(OStatus::CODE_404.code, response->getStatusCode()); } +TEST_F(WebControllerTest, SHOW_SEGMENTS) { + OString table_name = OString("test_milvus_web_segments_test_") + RandomName().c_str(); + + GenTable(table_name, 256, 1, "L2"); + + auto status = InsertData(table_name, 256, 2000); + ASSERT_TRUE(status.ok()) << status.message(); + + auto response = client_ptr->showSegments(table_name, "0", "10", "", conncetion_ptr); + ASSERT_EQ(OStatus::CODE_200.code, response->getStatusCode()) << response->readBodyToString()->c_str(); + + // validate result + auto result_json = nlohmann::json::parse(response->readBodyToString()->c_str()); + + ASSERT_TRUE(result_json.contains("count")); + + ASSERT_TRUE(result_json.contains("segments")); + auto segments_json = result_json["segments"]; + ASSERT_TRUE(segments_json.is_array()); +// ASSERT_EQ(10, segments_json.size()); +} + +TEST_F(WebControllerTest, GET_SEGMENT_INFO) { + OString table_name = OString("test_milvus_web_get_segment_info_test_") + RandomName().c_str(); + + GenTable(table_name, 16, 1, "L2"); + + auto status = InsertData(table_name, 16, 2000); + ASSERT_TRUE(status.ok()) << status.message(); + + auto response = client_ptr->showSegments(table_name, "0", "10", "", conncetion_ptr); + ASSERT_EQ(OStatus::CODE_200.code, response->getStatusCode()) << response->readBodyToString()->c_str(); + + // validate result + auto result_json = nlohmann::json::parse(response->readBodyToString()->c_str()); + + auto segment0_json = result_json["segments"][0]; + std::string segment_name = segment0_json["segment_name"]; + + + // get segment ids + response = client_ptr->getSegmentInfo(table_name, segment_name.c_str(), "ids", "0", "10"); + ASSERT_EQ(OStatus::CODE_200.code, response->getStatusCode()) << response->readBodyToString()->c_str(); + + auto ids_result_json = nlohmann::json::parse(response->readBodyToString()->c_str()); + ASSERT_TRUE(ids_result_json.contains("ids")); + auto ids_json = ids_result_json["ids"]; + ASSERT_TRUE(ids_json.is_array()); + ASSERT_EQ(10, ids_json.size()); + + // get segment vectors + response = client_ptr->getSegmentInfo(table_name, segment_name.c_str(), "vectors", "0", "10"); + ASSERT_EQ(OStatus::CODE_200.code, response->getStatusCode()) << response->readBodyToString()->c_str(); + + auto vecs_result_json = nlohmann::json::parse(response->readBodyToString()->c_str()); + ASSERT_TRUE(vecs_result_json.contains("vectors")); + auto vecs_json = vecs_result_json["vectors"]; + ASSERT_TRUE(vecs_json.is_array()); + ASSERT_EQ(10, vecs_json.size()); +} + +TEST_F(WebControllerTest, SEGMENT_FILTER) { + OString table_name = OString("test_milvus_web_segment_filter_test_") + RandomName().c_str(); + GenTable(table_name, 16, 1, "L2"); + + auto status = InsertData(table_name, 16, 1000); + ASSERT_TRUE(status.ok()) << status.message(); + + status = GenPartition(table_name, "tag01"); + ASSERT_TRUE(status.ok()) << status.message(); + + status = InsertData(table_name, 16, 1000, "tag01"); + ASSERT_TRUE(status.ok()) << status.message(); + + status = GenPartition(table_name, "tag02"); + ASSERT_TRUE(status.ok()) << status.message(); + + status = InsertData(table_name, 16, 1000, "tag02"); + ASSERT_TRUE(status.ok()) << status.message(); + + // show segments filtering tag + auto response = client_ptr->showSegments(table_name, "0", "10", "_default", conncetion_ptr); + ASSERT_EQ(OStatus::CODE_200.code, response->getStatusCode()) << response->readBodyToString()->c_str(); + + auto result_json = nlohmann::json::parse(response->readBodyToString()->c_str()); + ASSERT_TRUE(result_json.contains("count")); + + ASSERT_TRUE(result_json.contains("segments")); + auto segments_json = result_json["segments"]; + ASSERT_TRUE(segments_json.is_array()); + for (auto & s : segments_json) { + ASSERT_TRUE(s.contains("partition_tag")); + ASSERT_EQ("_default", s["partition_tag"].get()); + } +} + TEST_F(WebControllerTest, SEARCH) { const OString table_name = "test_search_table_test" + OString(RandomName().c_str()); GenTable(table_name, 64, 100, "L2"); // Insert 200 vectors into table - OQueryParams query_params; - auto insert_dto = milvus::server::web::InsertRequestDto::createShared(); - insert_dto->ids = insert_dto->ids->createShared(); - insert_dto->records = RandomRecordsDto(64, 200); // insert_dto->records->createShared(); - - auto response = client_ptr->insert(table_name, insert_dto, conncetion_ptr); - ASSERT_EQ(OStatus::CODE_201.code, response->getStatusCode()); - auto insert_result_dto = response->readBodyToDto(object_mapper.get()); - ASSERT_EQ(200, insert_result_dto->ids->count()); - - sleep(4); + auto status = InsertData(table_name, 64, 200); + ASSERT_TRUE(status.ok()) << status.message(); // Create partition and insert 200 vectors into it auto par_param = milvus::server::web::PartitionRequestDto::createShared(); par_param->partition_tag = "tag" + OString(RandomName().c_str()); - response = client_ptr->createPartition(table_name, par_param); + auto response = client_ptr->createPartition(table_name, par_param); ASSERT_EQ(OStatus::CODE_201.code, response->getStatusCode()) << "Error: " << response->getStatusDescription()->std_str(); - insert_dto->tag = par_param->partition_tag; - response = client_ptr->insert(table_name, insert_dto, conncetion_ptr); - ASSERT_EQ(OStatus::CODE_201.code, response->getStatusCode()); - sleep(2); + status = InsertData(table_name, 64, 200, par_param->partition_tag->std_str()); + ASSERT_TRUE(status.ok()) << status.message(); // Test search - auto search_request_dto = milvus::server::web::SearchRequestDto::createShared(); - response = client_ptr->search(table_name, search_request_dto, conncetion_ptr); + nlohmann::json search_json; + response = client_ptr->vectorsOp(table_name, search_json.dump().c_str(), conncetion_ptr); auto error_dto = response->readBodyToDto(object_mapper.get()); - ASSERT_EQ(milvus::server::web::StatusCode::BODY_FIELD_LOSS, error_dto->code); + ASSERT_NE(milvus::server::web::StatusCode::SUCCESS, error_dto->code); - search_request_dto->nprobe = 1; - response = client_ptr->search(table_name, search_request_dto, conncetion_ptr); + search_json["search"]["nprobe"] = 1; + response = client_ptr->vectorsOp(table_name, search_json.dump().c_str(), conncetion_ptr); error_dto = response->readBodyToDto(object_mapper.get()); ASSERT_EQ(milvus::server::web::StatusCode::BODY_FIELD_LOSS, error_dto->code); - search_request_dto->topk = 1; - response = client_ptr->search(table_name, search_request_dto, conncetion_ptr); + search_json["search"]["topk"] = 1; + response = client_ptr->vectorsOp(table_name, search_json.dump().c_str(), conncetion_ptr); error_dto = response->readBodyToDto(object_mapper.get()); - ASSERT_EQ(milvus::server::web::StatusCode::BODY_FIELD_LOSS, error_dto->code); + ASSERT_NE(milvus::server::web::StatusCode::SUCCESS, error_dto->code); - search_request_dto->records = RandomRecordsDto(64, 10); - response = client_ptr->search(table_name, search_request_dto, conncetion_ptr); + search_json["search"]["vectors"] = RandomRecordsJson(64, 10); + response = client_ptr->vectorsOp(table_name, search_json.dump().c_str(), conncetion_ptr); ASSERT_EQ(OStatus::CODE_200.code, response->getStatusCode()); - auto result_dto = response->readBodyToDto(object_mapper.get()); - ASSERT_EQ(10, result_dto->num); - ASSERT_EQ(10, result_dto->results->count()); - ASSERT_EQ(1, result_dto->results->get(0)->count()); + + auto result_json = nlohmann::json::parse(response->readBodyToString()->std_str()); + ASSERT_TRUE(result_json.contains("num")); + ASSERT_TRUE(result_json["num"].is_number()); + ASSERT_EQ(10, result_json["num"].get()); + + ASSERT_TRUE(result_json.contains("result")); + ASSERT_TRUE(result_json["result"].is_array()); + + auto result0_json = result_json["result"][0]; + ASSERT_TRUE(result0_json.is_array()); + ASSERT_EQ(1, result0_json.size()); // Test search with tags - search_request_dto->tags = search_request_dto->tags->createShared(); - search_request_dto->tags->pushBack(par_param->partition_tag); - response = client_ptr->search(table_name, search_request_dto, conncetion_ptr); + nlohmann::json par_json; + par_json.push_back(par_param->partition_tag->std_str()); + search_json["search"]["partition_tags"] = par_json; + + response = client_ptr->vectorsOp(table_name, search_json.dump().c_str(), conncetion_ptr); ASSERT_EQ(OStatus::CODE_200.code, response->getStatusCode()); // Test search without existing table - response = client_ptr->search(table_name + "999piyanning", search_request_dto, conncetion_ptr); + response = client_ptr->vectorsOp(table_name + "999piyanning", search_json.dump().c_str(), conncetion_ptr); ASSERT_EQ(OStatus::CODE_404.code, response->getStatusCode()); error_dto = response->readBodyToDto(object_mapper.get()); ASSERT_EQ(milvus::server::web::StatusCode::TABLE_NOT_EXISTS, error_dto->code->getValue()); @@ -1154,55 +1403,170 @@ TEST_F(WebControllerTest, SEARCH_BIN) { GenTable(table_name, 64, 100, "HAMMING"); // Insert 200 vectors into table - OQueryParams query_params; - auto insert_dto = milvus::server::web::InsertRequestDto::createShared(); - insert_dto->ids = insert_dto->ids->createShared(); - insert_dto->records_bin = RandomBinRecordsDto(64, 200); - - auto response = client_ptr->insert(table_name, insert_dto, conncetion_ptr); - ASSERT_EQ(OStatus::CODE_201.code, response->getStatusCode()); - - sleep(4); + auto status = InsertData(table_name, 64, 200, "", true); + ASSERT_TRUE(status.ok()) << status.message(); // Create partition and insert 200 vectors into it auto par_param = milvus::server::web::PartitionRequestDto::createShared(); par_param->partition_tag = "tag" + OString(RandomName().c_str()); - response = client_ptr->createPartition(table_name, par_param); + auto response = client_ptr->createPartition(table_name, par_param); ASSERT_EQ(OStatus::CODE_201.code, response->getStatusCode()) - << "Error: " << response->getStatusDescription()->std_str(); + << "Error: " << response->readBodyToString()->std_str(); - insert_dto->tag = par_param->partition_tag; - response = client_ptr->insert(table_name, insert_dto, conncetion_ptr); - ASSERT_EQ(OStatus::CODE_201.code, response->getStatusCode()); - sleep(5); + status = InsertData(table_name, 64, 200, par_param->partition_tag->std_str(), true); + ASSERT_TRUE(status.ok()) << status.message(); // Test search - auto search_request_dto = milvus::server::web::SearchRequestDto::createShared(); - response = client_ptr->search(table_name, search_request_dto, conncetion_ptr); + nlohmann::json search_json; + response = client_ptr->vectorsOp(table_name, search_json.dump().c_str(), conncetion_ptr); auto result_dto = response->readBodyToDto(object_mapper.get()); - ASSERT_EQ(milvus::server::web::StatusCode::BODY_FIELD_LOSS, result_dto->code); + ASSERT_NE(milvus::server::web::StatusCode::SUCCESS, result_dto->code); - search_request_dto->nprobe = 1; - response = client_ptr->search(table_name, search_request_dto, conncetion_ptr); + search_json["search"]["nprobe"] = 1; + response = client_ptr->vectorsOp(table_name, search_json.dump().c_str(), conncetion_ptr); result_dto = response->readBodyToDto(object_mapper.get()); - ASSERT_EQ(milvus::server::web::StatusCode::BODY_FIELD_LOSS, result_dto->code); + ASSERT_NE(milvus::server::web::StatusCode::SUCCESS, result_dto->code); - search_request_dto->topk = 1; - response = client_ptr->search(table_name, search_request_dto, conncetion_ptr); + search_json["search"]["topk"] = 1; + response = client_ptr->vectorsOp(table_name, search_json.dump().c_str(), conncetion_ptr); result_dto = response->readBodyToDto(object_mapper.get()); - ASSERT_EQ(milvus::server::web::StatusCode::BODY_FIELD_LOSS, result_dto->code); + ASSERT_NE(milvus::server::web::StatusCode::SUCCESS, result_dto->code); - search_request_dto->records_bin = RandomBinRecordsDto(64, 10); - response = client_ptr->search(table_name, search_request_dto, conncetion_ptr); + search_json["search"]["vectors"] = RandomBinRecordsJson(64, 10); + response = client_ptr->vectorsOp(table_name, search_json.dump().c_str(), conncetion_ptr); ASSERT_EQ(OStatus::CODE_200.code, response->getStatusCode()); + // validate search result + auto result_json = nlohmann::json::parse(response->readBodyToString()->c_str()); + ASSERT_TRUE(result_json.contains("result")); + ASSERT_TRUE(result_json["result"].is_array()); + ASSERT_EQ(10, result_json["result"].size()); + + auto result0_json = result_json["result"][0]; + ASSERT_TRUE(result0_json.is_array()); + ASSERT_EQ(1, result0_json.size()); + // Test search with tags - search_request_dto->tags = search_request_dto->tags->createShared(); - search_request_dto->tags->pushBack(par_param->partition_tag); - response = client_ptr->search(table_name, search_request_dto, conncetion_ptr); + search_json["search"]["partition_tags"] = std::vector(); + search_json["search"]["partition_tags"].push_back(par_param->partition_tag->std_str()); + response = client_ptr->vectorsOp(table_name, search_json.dump().c_str(), conncetion_ptr); ASSERT_EQ(OStatus::CODE_200.code, response->getStatusCode()); } +TEST_F(WebControllerTest, SEARCH_BY_ID) { +#ifdef MILVUS_GPU_VERSION + auto &config = milvus::server::Config::GetInstance(); + auto config_status = config.SetGpuResourceConfigEnable("false"); + ASSERT_TRUE(config_status.ok()) << config_status.message(); +#endif + + const OString table_name = "test_search_by_id_table_test_" + OString(RandomName().c_str()); + GenTable(table_name, 64, 100, "L2"); + + // Insert 100 vectors into table + std::vector ids; + for (size_t i = 0; i < 100; i++) { + ids.emplace_back(i); + } + + auto status = InsertData(table_name, 64, 100, ids); + ASSERT_TRUE(status.ok()) << status.message(); + + nlohmann::json search_json; + search_json["search"]["topk"] = 1; + search_json["search"]["nprobe"] = 1; + search_json["search"]["vector_id"] = ids.at(0); + + auto response = client_ptr->vectorsOp(table_name, search_json.dump().c_str(), conncetion_ptr); + ASSERT_EQ(OStatus::CODE_200.code, response->getStatusCode()) << response->readBodyToString()->c_str(); + + // validate search result + auto result_json = nlohmann::json::parse(response->readBodyToString()->c_str()); + ASSERT_TRUE(result_json.contains("result")); + ASSERT_TRUE(result_json["result"].is_array()); + ASSERT_EQ(1, result_json["result"].size()); + + auto result0_json = result_json["result"][0]; + ASSERT_TRUE(result0_json.is_array()); + ASSERT_EQ(1, result0_json.size()); + + auto result0_top0_json = result0_json[0]; + ASSERT_TRUE(result0_top0_json.contains("id")); + + auto id = result0_top0_json["id"]; + ASSERT_TRUE(id.is_string()); + ASSERT_EQ(std::to_string(ids.at(0)), id); +} + +TEST_F(WebControllerTest, GET_VECTOR_BY_ID) { + const OString table_name = "test_milvus_web_get_vector_by_id_test_" + OString(RandomName().c_str()); + GenTable(table_name, 64, 100, "L2"); + + // Insert 100 vectors into table + std::vector ids; + for (size_t i = 0; i < 100; i++) { + ids.emplace_back(i); + } + + auto status = InsertData(table_name, 64, 100, ids); + ASSERT_TRUE(status.ok()) << status.message(); + + /* test task load */ + auto id_str = std::to_string(ids.at(0)); + auto response = client_ptr->getVectors(table_name, id_str.c_str(), conncetion_ptr); + ASSERT_EQ(OStatus::CODE_200.code, response->getStatusCode()) << response->readBodyToString()->c_str(); + + // validate result + auto result_json = nlohmann::json::parse(response->readBodyToString()->c_str()); + ASSERT_TRUE(result_json.contains("vectors")); + + auto vectors_json = result_json["vectors"]; + ASSERT_TRUE(vectors_json.is_array()); + + auto vector_json = vectors_json[0]; + ASSERT_TRUE(vector_json.contains("id")); + ASSERT_EQ(std::to_string(ids[0]), vector_json["id"].get()); + ASSERT_TRUE(vector_json.contains("vector")); + + auto vec_json = vector_json["vector"]; + ASSERT_TRUE(vec_json.is_array()); + std::vector vec; + for (auto & v : vec_json) { + vec.emplace_back(v.get()); + } + + ASSERT_EQ(64, vec.size()); +} + +TEST_F(WebControllerTest, DELETE_BY_ID) { + const OString table_name = "test_search_bin_table_test" + OString(RandomName().c_str()); + GenTable(table_name, 64, 100, "L2"); + + // Insert 200 vectors into table + nlohmann::json insert_json; + insert_json["vectors"] = RandomRecordsJson(64, 2000); + auto response = client_ptr->insert(table_name, insert_json.dump().c_str(), conncetion_ptr); + ASSERT_EQ(OStatus::CODE_201.code, response->getStatusCode()) << response->readBodyToString()->c_str(); + + auto insert_result_json = nlohmann::json::parse(response->readBodyToString()->c_str()); + ASSERT_TRUE(insert_result_json.contains("ids")); + auto ids_json = insert_result_json["ids"]; + ASSERT_TRUE(ids_json.is_array()); + + std::vector ids; + for (auto & id : ids_json) { + ids.emplace_back(std::stol(id.get())); + } + + auto delete_ids = std::vector(ids.begin(), ids.begin() + 10); + + nlohmann::json delete_json; + delete_json["delete"]["ids"] = delete_ids; + + response = client_ptr->vectorsOp(table_name, delete_json.dump().c_str(), conncetion_ptr); + ASSERT_EQ(OStatus::CODE_200.code, response->getStatusCode()) << response->readBodyToString()->c_str(); +} + TEST_F(WebControllerTest, CMD) { auto response = client_ptr->cmd("status", "", "", conncetion_ptr); ASSERT_EQ(OStatus::CODE_200.code, response->getStatusCode()); @@ -1210,6 +1574,7 @@ TEST_F(WebControllerTest, CMD) { response = client_ptr->cmd("version", "", "", conncetion_ptr); ASSERT_EQ(OStatus::CODE_200.code, response->getStatusCode()); + // test invalid body response = client_ptr->cmd("mode", "", "", conncetion_ptr); ASSERT_EQ(OStatus::CODE_200.code, response->getStatusCode()); @@ -1218,49 +1583,32 @@ TEST_F(WebControllerTest, CMD) { response = client_ptr->cmd("info", "", "", conncetion_ptr); ASSERT_EQ(OStatus::CODE_200.code, response->getStatusCode()); - - response = client_ptr->cmd("info", "", "", conncetion_ptr); - ASSERT_EQ(OStatus::CODE_200.code, response->getStatusCode()); - - response = client_ptr->cmd("config", "", "", conncetion_ptr); - ASSERT_EQ(OStatus::CODE_200.code, response->getStatusCode()) << response->readBodyToString()->c_str(); } -TEST_F(WebControllerTest, SYSTEM_OP_TEST) { +TEST_F(WebControllerTest, CONFIG) { std::string config_path = std::string(CONTROLLER_TEST_CONFIG_DIR).append(CONTROLLER_TEST_CONFIG_FILE); std::fstream fs(config_path.c_str(), std::ios_base::out); fs << CONTROLLER_TEST_VALID_CONFIG_STR; fs.flush(); + fs.close(); milvus::server::Config& config = milvus::server::Config::GetInstance(); auto status = config.LoadConfigFile(config_path); ASSERT_TRUE(status.ok()) << status.message(); - /* test task load */ - auto response = - client_ptr->exec("task", "{\"load\": {\"table_name\": \"milvus_non_existent_table_test\"}}", conncetion_ptr); - ASSERT_EQ(OStatus::CODE_400.code, response->getStatusCode()); +#ifdef MILVUS_GPU_VERSION + status = config.SetGpuResourceConfigEnable("true"); + ASSERT_TRUE(status.ok()) << status.message(); + status = config.SetGpuResourceConfigCacheCapacity("1"); + ASSERT_TRUE(status.ok()) << status.message(); + status = config.SetGpuResourceConfigBuildIndexResources("gpu0"); + ASSERT_TRUE(status.ok()) << status.message(); + status = config.SetGpuResourceConfigSearchResources("gpu0"); + ASSERT_TRUE(status.ok()) << status.message(); +#endif - /* test flush */ - response = - client_ptr->exec("task", "{\"flush\": {\"table_names\": \"milvus_non_existent_table_test\"}}", conncetion_ptr); - ASSERT_EQ(OStatus::CODE_400.code, response->getStatusCode()); - - response = client_ptr->exec( - "task", "{\"flush\": {\"table_names\": \"[milvus_non_existent_table_test_for_flush]\"}}", conncetion_ptr); - ASSERT_EQ(OStatus::CODE_400.code, response->getStatusCode()); - - response = client_ptr->exec("config", "{\"cache_config\": {\"cpu_cache_capacity\": 1}}", conncetion_ptr); - ASSERT_EQ(OStatus::CODE_200.code, response->getStatusCode()); - - response = client_ptr->exec("config", "{\"cache_config\": {\"cpu_cache_capacity\": 10000}}", conncetion_ptr); - ASSERT_EQ(OStatus::CODE_400.code, response->getStatusCode()); - - // test invalid body - response = client_ptr->exec("config", "{1}}", conncetion_ptr); - ASSERT_EQ(OStatus::CODE_400.code, response->getStatusCode()); - - fs.close(); + auto response = client_ptr->cmd("config", "", "", conncetion_ptr); + ASSERT_EQ(OStatus::CODE_200.code, response->getStatusCode()) << response->readBodyToString()->c_str(); } TEST_F(WebControllerTest, ADVANCED_CONFIG) { @@ -1291,18 +1639,16 @@ TEST_F(WebControllerTest, ADVANCED_CONFIG) { ASSERT_EQ(OStatus::CODE_200.code, response->getStatusCode()); #ifdef MILVUS_GPU_VERSION - config_dto->gpu_search_threshold = 1000; response = client_ptr->setAdvanced(config_dto, conncetion_ptr); ASSERT_EQ(OStatus::CODE_200.code, response->getStatusCode()); - #endif config_dto->use_blas_threshold = 1000; response = client_ptr->setAdvanced(config_dto, conncetion_ptr); ASSERT_EQ(OStatus::CODE_200.code, response->getStatusCode()); - //// test fault + // test fault // cpu cache capacity exceed total memory config_dto->cpu_cache_capacity = 10000000; response = client_ptr->setAdvanced(config_dto, conncetion_ptr); @@ -1372,10 +1718,62 @@ TEST_F(WebControllerTest, GPU_CONFIG) { response = client_ptr->setGPUConfig(gpu_config_dto, conncetion_ptr); ASSERT_EQ(OStatus::CODE_400.code, response->getStatusCode()); } - #endif TEST_F(WebControllerTest, DEVICES_CONFIG) { - auto response = WebControllerTest::client_ptr->getDevices(conncetion_ptr); + auto response = client_ptr->getDevices(conncetion_ptr); ASSERT_EQ(OStatus::CODE_200.code, response->getStatusCode()); } + +TEST_F(WebControllerTest, FLUSH) { + auto table_name = milvus::server::web::OString(TABLE_NAME) + RandomName().c_str(); + GenTable(table_name, 16, 10, "L2"); + + auto status = InsertData(table_name, 16, 1000); + ASSERT_TRUE(status.ok()) << status.message(); + + nlohmann::json flush_json; + flush_json["flush"]["table_names"] = {table_name->std_str()}; + auto response = client_ptr->op("task", flush_json.dump().c_str(), conncetion_ptr); + ASSERT_EQ(OStatus::CODE_200.code, response->getStatusCode()); + + // invalid payload format + flush_json["flush"]["table_names"] = table_name->std_str(); + response = client_ptr->op("task", flush_json.dump().c_str(), conncetion_ptr); + ASSERT_EQ(OStatus::CODE_400.code, response->getStatusCode()); + + // non-existent name + flush_json["flush"]["table_names"] = {"afafaf444353"}; + response = client_ptr->op("task", flush_json.dump().c_str(), conncetion_ptr); + ASSERT_EQ(OStatus::CODE_400.code, response->getStatusCode()); +} + +TEST_F(WebControllerTest, COMPACT) { + auto table_name = milvus::server::web::OString("milvus_web_test_compact_") + RandomName().c_str(); + GenTable(table_name, 16, 10, "L2"); + + auto status = InsertData(table_name, 16, 1000); + ASSERT_TRUE(status.ok()) << status.message(); + + nlohmann::json compact_json; + compact_json["compact"]["table_name"] = table_name->std_str(); + auto response = client_ptr->op("task", compact_json.dump().c_str(), conncetion_ptr); + ASSERT_EQ(OStatus::CODE_200.code, response->getStatusCode()) << response->readBodyToString()->c_str(); +} + +TEST_F(WebControllerTest, LOAD) { + OString table_name = "milvus_web_test_load_" + OString(RandomName().c_str()); + GenTable(table_name, 128, 100, "L2"); + + nlohmann::json load_json; + load_json["load"]["table_name"] = table_name->c_str(); + + auto response = client_ptr->op("task", load_json.dump().c_str(), conncetion_ptr); + ASSERT_EQ(OStatus::CODE_200.code, response->getStatusCode()); + + // load with a non-existent name + load_json["load"]["table_name"] = "sssssssssssssssssssssssfsfsfsrrrttt"; + + response = client_ptr->op("task", load_json.dump().c_str(), conncetion_ptr); + ASSERT_EQ(OStatus::CODE_400.code, response->getStatusCode()); +} diff --git a/tests/milvus_python_test/test_table_info.py b/tests/milvus_python_test/test_table_info.py index c9c5476a92..0c6d6dd4ef 100644 --- a/tests/milvus_python_test/test_table_info.py +++ b/tests/milvus_python_test/test_table_info.py @@ -286,4 +286,4 @@ class TestTableInfoBase: index_type = index_param["index_type"] match = self.index_string_convert(index_string, index_type) assert match - assert nb == info.partitions_stat[0].segments_stat[0].count \ No newline at end of file + assert nb == info.partitions_stat[0].segments_stat[0].count