diff --git a/CHANGELOG.md b/CHANGELOG.md index 9dbb3a24aa..4568a001f8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,7 @@ Please mark all change in change log and use the issue from GitHub - \#2466 optimize k-selection implemention of faiss gpu version - \#2489 Add exception throw on mysql meta error - \#2495 Add creating lock file failure reason. +- \#2516 Improve unit test coverage ## Task diff --git a/ci/jenkins/Jenkinsfile b/ci/jenkins/Jenkinsfile index f6c51c0001..88c79345d8 100644 --- a/ci/jenkins/Jenkinsfile +++ b/ci/jenkins/Jenkinsfile @@ -30,7 +30,7 @@ pipeline { LOWER_BUILD_TYPE = params.BUILD_TYPE.toLowerCase() SEMVER = "${BRANCH_NAME.contains('/') ? BRANCH_NAME.substring(BRANCH_NAME.lastIndexOf('/') + 1) : BRANCH_NAME}" PIPELINE_NAME = "milvus-ci" - HELM_BRANCH = "0.10.0" + HELM_BRANCH = "master" } stages { diff --git a/core/src/server/delivery/hybrid_request/HybridSearchRequest.cpp b/core/src/server/delivery/hybrid_request/HybridSearchRequest.cpp index 7e768b277d..8ef7731f69 100644 --- a/core/src/server/delivery/hybrid_request/HybridSearchRequest.cpp +++ b/core/src/server/delivery/hybrid_request/HybridSearchRequest.cpp @@ -54,7 +54,7 @@ HybridSearchRequest::Create(const std::shared_ptr& cont Status HybridSearchRequest::OnExecute() { try { - fiu_do_on("SearchRequest.OnExecute.throw_std_exception", throw std::exception()); + fiu_do_on("HybridSearchRequest.OnExecute.throw_std_exception", throw std::exception()); std::string hdr = "SearchRequest(table=" + collection_name_; TimeRecorder rc(hdr); @@ -71,7 +71,8 @@ HybridSearchRequest::OnExecute() { engine::meta::hybrid::FieldsSchema fields_schema; collection_schema.collection_id_ = collection_name_; status = DBWrapper::DB()->DescribeHybridCollection(collection_schema, fields_schema); - fiu_do_on("SearchRequest.OnExecute.describe_table_fail", status = Status(milvus::SERVER_UNEXPECTED_ERROR, "")); + fiu_do_on("HybridSearchRequest.OnExecute.describe_table_fail", + status = Status(milvus::SERVER_UNEXPECTED_ERROR, "")); if (!status.ok()) { if (status.code() == DB_NOT_FOUND) { return Status(SERVER_COLLECTION_NOT_EXIST, CollectionNotExistMsg(collection_name_)); @@ -102,11 +103,11 @@ HybridSearchRequest::OnExecute() { ProfilerStop(); #endif - fiu_do_on("SearchRequest.OnExecute.query_fail", status = Status(milvus::SERVER_UNEXPECTED_ERROR, "")); + fiu_do_on("HybridSearchRequest.OnExecute.query_fail", status = Status(milvus::SERVER_UNEXPECTED_ERROR, "")); if (!status.ok()) { return status; } - fiu_do_on("SearchRequest.OnExecute.empty_result_ids", result_ids.clear()); + fiu_do_on("HybridSearchRequest.OnExecute.empty_result_ids", result_ids.clear()); if (result_ids.empty()) { return Status::OK(); // empty table } diff --git a/core/src/server/delivery/request/ReLoadSegmentsRequest.cpp b/core/src/server/delivery/request/ReLoadSegmentsRequest.cpp index fe3250c7a7..de34e3315b 100644 --- a/core/src/server/delivery/request/ReLoadSegmentsRequest.cpp +++ b/core/src/server/delivery/request/ReLoadSegmentsRequest.cpp @@ -11,6 +11,8 @@ #include "server/delivery/request/ReLoadSegmentsRequest.h" +#include + #include "config/Config.h" #include "server/DBWrapper.h" #include "utils/TimeRecorder.h" @@ -41,6 +43,7 @@ ReLoadSegmentsRequest::OnExecute() { return status; } + fiu_do_on("ReLoadSegmentsRequest.OnExecute.readonly", deploy_mode = "cluster_readonly"); if (deploy_mode == "single" || deploy_mode == "cluster_writable") { // TODO: No need to reload segment files return Status(SERVER_SUCCESS, ""); diff --git a/core/src/server/init/InstanceLockCheck.cpp b/core/src/server/init/InstanceLockCheck.cpp index 0fca2fd99a..bd89f9631c 100644 --- a/core/src/server/init/InstanceLockCheck.cpp +++ b/core/src/server/init/InstanceLockCheck.cpp @@ -10,13 +10,16 @@ // or implied. See the License for the specific language governing permissions and limitations under the License. #include "server/init/InstanceLockCheck.h" -#include "utils/Log.h" #include #include #include #include +#include + +#include "utils/Log.h" + namespace milvus { namespace server { @@ -24,6 +27,7 @@ Status InstanceLockCheck::Check(const std::string& path) { std::string lock_path = path + "/lock"; auto fd = open(lock_path.c_str(), O_RDWR | O_CREAT | O_NOFOLLOW, 0640); + fiu_do_on("InstanceLockCheck.Check.fd", fd = -1); if (fd < 0) { std::string msg; if (errno == EROFS) { @@ -41,7 +45,9 @@ InstanceLockCheck::Check(const std::string& path) { fl.l_whence = SEEK_SET; fl.l_start = 0; fl.l_len = 0; - if (fcntl(fd, F_SETLK, &fl) == -1) { + auto fcl = fcntl(fd, F_SETLK, &fl); + fiu_do_on("InstanceLockCheck.Check.fcntl", fcl = -1); + if (fcl == -1) { std::string msg = "Can't lock file: " + lock_path + ", due to "; if (errno == EACCES || errno == EAGAIN) { msg += "permission denied. "; diff --git a/core/src/server/web_impl/dto/VectorDto.hpp b/core/src/server/web_impl/dto/VectorDto.hpp index 01b73fdc51..54f3a1ac11 100644 --- a/core/src/server/web_impl/dto/VectorDto.hpp +++ b/core/src/server/web_impl/dto/VectorDto.hpp @@ -26,20 +26,6 @@ class VectorIdsDto : public oatpp::data::mapping::type::Object { DTO_FIELD(List::ObjectWrapper, ids); }; -class ResultDto : public oatpp::data::mapping::type::Object { - DTO_INIT(ResultDto, Object) - - DTO_FIELD(String, id); - DTO_FIELD(String, dit, "distance"); -}; - -class TopkResultsDto : public OObject { - DTO_INIT(TopkResultsDto, Object); - - DTO_FIELD(Int64, num); - DTO_FIELD(List::ObjectWrapper>::ObjectWrapper, results); -}; - #include OATPP_CODEGEN_END(DTO) } // namespace web diff --git a/core/src/server/web_impl/handler/WebRequestHandler.cpp b/core/src/server/web_impl/handler/WebRequestHandler.cpp index da96659e66..22ce4880d0 100644 --- a/core/src/server/web_impl/handler/WebRequestHandler.cpp +++ b/core/src/server/web_impl/handler/WebRequestHandler.cpp @@ -12,12 +12,13 @@ #include "server/web_impl/handler/WebRequestHandler.h" #include -#include #include #include #include #include +#include + #include "config/Config.h" #include "metrics/SystemInfo.h" #include "server/delivery/request/BaseRequest.h" @@ -1806,6 +1807,9 @@ WebRequestHandler::SystemOp(const OString& op, const OString& body_str, OString& Status status = Status::OK(); std::string result_str; try { + fiu_do_on("WebRequestHandler.SystemOp.raise_parse_error", + throw nlohmann::detail::parse_error::create(0, 0, "")); + fiu_do_on("WebRequestHandler.SystemOp.raise_type_error", throw nlohmann::detail::type_error::create(0, "")); nlohmann::json j = nlohmann::json::parse(body_str->c_str()); if (op->equals("task")) { if (j.contains("load")) { diff --git a/core/src/storage/disk/DiskOperation.cpp b/core/src/storage/disk/DiskOperation.cpp index 5c6b1d08e4..8ce5af10ce 100644 --- a/core/src/storage/disk/DiskOperation.cpp +++ b/core/src/storage/disk/DiskOperation.cpp @@ -17,6 +17,8 @@ #include +#include + #include "storage/disk/DiskOperation.h" #include "utils/Exception.h" #include "utils/Log.h" @@ -29,8 +31,11 @@ DiskOperation::DiskOperation(const std::string& dir_path) : dir_path_(dir_path) void DiskOperation::CreateDirectory() { - if (!boost::filesystem::is_directory(dir_path_)) { + bool is_dir = boost::filesystem::is_directory(dir_path_); + fiu_do_on("DiskOperation.CreateDirectory.is_directory", is_dir = false); + if (!is_dir) { auto ret = boost::filesystem::create_directory(dir_path_); + fiu_do_on("DiskOperation.CreateDirectory.create_directory", ret = false); if (!ret) { std::string err_msg = "Failed to create directory: " + dir_path_; LOG_ENGINE_ERROR_ << err_msg; diff --git a/core/src/utils/LogUtil.cpp b/core/src/utils/LogUtil.cpp index 60eba4074c..d6883bafb7 100644 --- a/core/src/utils/LogUtil.cpp +++ b/core/src/utils/LogUtil.cpp @@ -15,6 +15,7 @@ #include #include +#include #include #include @@ -137,6 +138,7 @@ InitLog(bool trace_enable, bool debug_enable, bool info_enable, bool warning_ena std::string info_log_path = logs_reg_path + "milvus-%datetime{%y-%M-%d-%H:%m}-info.log"; defaultConf.set(el::Level::Info, el::ConfigurationType::Filename, info_log_path.c_str()); + fiu_do_on("LogUtil.InitLog.info_enable_to_false", info_enable = false); if (info_enable) { defaultConf.set(el::Level::Info, el::ConfigurationType::Enabled, "true"); } else { @@ -145,6 +147,7 @@ InitLog(bool trace_enable, bool debug_enable, bool info_enable, bool warning_ena std::string debug_log_path = logs_reg_path + "milvus-%datetime{%y-%M-%d-%H:%m}-debug.log"; defaultConf.set(el::Level::Debug, el::ConfigurationType::Filename, debug_log_path.c_str()); + fiu_do_on("LogUtil.InitLog.debug_enable_to_false", debug_enable = false); if (debug_enable) { defaultConf.set(el::Level::Debug, el::ConfigurationType::Enabled, "true"); } else { @@ -153,6 +156,7 @@ InitLog(bool trace_enable, bool debug_enable, bool info_enable, bool warning_ena std::string warning_log_path = logs_reg_path + "milvus-%datetime{%y-%M-%d-%H:%m}-warning.log"; defaultConf.set(el::Level::Warning, el::ConfigurationType::Filename, warning_log_path.c_str()); + fiu_do_on("LogUtil.InitLog.warning_enable_to_false", warning_enable = false); if (warning_enable) { defaultConf.set(el::Level::Warning, el::ConfigurationType::Enabled, "true"); } else { @@ -161,6 +165,7 @@ InitLog(bool trace_enable, bool debug_enable, bool info_enable, bool warning_ena std::string trace_log_path = logs_reg_path + "milvus-%datetime{%y-%M-%d-%H:%m}-trace.log"; defaultConf.set(el::Level::Trace, el::ConfigurationType::Filename, trace_log_path.c_str()); + fiu_do_on("LogUtil.InitLog.trace_enable_to_false", trace_enable = false); if (trace_enable) { defaultConf.set(el::Level::Trace, el::ConfigurationType::Enabled, "true"); } else { @@ -169,6 +174,7 @@ InitLog(bool trace_enable, bool debug_enable, bool info_enable, bool warning_ena std::string error_log_path = logs_reg_path + "milvus-%datetime{%y-%M-%d-%H:%m}-error.log"; defaultConf.set(el::Level::Error, el::ConfigurationType::Filename, error_log_path.c_str()); + fiu_do_on("LogUtil.InitLog.error_enable_to_false", error_enable = false); if (error_enable) { defaultConf.set(el::Level::Error, el::ConfigurationType::Enabled, "true"); } else { @@ -177,12 +183,15 @@ InitLog(bool trace_enable, bool debug_enable, bool info_enable, bool warning_ena std::string fatal_log_path = logs_reg_path + "milvus-%datetime{%y-%M-%d-%H:%m}-fatal.log"; defaultConf.set(el::Level::Fatal, el::ConfigurationType::Filename, fatal_log_path.c_str()); + fiu_do_on("LogUtil.InitLog.fatal_enable_to_false", fatal_enable = false); if (fatal_enable) { defaultConf.set(el::Level::Fatal, el::ConfigurationType::Enabled, "true"); } else { defaultConf.set(el::Level::Fatal, el::ConfigurationType::Enabled, "false"); } + fiu_do_on("LogUtil.InitLog.set_max_log_size_small_than_min", + max_log_file_size = CONFIG_LOGS_MAX_LOG_FILE_SIZE_MIN - 1); if (max_log_file_size < CONFIG_LOGS_MAX_LOG_FILE_SIZE_MIN || max_log_file_size > CONFIG_LOGS_MAX_LOG_FILE_SIZE_MAX) { return Status(SERVER_UNEXPECTED_ERROR, "max_log_file_size must in range[" + @@ -198,6 +207,7 @@ InitLog(bool trace_enable, bool debug_enable, bool info_enable, bool warning_ena // set delete_exceeds = 0 means disable throw away log file even they reach certain limit. if (delete_exceeds != 0) { + fiu_do_on("LogUtil.InitLog.delete_exceeds_small_than_min", delete_exceeds = CONFIG_LOGS_LOG_ROTATE_NUM_MIN - 1); if (delete_exceeds < CONFIG_LOGS_LOG_ROTATE_NUM_MIN || delete_exceeds > CONFIG_LOGS_LOG_ROTATE_NUM_MAX) { return Status(SERVER_UNEXPECTED_ERROR, "delete_exceeds must in range[" + std::to_string(CONFIG_LOGS_LOG_ROTATE_NUM_MIN) + ", " + diff --git a/core/unittest/db/test_db.cpp b/core/unittest/db/test_db.cpp index a2d2762a42..1c5882d25d 100644 --- a/core/unittest/db/test_db.cpp +++ b/core/unittest/db/test_db.cpp @@ -1282,7 +1282,7 @@ TEST_F(DBTest2, INSERT_DUPLICATE_ID) { milvus::engine::meta::CollectionSchema collection_schema = BuildCollectionSchema(); auto stat = db_->CreateCollection(collection_schema); - ASSERT_TRUE(stat.ok()); + ASSERT_TRUE(stat.ok()) << " CreateCollection: " << stat.message(); uint64_t size = 20; milvus::engine::VectorsData vector; @@ -1293,10 +1293,10 @@ TEST_F(DBTest2, INSERT_DUPLICATE_ID) { } stat = db_->InsertVectors(COLLECTION_NAME, "", vector); - ASSERT_TRUE(stat.ok()); + ASSERT_TRUE(stat.ok()) << " InsertVectors: " << stat.message(); stat = db_->Flush(COLLECTION_NAME); - ASSERT_TRUE(stat.ok()); + ASSERT_TRUE(stat.ok()) << " Flush: " << stat.message(); } /* diff --git a/core/unittest/server/test_check.cpp b/core/unittest/server/test_check.cpp index e00312be78..c6d74a84c1 100644 --- a/core/unittest/server/test_check.cpp +++ b/core/unittest/server/test_check.cpp @@ -17,6 +17,7 @@ #include "config/Config.h" #include "server/init/CpuChecker.h" +#include "server/init/InstanceLockCheck.h" #ifdef MILVUS_GPU_VERSION #include "server/init/GpuChecker.h" #endif @@ -195,4 +196,20 @@ TEST_F(ServerCheckerTest, GPU_FAIL_TEST) { fiu_disable("GpuChecker.CheckGpuEnvironment.nvml_shutdown_fail"); } +TEST_F(ServerCheckerTest, LOCK_TEST) { + fiu_init(0); + fiu_enable("InstanceLockCheck.Check.fd", 1, NULL, 0); + auto status = milvus::server::InstanceLockCheck::Check(db_primary_path); + ASSERT_FALSE(status.ok()); + fiu_disable("InstanceLockCheck.Check.fd"); + + fiu_enable("InstanceLockCheck.Check.fcntl", 1, NULL, 0); + status = milvus::server::InstanceLockCheck::Check(db_primary_path); + ASSERT_FALSE(status.ok()); + fiu_disable("InstanceLockCheck.Check.fcntl"); + + status = milvus::server::InstanceLockCheck::Check(db_primary_path); + ASSERT_TRUE(status.ok()) << status.message(); +} + #endif diff --git a/core/unittest/server/test_rpc.cpp b/core/unittest/server/test_rpc.cpp index ad5de73cfb..49e625ceea 100644 --- a/core/unittest/server/test_rpc.cpp +++ b/core/unittest/server/test_rpc.cpp @@ -434,7 +434,7 @@ TEST_F(RpcHandlerTest, COMBINE_SEARCH_TEST) { handler->RegisterRequestHandler(milvus::server::RequestHandler()); // create collection - std::string collection_name = "combine"; + std::string collection_name = "search_combines"; ::milvus::grpc::CollectionSchema collection_schema; collection_schema.set_collection_name(collection_name); collection_schema.set_dimension(COLLECTION_DIM); @@ -442,7 +442,7 @@ TEST_F(RpcHandlerTest, COMBINE_SEARCH_TEST) { collection_schema.set_metric_type(1); // L2 metric ::milvus::grpc::Status status; handler->CreateCollection(&context, &collection_schema, &status); - ASSERT_EQ(status.error_code(), 0); + ASSERT_EQ(status.error_code(), 0) << status.reason(); // insert vectors std::vector> record_array; @@ -844,6 +844,128 @@ TEST_F(RpcHandlerTest, TABLES_TEST) { } } +TEST_F(RpcHandlerTest, GET_VECTOR_BY_ID_TEST) { + ::grpc::ServerContext context; + handler->SetContext(&context, dummy_context); + handler->RegisterRequestHandler(milvus::server::RequestHandler()); + + ::milvus::grpc::Status response; + + ::milvus::grpc::InsertParam request; + request.set_collection_name(COLLECTION_NAME); + std::vector> record_array; + BuildVectors(0, VECTOR_COUNT, record_array); + ::milvus::grpc::VectorIds vector_ids; + for (auto& record : record_array) { + ::milvus::grpc::RowRecord* grpc_record = request.add_row_record_array(); + CopyRowRecord(grpc_record, record); + } + handler->Insert(&context, &request, &vector_ids); + ASSERT_EQ(vector_ids.vector_id_array_size(), VECTOR_COUNT); + + ::milvus::grpc::FlushParam flush_param; + flush_param.add_collection_name_array(COLLECTION_NAME); + handler->Flush(&context, &flush_param, &response); + + ::milvus::grpc::VectorsIdentity vectors_identity; + vectors_identity.set_collection_name(COLLECTION_NAME); + for (size_t i = 0; i < 10; i++) { + vectors_identity.mutable_id_array()->Add(vector_ids.vector_id_array(i)); + } + + ::milvus::grpc::VectorsData vectors_data; + handler->GetVectorsByID(&context, &vectors_identity, &vectors_data); + ASSERT_EQ(10, vectors_data.vectors_data_size()); +} + +TEST_F(RpcHandlerTest, GET_BIN_VECTORS_BY_IDS_TEST) { + ::grpc::ServerContext context; + handler->SetContext(&context, dummy_context); + handler->RegisterRequestHandler(milvus::server::RequestHandler()); + + ::milvus::grpc::CollectionSchema collection_schema; + ::milvus::grpc::Status response; + std::string str_collection_name = "tbl_get_bin_vector_by_ids"; + collection_schema.set_collection_name(str_collection_name); + collection_schema.set_dimension(COLLECTION_DIM); + collection_schema.set_index_file_size(INDEX_FILE_SIZE); + collection_schema.set_metric_type(3); + handler->CreateCollection(&context, &collection_schema, &response); + + ::milvus::grpc::InsertParam request; + request.set_collection_name(str_collection_name); + std::vector> bin_record_array; + BuildBinVectors(0, VECTOR_COUNT, bin_record_array); + ::milvus::grpc::VectorIds vector_ids; + for (auto& record : bin_record_array) { + ::milvus::grpc::RowRecord* grpc_record = request.add_row_record_array(); + CopyBinRowRecord(grpc_record, record); + } + handler->Insert(&context, &request, &vector_ids); + ASSERT_EQ(vector_ids.vector_id_array_size(), VECTOR_COUNT); + + ::milvus::grpc::FlushParam flush_param; + flush_param.add_collection_name_array(str_collection_name); + handler->Flush(&context, &flush_param, &response); + + ::milvus::grpc::VectorsIdentity vectors_identity; + vectors_identity.set_collection_name(str_collection_name); + for (size_t i = 0; i < 10; i++) { + vectors_identity.mutable_id_array()->Add(vector_ids.vector_id_array(i)); + } + + ::milvus::grpc::VectorsData vectors_data; + handler->GetVectorsByID(&context, &vectors_identity, &vectors_data); + ASSERT_EQ(10, vectors_data.vectors_data_size()); +} + +TEST_F(RpcHandlerTest, GET_VECTOR_IDS_TEST) { + ::grpc::ServerContext context; + handler->SetContext(&context, dummy_context); + handler->RegisterRequestHandler(milvus::server::RequestHandler()); + + ::milvus::grpc::CollectionSchema collection_schema; + ::milvus::grpc::Status response; + std::string str_collection_name = "tbl_get_vector_ids"; + collection_schema.set_collection_name(str_collection_name); + collection_schema.set_dimension(COLLECTION_DIM); + collection_schema.set_index_file_size(INDEX_FILE_SIZE); + collection_schema.set_metric_type(1); + handler->CreateCollection(&context, &collection_schema, &response); + + ::milvus::grpc::InsertParam request; + request.set_collection_name(str_collection_name); + std::vector> record_array; + BuildVectors(0, VECTOR_COUNT, record_array); + ::milvus::grpc::VectorIds vector_ids; + for (auto& record : record_array) { + ::milvus::grpc::RowRecord* grpc_record = request.add_row_record_array(); + CopyRowRecord(grpc_record, record); + } + handler->Insert(&context, &request, &vector_ids); + ASSERT_EQ(vector_ids.vector_id_array_size(), VECTOR_COUNT); + + ::milvus::grpc::FlushParam flush_param; + flush_param.add_collection_name_array(str_collection_name); + handler->Flush(&context, &flush_param, &response); + + ::milvus::grpc::CollectionName collection_name; + collection_name.set_collection_name(str_collection_name); + ::milvus::grpc::CollectionInfo collection_info; + handler->ShowCollectionInfo(&context, &collection_name, &collection_info); + + std::string json_info = collection_info.json_info(); + auto info_json = nlohmann::json::parse(json_info); + std::string segment0 = info_json["partitions"][0]["segments"][0]["name"]; + + ::milvus::grpc::GetVectorIDsParam vector_ids_param; + vector_ids_param.set_collection_name(str_collection_name); + vector_ids_param.set_segment_name(segment0); + ::milvus::grpc::VectorIds vectors_ids; + handler->GetVectorIDs(&context, &vector_ids_param, &vectors_ids); + ASSERT_EQ(0, vectors_ids.status().error_code()); +} + TEST_F(RpcHandlerTest, PARTITION_TEST) { ::grpc::ServerContext context; handler->SetContext(&context, dummy_context); @@ -911,6 +1033,15 @@ TEST_F(RpcHandlerTest, PARTITION_TEST) { ASSERT_NE(response.error_code(), ::grpc::Status::OK.error_code()); fiu_disable("CreatePartitionRequest.OnExecute.throw_std_exception"); + // test has partition + ::milvus::grpc::PartitionParam has_partition_param; + has_partition_param.set_collection_name(str_collection_name); + has_partition_param.set_tag(partition_tag); + ::milvus::grpc::BoolReply reply; + handler->HasPartition(&context, &has_partition_param, &reply); + ASSERT_EQ(0, reply.status().error_code()); + ASSERT_EQ(true, reply.bool_reply()); + ::milvus::grpc::PartitionParam partition_parm; partition_parm.set_collection_name(str_collection_name); partition_parm.set_tag(partition_tag); @@ -924,6 +1055,114 @@ TEST_F(RpcHandlerTest, PARTITION_TEST) { ASSERT_EQ(response.error_code(), ::grpc::Status::OK.error_code()); } +TEST_F(RpcHandlerTest, RELOAD_SEGMENTS_TEST) { + ::grpc::ServerContext context; + handler->SetContext(&context, dummy_context); + handler->RegisterRequestHandler(milvus::server::RequestHandler()); + + ::milvus::grpc::CollectionSchema collection_schema; + ::milvus::grpc::Status response; + std::string str_collection_name = "tbl_reload_segments"; + collection_schema.set_collection_name(str_collection_name); + collection_schema.set_dimension(COLLECTION_DIM); + collection_schema.set_index_file_size(INDEX_FILE_SIZE); + collection_schema.set_metric_type(1); + handler->CreateCollection(&context, &collection_schema, &response); + + ::milvus::grpc::InsertParam request; + request.set_collection_name(str_collection_name); + std::vector> record_array; + BuildVectors(0, VECTOR_COUNT, record_array); + ::milvus::grpc::VectorIds vector_ids; + for (auto& record : record_array) { + ::milvus::grpc::RowRecord* grpc_record = request.add_row_record_array(); + CopyRowRecord(grpc_record, record); + } + handler->Insert(&context, &request, &vector_ids); + ASSERT_EQ(vector_ids.vector_id_array_size(), VECTOR_COUNT); + + ::milvus::grpc::FlushParam flush_param; + flush_param.add_collection_name_array(str_collection_name); + handler->Flush(&context, &flush_param, &response); + + ::milvus::grpc::DeleteByIDParam delete_param; + delete_param.set_collection_name(str_collection_name); + + for (size_t i = 0; i < 10; i++) { + delete_param.mutable_id_array()->Add(vector_ids.vector_id_array(i)); + } + + handler->DeleteByID(&context, &delete_param, &response); + ASSERT_EQ(0, response.error_code()) << response.reason(); + +// ::milvus::grpc::FlushParam flush_param; + flush_param.clear_collection_name_array(); + flush_param.add_collection_name_array(str_collection_name); + handler->Flush(&context, &flush_param, &response); + + fiu_enable("ReLoadSegmentsRequest.OnExecute.readonly", 1, NULL, 0); + ::milvus::grpc::ReLoadSegmentsParam reload_request; + reload_request.set_collection_name(str_collection_name); + ::milvus::grpc::Status reload_response; + + bool found = false; + for (size_t i = 0; i < 5000; i++) { + reload_request.clear_segment_id_array(); + reload_request.add_segment_id_array(std::to_string(i)); + handler->ReloadSegments(&context, &reload_request, &reload_response); + if (reload_response.error_code() == 0) { + found = true; + break; + } + } + + fiu_disable("ReLoadSegmentsRequest.OnExecute.readonly"); + + ASSERT_TRUE(found) << reload_response.reason(); +} + +TEST_F(RpcHandlerTest, COMPACT) { + ::grpc::ServerContext context; + handler->SetContext(&context, dummy_context); + handler->RegisterRequestHandler(milvus::server::RequestHandler()); + + ::milvus::grpc::CollectionSchema collection_schema; + ::milvus::grpc::Status response; + std::string str_collection_name = "tbl_rpc_compact"; + collection_schema.set_collection_name(str_collection_name); + collection_schema.set_dimension(COLLECTION_DIM); + collection_schema.set_index_file_size(INDEX_FILE_SIZE); + collection_schema.set_metric_type(1); + handler->CreateCollection(&context, &collection_schema, &response); + + ::milvus::grpc::InsertParam request; + request.set_collection_name(str_collection_name); + std::vector> bin_record_array; + BuildVectors(0, VECTOR_COUNT, bin_record_array); + ::milvus::grpc::VectorIds vector_ids; + for (auto& record : bin_record_array) { + ::milvus::grpc::RowRecord* grpc_record = request.add_row_record_array(); + CopyRowRecord(grpc_record, record); + } + handler->Insert(&context, &request, &vector_ids); + ASSERT_EQ(vector_ids.vector_id_array_size(), VECTOR_COUNT); + + ::milvus::grpc::DeleteByIDParam delete_param; + delete_param.set_collection_name(str_collection_name); + + for (size_t i = 0; i < 10; i++) { + delete_param.mutable_id_array()->Add(vector_ids.vector_id_array(i)); + } + + handler->DeleteByID(&context, &delete_param, &response); + ASSERT_EQ(0, response.error_code()) << response.reason(); + + ::milvus::grpc::CollectionName collection_name; + collection_name.set_collection_name(str_collection_name); + handler->Compact(&context, &collection_name, &response); + ASSERT_EQ(0, response.error_code()); +} + TEST_F(RpcHandlerTest, CMD_TEST) { ::grpc::ServerContext context; handler->SetContext(&context, dummy_context); diff --git a/core/unittest/server/test_util.cpp b/core/unittest/server/test_util.cpp index c6a149e641..43a1594dbc 100644 --- a/core/unittest/server/test_util.cpp +++ b/core/unittest/server/test_util.cpp @@ -246,14 +246,53 @@ TEST(UtilTest, BLOCKINGQUEUE_TEST) { } TEST(UtilTest, LOG_TEST) { + fiu_init(0); + + fiu_enable("LogUtil.InitLog.set_max_log_size_small_than_min", 1, NULL, 0); auto status = milvus::server::InitLog(true, true, true, true, true, true, "/tmp/test_util", 1024, 10); - ASSERT_TRUE(status.ok()); + ASSERT_FALSE(status.ok()); + fiu_disable("LogUtil.InitLog.set_max_log_size_small_than_min"); + + fiu_enable("LogUtil.InitLog.delete_exceeds_small_than_min", 1, NULL, 0); + status = milvus::server::InitLog(true, true, true, true, true, true, "/tmp/test_util", 1024, 10); + ASSERT_FALSE(status.ok()); + fiu_disable("LogUtil.InitLog.delete_exceeds_small_than_min"); + + fiu_enable("LogUtil.InitLog.info_enable_to_false", 1, NULL, 0); + fiu_enable("LogUtil.InitLog.debug_enable_to_false", 1, NULL, 0); + fiu_enable("LogUtil.InitLog.warning_enable_to_false", 1, NULL, 0); + fiu_enable("LogUtil.InitLog.trace_enable_to_false", 1, NULL, 0); + fiu_enable("LogUtil.InitLog.error_enable_to_false", 1, NULL, 0); + fiu_enable("LogUtil.InitLog.fatal_enable_to_false", 1, NULL, 0); + status = milvus::server::InitLog(true, true, true, true, true, true, "/tmp/test_util", 1024, 10); + ASSERT_TRUE(status.ok()) << status.message(); + fiu_disable("LogUtil.InitLog.fatal_enable_to_false"); + fiu_disable("LogUtil.InitLog.error_enable_to_false"); + fiu_disable("LogUtil.InitLog.trace_enable_to_false"); + fiu_disable("LogUtil.InitLog.warning_enable_to_false"); + fiu_disable("LogUtil.InitLog.debug_enable_to_false"); + fiu_disable("LogUtil.InitLog.info_enable_to_false"); + + status = milvus::server::InitLog(true, true, true, true, true, true, "/tmp/test_util", 1024, 10); + ASSERT_TRUE(status.ok()) << status.message(); EXPECT_FALSE(el::Loggers::hasFlag(el::LoggingFlag::NewLineForContainer)); EXPECT_FALSE(el::Loggers::hasFlag(el::LoggingFlag::LogDetailedCrashReason)); std::string fname = milvus::server::CommonUtil::GetFileName(LOG_FILE_PATH); ASSERT_EQ(fname, "log_config.conf"); + + ASSERT_NO_THROW(milvus::server::LogConfigInMem()); + ASSERT_NO_THROW(milvus::server::LogCpuInfo()); + + // test log config file + ASSERT_ANY_THROW(milvus::server::LogConfigInFile("log_config.conf")); + const char * config_str = "server_config:\n address: 0.0.0.0\n port: 19530"; + std::fstream fs("/tmp/config.yaml", std::ios_base::out); + fs << config_str; + fs.close(); + ASSERT_NO_THROW(milvus::server::LogConfigInFile("/tmp/config.yaml")); + boost::filesystem::remove("/tmp/config.yaml"); } TEST(UtilTest, TIMERECORDER_TEST) { diff --git a/core/unittest/server/test_web.cpp b/core/unittest/server/test_web.cpp index 7712318ffc..42a0139774 100644 --- a/core/unittest/server/test_web.cpp +++ b/core/unittest/server/test_web.cpp @@ -15,6 +15,8 @@ #include #include +#include +#include #include #include #include @@ -35,7 +37,7 @@ #include "utils/CommonUtil.h" #include "utils/StringHelpFunctions.h" -static const char* COLLECTION_NAME = "test_web"; +static const char* COLLECTION_NAME = "test_milvus_web_collection"; using OStatus = oatpp::web::protocol::http::Status; using OString = milvus::server::web::OString; @@ -1435,6 +1437,17 @@ TEST_F(WebControllerTest, CONFIG) { auto get_result_json = nlohmann::json::parse(response->readBodyToString()->c_str()); ASSERT_TRUE(get_result_json.contains("restart_required")); ASSERT_EQ(true, get_result_json["restart_required"].get()); + + fiu_init(0); + fiu_enable("WebRequestHandler.SystemOp.raise_parse_error", 1, NULL, 0); + response = client_ptr->op("config", body_str, conncetion_ptr); + ASSERT_NE(OStatus::CODE_200.code, response->getStatusCode()); + fiu_disable("WebRequestHandler.SystemOp.raise_parse_error"); + + fiu_enable("WebRequestHandler.SystemOp.raise_type_error", 1, NULL, 0); + response = client_ptr->op("config", body_str, conncetion_ptr); + ASSERT_NE(OStatus::CODE_200.code, response->getStatusCode()); + fiu_disable("WebRequestHandler.SystemOp.raise_type_error"); } TEST_F(WebControllerTest, ADVANCED_CONFIG) { diff --git a/core/unittest/storage/test_disk.cpp b/core/unittest/storage/test_disk.cpp index ef43594687..0f673c8317 100644 --- a/core/unittest/storage/test_disk.cpp +++ b/core/unittest/storage/test_disk.cpp @@ -9,11 +9,14 @@ // is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express // or implied. See the License for the specific language governing permissions and limitations under the License. +#include +#include #include #include "easyloggingpp/easylogging++.h" #include "storage/disk/DiskIOReader.h" #include "storage/disk/DiskIOWriter.h" +#include "storage/disk/DiskOperation.h" #include "storage/utils.h" INITIALIZE_EASYLOGGINGPP @@ -60,3 +63,21 @@ TEST_F(StorageTest, DISK_RW_TEST) { reader.close(); } } + +TEST_F(StorageTest, DISK_OPERATION_TEST) { + auto disk_operation = milvus::storage::DiskOperation("/tmp/milvus_test/milvus_disk_operation_test"); + + fiu_init(0); + fiu_enable("DiskOperation.CreateDirectory.is_directory", 1, NULL, 0); + fiu_enable("DiskOperation.CreateDirectory.create_directory", 1, NULL, 0); + ASSERT_ANY_THROW(disk_operation.CreateDirectory()); + fiu_disable("DiskOperation.CreateDirectory.create_directory"); + fiu_disable("DiskOperation.CreateDirectory.is_directory"); + + std::vector file_paths; + ASSERT_NO_THROW(disk_operation.ListDirectory(file_paths)); + + for (auto & path : file_paths) { + ASSERT_TRUE(disk_operation.DeleteFile(path)); + } +}