diff --git a/cpp/.gitignore b/cpp/.gitignore index b9ba83ac8a..03149cde32 100644 --- a/cpp/.gitignore +++ b/cpp/.gitignore @@ -1,4 +1,10 @@ milvus/ conf/server_config.yaml +conf/log_config.conf version.h megasearch/ +lcov_out/ +base.info +output.info +output_new.info +server.info \ No newline at end of file diff --git a/cpp/CHANGELOG.md b/cpp/CHANGELOG.md index c9b6db07ac..23409def53 100644 --- a/cpp/CHANGELOG.md +++ b/cpp/CHANGELOG.md @@ -36,6 +36,8 @@ Please mark all change in change log and use the ticket from JIRA. - MS-115 - Change is_startup of metric_config switch from true to on - MS-122 - Archive criteria config - MS-124 - HasTable interface +- MS-126 - Add more error code +- MS-128 - Change default db path ## New Feature @@ -57,7 +59,7 @@ Please mark all change in change log and use the ticket from JIRA. - MS-85 - add NetIO metric - MS-96 - add new query interface for specified files - MS-97 - Add S3 SDK for MinIO Storage - +- MS-130 - Add prometheus_test ## Task - MS-74 - Change README.md in cpp - MS-88 - Add support for arm architecture diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 6cddd0843f..c3a5957045 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -135,12 +135,19 @@ if (BUILD_COVERAGE STREQUAL "ON") endif() -if (BUILD_UNIT_TEST) +if (BUILD_UNIT_TEST STREQUAL "ON") add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/unittest) -endif(BUILD_UNIT_TEST) +endif() add_custom_target(Clean-All COMMAND ${CMAKE_BUILD_TOOL} clean) + +if("${MILVUS_DB_PATH}" STREQUAL "") + set(MILVUS_DB_PATH "/tmp/milvus") +endif() +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/conf/server_config.template ${CMAKE_CURRENT_SOURCE_DIR}/conf/server_config.yaml) +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/conf/log_config.template ${CMAKE_CURRENT_SOURCE_DIR}/conf/log_config.conf) + #install install(FILES scripts/start_server.sh @@ -153,4 +160,6 @@ install(FILES DESTINATION conf) + + config_summary() diff --git a/cpp/README.md b/cpp/README.md index 0c7706df23..69f7f8c0e0 100644 --- a/cpp/README.md +++ b/cpp/README.md @@ -36,6 +36,11 @@ If you encounter the following error when building: or ./build.sh --unittest +#### To run code coverage + + apt-get install lcov + ./build.sh -u -c + ### Launch server Set config in cpp/conf/server_config.yaml diff --git a/cpp/build.sh b/cpp/build.sh index a7ece5fee6..80be1d7ddb 100755 --- a/cpp/build.sh +++ b/cpp/build.sh @@ -1,13 +1,14 @@ #!/bin/bash BUILD_TYPE="Debug" -BUILD_UNITTEST="off" +BUILD_UNITTEST="OFF" LICENSE_CHECK="OFF" INSTALL_PREFIX=$(pwd)/milvus MAKE_CLEAN="OFF" BUILD_COVERAGE="OFF" +DB_PATH="/opt/milvus" -while getopts "p:t:uhlrc" arg +while getopts "p:d:t:uhlrc" arg do case $arg in t) @@ -15,11 +16,14 @@ do ;; u) echo "Build and run unittest cases" ; - BUILD_UNITTEST="on"; + BUILD_UNITTEST="ON"; ;; p) INSTALL_PREFIX=$OPTARG ;; + d) + DB_PATH=$OPTARG + ;; l) LICENSE_CHECK="ON" ;; @@ -36,12 +40,13 @@ do echo " parameter: --t: build type --u: building unit test options --p: install prefix --l: build license version --r: remove previous build directory --c: code coverage +-t: build type(default: Debug) +-u: building unit test options(default: OFF) +-p: install prefix(default: $(pwd)/milvus) +-d: db path(default: /opt/milvus) +-l: build license version(default: OFF) +-r: remove previous build directory(default: OFF) +-c: code coverage(default: OFF) usage: ./build.sh -t \${BUILD_TYPE} [-u] [-h] [-g] [-r] [-c] @@ -71,6 +76,7 @@ if [[ ${MAKE_CLEAN} == "ON" ]]; then -DCMAKE_CUDA_COMPILER=${CUDA_COMPILER} \ -DCMAKE_LICENSE_CHECK=${LICENSE_CHECK} \ -DBUILD_COVERAGE=${BUILD_COVERAGE} \ + -DMILVUS_DB_PATH=${DB_PATH} \ $@ ../" echo ${CMAKE_CMD} diff --git a/cpp/conf/log_config.template b/cpp/conf/log_config.template new file mode 100644 index 0000000000..f4f3d3684c --- /dev/null +++ b/cpp/conf/log_config.template @@ -0,0 +1,27 @@ +* GLOBAL: + FORMAT = "%datetime | %level | %logger | %msg" + FILENAME = "@MILVUS_DB_PATH@/logs/milvus-%datetime{%H:%m}-global.log" + ENABLED = true + TO_FILE = true + TO_STANDARD_OUTPUT = false + SUBSECOND_PRECISION = 3 + PERFORMANCE_TRACKING = false + MAX_LOG_FILE_SIZE = 2097152 ## Throw log files away after 2MB +* DEBUG: + FILENAME = "@MILVUS_DB_PATH@/logs/milvus-%datetime{%H:%m}-debug.log" + ENABLED = true +* WARNING: + FILENAME = "@MILVUS_DB_PATH@/logs/milvus-%datetime{%H:%m}-warning.log" +* TRACE: + FILENAME = "@MILVUS_DB_PATH@/logs/milvus-%datetime{%H:%m}-trace.log" +* VERBOSE: + FORMAT = "%datetime{%d/%M/%y} | %level-%vlevel | %msg" + TO_FILE = false + TO_STANDARD_OUTPUT = false +## Error logs +* ERROR: + ENABLED = true + FILENAME = "@MILVUS_DB_PATH@/logs/milvus-%datetime{%H:%m}-error.log" +* FATAL: + ENABLED = true + FILENAME = "@MILVUS_DB_PATH@/logs/milvus-%datetime{%H:%m}-fatal.log" \ No newline at end of file diff --git a/cpp/conf/server_config.template b/cpp/conf/server_config.template new file mode 100644 index 0000000000..1a1c8303f2 --- /dev/null +++ b/cpp/conf/server_config.template @@ -0,0 +1,28 @@ +server_config: + address: 0.0.0.0 + port: 19530 # the port milvus listen to, default: 19530, range: 1025 ~ 65534 + gpu_index: 0 # the gpu milvus use, default: 0, range: 0 ~ gpu number - 1 + mode: single # milvus deployment type: single, cluster + +db_config: + db_path: @MILVUS_DB_PATH@ # milvus data storage path + db_backend_url: http://127.0.0.1 # meta database uri + index_building_threshold: 1024 # index building trigger threshold, default: 1024, unit: MB + archive_disk_threshold: 512 # triger archive action if storage size exceed this value, unit: GB + archive_days_threshold: 30 # files older than x days will be archived, unit: day + +metric_config: + is_startup: off # if monitoring start: on, off + collector: prometheus # metrics collector: prometheus + prometheus_config: # following are prometheus configure + collect_type: pull # prometheus collect data method + port: 8080 # the port prometheus use to fetch metrics + push_gateway_ip_address: 127.0.0.1 # push method configure: push gateway ip address + push_gateway_port: 9091 # push method configure: push gateway port + + +license_config: # license configure + license_path: "@MILVUS_DB_PATH@/system.license" # license file path + +cache_config: # cache configure + cpu_cache_capacity: 16 # how many memory are used as cache, unit: GB, range: 0 ~ less than total memory \ No newline at end of file diff --git a/cpp/src/db/DBMetaImpl.cpp b/cpp/src/db/DBMetaImpl.cpp index 1a8419eac2..d834d6cff2 100644 --- a/cpp/src/db/DBMetaImpl.cpp +++ b/cpp/src/db/DBMetaImpl.cpp @@ -192,9 +192,11 @@ Status DBMetaImpl::CreateTable(TableSchema &table_schema) { auto table = ConnectorPtr->select(columns(&TableSchema::state_), where(c(&TableSchema::table_id_) == table_schema.table_id_)); if (table.size() == 1) { - std::string msg = (TableSchema::TO_DELETE == std::get<0>(table[0])) ? - "Table already exists and it is in delete state, please wait a second" : "Table already exists"; - return Status::Error(msg); + if(TableSchema::TO_DELETE == std::get<0>(table[0])) { + return Status::Error("Table already exists and it is in delete state, please wait a second"); + } else { + return Status::OK();//table already exists, no error + } } } @@ -328,7 +330,7 @@ Status DBMetaImpl::HasTable(const std::string &table_id, bool &has_or_not) { } } catch (std::exception &e) { - HandleException("Encounter exception when lookup table", e); + return HandleException("Encounter exception when lookup table", e); } return Status::OK(); @@ -358,7 +360,7 @@ Status DBMetaImpl::AllTables(std::vector& table_schema_array) { } } catch (std::exception &e) { - HandleException("Encounter exception when lookup all tables", e); + return HandleException("Encounter exception when lookup all tables", e); } return Status::OK(); diff --git a/cpp/src/license/GPUInfoFile.h b/cpp/src/license/GPUInfoFile.h index d22765bd71..eaad3efefe 100644 --- a/cpp/src/license/GPUInfoFile.h +++ b/cpp/src/license/GPUInfoFile.h @@ -1,69 +1,69 @@ -/******************************************************************************* - * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved - * Unauthorized copying of this file, via any medium is strictly prohibited. - * Proprietary and confidential. - ******************************************************************************/ -#pragma once - -#include -#include -#include - - -class GPUInfoFile { - public: - GPUInfoFile() = default; - - GPUInfoFile(const int &device_count, const std::map &uuid_encryption_map) - : device_count_(device_count), uuid_encryption_map_(uuid_encryption_map) {} - - int get_device_count() { - return device_count_; - } - std::map &get_uuid_encryption_map() { - return uuid_encryption_map_; - } - - - public: - friend class boost::serialization::access; - - template - void serialize(Archive &ar, const unsigned int version) { - ar & device_count_; - ar & uuid_encryption_map_; - } - - public: - int device_count_ = 0; - std::map uuid_encryption_map_; -}; - -class SerializedGPUInfoFile { - public: - ~SerializedGPUInfoFile() { - if (gpu_info_file_ != nullptr) { - delete (gpu_info_file_); - gpu_info_file_ = nullptr; - } - } - - void - set_gpu_info_file(GPUInfoFile *gpu_info_file) { - gpu_info_file_ = gpu_info_file; - } - - GPUInfoFile *get_gpu_info_file() { - return gpu_info_file_; - } - private: - friend class boost::serialization::access; - - template - void serialize(Archive &ar, const unsigned int version) { - ar & gpu_info_file_; - } - - private: - GPUInfoFile *gpu_info_file_ = nullptr; -}; +///******************************************************************************* +// * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved +// * Unauthorized copying of this file, via any medium is strictly prohibited. +// * Proprietary and confidential. +// ******************************************************************************/ +//#pragma once +// +//#include +//#include +//#include +// +// +//class GPUInfoFile { +// public: +// GPUInfoFile() = default; +// +// GPUInfoFile(const int &device_count, const std::map &uuid_encryption_map) +// : device_count_(device_count), uuid_encryption_map_(uuid_encryption_map) {} +// +// int get_device_count() { +// return device_count_; +// } +// std::map &get_uuid_encryption_map() { +// return uuid_encryption_map_; +// } +// +// +// public: +// friend class boost::serialization::access; +// +// template +// void serialize(Archive &ar, const unsigned int version) { +// ar & device_count_; +// ar & uuid_encryption_map_; +// } +// +// public: +// int device_count_ = 0; +// std::map uuid_encryption_map_; +//}; +// +//class SerializedGPUInfoFile { +// public: +// ~SerializedGPUInfoFile() { +// if (gpu_info_file_ != nullptr) { +// delete (gpu_info_file_); +// gpu_info_file_ = nullptr; +// } +// } +// +// void +// set_gpu_info_file(GPUInfoFile *gpu_info_file) { +// gpu_info_file_ = gpu_info_file; +// } +// +// GPUInfoFile *get_gpu_info_file() { +// return gpu_info_file_; +// } +// private: +// friend class boost::serialization::access; +// +// template +// void serialize(Archive &ar, const unsigned int version) { +// ar & gpu_info_file_; +// } +// +// private: +// GPUInfoFile *gpu_info_file_ = nullptr; +//}; diff --git a/cpp/src/license/GetSysInfo.cpp b/cpp/src/license/GetSysInfo.cpp index 4746fe5eac..7a30083469 100644 --- a/cpp/src/license/GetSysInfo.cpp +++ b/cpp/src/license/GetSysInfo.cpp @@ -1,83 +1,83 @@ - -#include "utils/Log.h" -#include "LicenseLibrary.h" -#include "utils/Error.h" - -#include -#include -#include -// Not provide path: current work path will be used and system.info. -using namespace zilliz::milvus; - -void -print_usage(const std::string &app_name) { - printf("\n Usage: %s [OPTIONS]\n\n", app_name.c_str()); - printf(" Options:\n"); - printf(" -h --help Print this help\n"); - printf(" -s --sysinfo filename Generate system info file as given name\n"); - printf("\n"); -} - -int main(int argc, char *argv[]) { - std::string app_name = argv[0]; - if (argc != 1 && argc != 3) { - print_usage(app_name); - return EXIT_FAILURE; - } - - static struct option long_options[] = {{"system_info", required_argument, 0, 's'}, - {"help", no_argument, 0, 'h'}, - {NULL, 0, 0, 0}}; - int value = 0; - int option_index = 0; - std::string system_info_filename = "./system.info"; - while ((value = getopt_long(argc, argv, "s:h", long_options, &option_index)) != -1) { - switch (value) { - case 's': { - char *system_info_filename_ptr = strdup(optarg); - system_info_filename = system_info_filename_ptr; - free(system_info_filename_ptr); -// printf("Generate system info file: %s\n", system_info_filename.c_str()); - break; - } - case 'h':print_usage(app_name); - return EXIT_SUCCESS; - case '?':print_usage(app_name); - return EXIT_FAILURE; - default:print_usage(app_name); - break; - } - } - - int device_count = 0; - server::ServerError err = server::LicenseLibrary::GetDeviceCount(device_count); - if (err != server::SERVER_SUCCESS) return -1; - - // 1. Get All GPU UUID - std::vector uuid_array; - err = server::LicenseLibrary::GetUUID(device_count, uuid_array); - if (err != server::SERVER_SUCCESS) return -1; - - // 2. Get UUID SHA256 - std::vector uuid_sha256_array; - err = server::LicenseLibrary::GetUUIDSHA256(device_count, uuid_array, uuid_sha256_array); - if (err != server::SERVER_SUCCESS) return -1; - - // 3. Generate GPU ID map with GPU UUID - std::map uuid_encrption_map; - for (int i = 0; i < device_count; ++i) { - uuid_encrption_map[i] = uuid_sha256_array[i]; - } - - - // 4. Generate GPU_info File - err = server::LicenseLibrary::GPUinfoFileSerialization(system_info_filename, - device_count, - uuid_encrption_map); - if (err != server::SERVER_SUCCESS) return -1; - - printf("Generate GPU_info File Success\n"); - - - return 0; -} \ No newline at end of file +// +//#include "utils/Log.h" +//#include "LicenseLibrary.h" +//#include "utils/Error.h" +// +//#include +//#include +//#include +//// Not provide path: current work path will be used and system.info. +//using namespace zilliz::milvus; +// +//void +//print_usage(const std::string &app_name) { +// printf("\n Usage: %s [OPTIONS]\n\n", app_name.c_str()); +// printf(" Options:\n"); +// printf(" -h --help Print this help\n"); +// printf(" -s --sysinfo filename Generate system info file as given name\n"); +// printf("\n"); +//} +// +//int main(int argc, char *argv[]) { +// std::string app_name = argv[0]; +// if (argc != 1 && argc != 3) { +// print_usage(app_name); +// return EXIT_FAILURE; +// } +// +// static struct option long_options[] = {{"system_info", required_argument, 0, 's'}, +// {"help", no_argument, 0, 'h'}, +// {NULL, 0, 0, 0}}; +// int value = 0; +// int option_index = 0; +// std::string system_info_filename = "./system.info"; +// while ((value = getopt_long(argc, argv, "s:h", long_options, &option_index)) != -1) { +// switch (value) { +// case 's': { +// char *system_info_filename_ptr = strdup(optarg); +// system_info_filename = system_info_filename_ptr; +// free(system_info_filename_ptr); +//// printf("Generate system info file: %s\n", system_info_filename.c_str()); +// break; +// } +// case 'h':print_usage(app_name); +// return EXIT_SUCCESS; +// case '?':print_usage(app_name); +// return EXIT_FAILURE; +// default:print_usage(app_name); +// break; +// } +// } +// +// int device_count = 0; +// server::ServerError err = server::LicenseLibrary::GetDeviceCount(device_count); +// if (err != server::SERVER_SUCCESS) return -1; +// +// // 1. Get All GPU UUID +// std::vector uuid_array; +// err = server::LicenseLibrary::GetUUID(device_count, uuid_array); +// if (err != server::SERVER_SUCCESS) return -1; +// +// // 2. Get UUID SHA256 +// std::vector uuid_sha256_array; +// err = server::LicenseLibrary::GetUUIDSHA256(device_count, uuid_array, uuid_sha256_array); +// if (err != server::SERVER_SUCCESS) return -1; +// +// // 3. Generate GPU ID map with GPU UUID +// std::map uuid_encrption_map; +// for (int i = 0; i < device_count; ++i) { +// uuid_encrption_map[i] = uuid_sha256_array[i]; +// } +// +// +// // 4. Generate GPU_info File +// err = server::LicenseLibrary::GPUinfoFileSerialization(system_info_filename, +// device_count, +// uuid_encrption_map); +// if (err != server::SERVER_SUCCESS) return -1; +// +// printf("Generate GPU_info File Success\n"); +// +// +// return 0; +//} \ No newline at end of file diff --git a/cpp/src/license/LicenseCheck.cpp b/cpp/src/license/LicenseCheck.cpp index 8821795540..8ea396ef41 100644 --- a/cpp/src/license/LicenseCheck.cpp +++ b/cpp/src/license/LicenseCheck.cpp @@ -1,131 +1,131 @@ -#include "LicenseCheck.h" -#include -#include - -#include -#include -//#include -//#include -#include -#include -#include -#include -#include - - -namespace zilliz { -namespace milvus { -namespace server { - -LicenseCheck::LicenseCheck() { - -} - -LicenseCheck::~LicenseCheck() { - StopCountingDown(); -} - -ServerError -LicenseCheck::LegalityCheck(const std::string &license_file_path) { - - int device_count; - LicenseLibrary::GetDeviceCount(device_count); - std::vector uuid_array; - LicenseLibrary::GetUUID(device_count, uuid_array); - - std::vector sha_array; - LicenseLibrary::GetUUIDSHA256(device_count, uuid_array, sha_array); - - int output_device_count; - std::map uuid_encryption_map; - time_t starting_time; - time_t end_time; - ServerError err = LicenseLibrary::LicenseFileDeserialization(license_file_path, - output_device_count, - uuid_encryption_map, - starting_time, - end_time); - if(err !=SERVER_SUCCESS) - { - std::cout << "License check error: 01" << std::endl; - return SERVER_UNEXPECTED_ERROR; - } - time_t system_time; - LicenseLibrary::GetSystemTime(system_time); - - if (device_count != output_device_count) { - std::cout << "License check error: 02" << std::endl; - return SERVER_UNEXPECTED_ERROR; - } - for (int i = 0; i < device_count; ++i) { - if (sha_array[i] != uuid_encryption_map[i]) { - std::cout << "License check error: 03" << std::endl; - return SERVER_UNEXPECTED_ERROR; - } - } - if (system_time < starting_time || system_time > end_time) { - std::cout << "License check error: 04" << std::endl; - return SERVER_UNEXPECTED_ERROR; - } - std::cout << "Legality Check Success" << std::endl; - return SERVER_SUCCESS; -} - -// Part 2: Timing check license - -ServerError -LicenseCheck::AlterFile(const std::string &license_file_path, - const boost::system::error_code &ec, - boost::asio::deadline_timer *pt) { - - ServerError err = LicenseCheck::LegalityCheck(license_file_path); - if(err!=SERVER_SUCCESS) { - std::cout << "license file check error" << std::endl; - exit(1); - } - - std::cout << "---runing---" << std::endl; - pt->expires_at(pt->expires_at() + boost::posix_time::hours(1)); - pt->async_wait(boost::bind(LicenseCheck::AlterFile, license_file_path, boost::asio::placeholders::error, pt)); - - return SERVER_SUCCESS; - -} - -ServerError -LicenseCheck::StartCountingDown(const std::string &license_file_path) { - - if (!LicenseLibrary::IsFileExistent(license_file_path)) { - std::cout << "license file not exist" << std::endl; - exit(1); - } - - //create a thread to run AlterFile - if(counting_thread_ == nullptr) { - counting_thread_ = std::make_shared([&]() { - boost::asio::deadline_timer t(io_service_, boost::posix_time::hours(1)); - t.async_wait(boost::bind(LicenseCheck::AlterFile, license_file_path, boost::asio::placeholders::error, &t)); - io_service_.run();//this thread will block here - }); - } - - return SERVER_SUCCESS; -} - -ServerError -LicenseCheck::StopCountingDown() { - if(!io_service_.stopped()) { - io_service_.stop(); - } - - if(counting_thread_ != nullptr) { - counting_thread_->join(); - counting_thread_ = nullptr; - } - - return SERVER_SUCCESS; -} - -} -} -} \ No newline at end of file +//#include "LicenseCheck.h" +//#include +//#include +// +//#include +//#include +////#include +////#include +//#include +//#include +//#include +//#include +//#include +// +// +//namespace zilliz { +//namespace milvus { +//namespace server { +// +//LicenseCheck::LicenseCheck() { +// +//} +// +//LicenseCheck::~LicenseCheck() { +// StopCountingDown(); +//} +// +//ServerError +//LicenseCheck::LegalityCheck(const std::string &license_file_path) { +// +// int device_count; +// LicenseLibrary::GetDeviceCount(device_count); +// std::vector uuid_array; +// LicenseLibrary::GetUUID(device_count, uuid_array); +// +// std::vector sha_array; +// LicenseLibrary::GetUUIDSHA256(device_count, uuid_array, sha_array); +// +// int output_device_count; +// std::map uuid_encryption_map; +// time_t starting_time; +// time_t end_time; +// ServerError err = LicenseLibrary::LicenseFileDeserialization(license_file_path, +// output_device_count, +// uuid_encryption_map, +// starting_time, +// end_time); +// if(err !=SERVER_SUCCESS) +// { +// std::cout << "License check error: 01" << std::endl; +// return SERVER_UNEXPECTED_ERROR; +// } +// time_t system_time; +// LicenseLibrary::GetSystemTime(system_time); +// +// if (device_count != output_device_count) { +// std::cout << "License check error: 02" << std::endl; +// return SERVER_UNEXPECTED_ERROR; +// } +// for (int i = 0; i < device_count; ++i) { +// if (sha_array[i] != uuid_encryption_map[i]) { +// std::cout << "License check error: 03" << std::endl; +// return SERVER_UNEXPECTED_ERROR; +// } +// } +// if (system_time < starting_time || system_time > end_time) { +// std::cout << "License check error: 04" << std::endl; +// return SERVER_UNEXPECTED_ERROR; +// } +// std::cout << "Legality Check Success" << std::endl; +// return SERVER_SUCCESS; +//} +// +//// Part 2: Timing check license +// +//ServerError +//LicenseCheck::AlterFile(const std::string &license_file_path, +// const boost::system::error_code &ec, +// boost::asio::deadline_timer *pt) { +// +// ServerError err = LicenseCheck::LegalityCheck(license_file_path); +// if(err!=SERVER_SUCCESS) { +// std::cout << "license file check error" << std::endl; +// exit(1); +// } +// +// std::cout << "---runing---" << std::endl; +// pt->expires_at(pt->expires_at() + boost::posix_time::hours(1)); +// pt->async_wait(boost::bind(LicenseCheck::AlterFile, license_file_path, boost::asio::placeholders::error, pt)); +// +// return SERVER_SUCCESS; +// +//} +// +//ServerError +//LicenseCheck::StartCountingDown(const std::string &license_file_path) { +// +// if (!LicenseLibrary::IsFileExistent(license_file_path)) { +// std::cout << "license file not exist" << std::endl; +// exit(1); +// } +// +// //create a thread to run AlterFile +// if(counting_thread_ == nullptr) { +// counting_thread_ = std::make_shared([&]() { +// boost::asio::deadline_timer t(io_service_, boost::posix_time::hours(1)); +// t.async_wait(boost::bind(LicenseCheck::AlterFile, license_file_path, boost::asio::placeholders::error, &t)); +// io_service_.run();//this thread will block here +// }); +// } +// +// return SERVER_SUCCESS; +//} +// +//ServerError +//LicenseCheck::StopCountingDown() { +// if(!io_service_.stopped()) { +// io_service_.stop(); +// } +// +// if(counting_thread_ != nullptr) { +// counting_thread_->join(); +// counting_thread_ = nullptr; +// } +// +// return SERVER_SUCCESS; +//} +// +//} +//} +//} \ No newline at end of file diff --git a/cpp/src/license/LicenseCheck.h b/cpp/src/license/LicenseCheck.h index aa66d132d0..7b20d014b5 100644 --- a/cpp/src/license/LicenseCheck.h +++ b/cpp/src/license/LicenseCheck.h @@ -1,52 +1,52 @@ -#pragma once - -#include "utils/Error.h" -#include "LicenseLibrary.h" - -#include - -#include -#include - -namespace zilliz { -namespace milvus { -namespace server { - -class LicenseCheck { -private: - LicenseCheck(); - ~LicenseCheck(); - -public: - static LicenseCheck & - GetInstance() { - static LicenseCheck instance; - return instance; - }; - - static ServerError - LegalityCheck(const std::string &license_file_path); - - ServerError - StartCountingDown(const std::string &license_file_path); - - ServerError - StopCountingDown(); - -private: - static ServerError - AlterFile(const std::string &license_file_path, - const boost::system::error_code &ec, - boost::asio::deadline_timer *pt); - -private: - boost::asio::io_service io_service_; - std::shared_ptr counting_thread_; - -}; - -} -} -} - - +//#pragma once +// +//#include "utils/Error.h" +//#include "LicenseLibrary.h" +// +//#include +// +//#include +//#include +// +//namespace zilliz { +//namespace milvus { +//namespace server { +// +//class LicenseCheck { +//private: +// LicenseCheck(); +// ~LicenseCheck(); +// +//public: +// static LicenseCheck & +// GetInstance() { +// static LicenseCheck instance; +// return instance; +// }; +// +// static ServerError +// LegalityCheck(const std::string &license_file_path); +// +// ServerError +// StartCountingDown(const std::string &license_file_path); +// +// ServerError +// StopCountingDown(); +// +//private: +// static ServerError +// AlterFile(const std::string &license_file_path, +// const boost::system::error_code &ec, +// boost::asio::deadline_timer *pt); +// +//private: +// boost::asio::io_service io_service_; +// std::shared_ptr counting_thread_; +// +//}; +// +//} +//} +//} +// +// diff --git a/cpp/src/license/LicenseFile.h b/cpp/src/license/LicenseFile.h index bf076b0d68..deefe19d14 100644 --- a/cpp/src/license/LicenseFile.h +++ b/cpp/src/license/LicenseFile.h @@ -1,86 +1,86 @@ -/******************************************************************************* - * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved - * Unauthorized copying of this file, via any medium is strictly prohibited. - * Proprietary and confidential. - ******************************************************************************/ -#pragma once - - -#include -#include -#include - - -class LicenseFile { - public: - LicenseFile() = default; - - LicenseFile(const int &device_count, - const std::map &uuid_encryption_map, - const time_t &starting_time, - const time_t &end_time) - : device_count_(device_count), - uuid_encryption_map_(uuid_encryption_map), - starting_time_(starting_time), - end_time_(end_time) {} - - int get_device_count() { - return device_count_; - } - std::map &get_uuid_encryption_map() { - return uuid_encryption_map_; - } - time_t get_starting_time() { - return starting_time_; - } - time_t get_end_time() { - return end_time_; - } - - public: - friend class boost::serialization::access; - - template - void serialize(Archive &ar, const unsigned int version) { - ar & device_count_; - ar & uuid_encryption_map_; - ar & starting_time_; - ar & end_time_; - } - - public: - int device_count_ = 0; - std::map uuid_encryption_map_; - time_t starting_time_ = 0; - time_t end_time_ = 0; -}; - -class SerializedLicenseFile { - public: - ~SerializedLicenseFile() { - if (license_file_ != nullptr) { - delete (license_file_); - license_file_ = nullptr; - } - } - - void - set_license_file(LicenseFile *license_file) { - license_file_ = license_file; - } - - LicenseFile *get_license_file() { - return license_file_; - } - private: - friend class boost::serialization::access; - - template - void serialize(Archive &ar, const unsigned int version) { - ar & license_file_; - } - - private: - LicenseFile *license_file_ = nullptr; -}; - +///******************************************************************************* +// * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved +// * Unauthorized copying of this file, via any medium is strictly prohibited. +// * Proprietary and confidential. +// ******************************************************************************/ +//#pragma once +// +// +//#include +//#include +//#include +// +// +//class LicenseFile { +// public: +// LicenseFile() = default; +// +// LicenseFile(const int &device_count, +// const std::map &uuid_encryption_map, +// const time_t &starting_time, +// const time_t &end_time) +// : device_count_(device_count), +// uuid_encryption_map_(uuid_encryption_map), +// starting_time_(starting_time), +// end_time_(end_time) {} +// +// int get_device_count() { +// return device_count_; +// } +// std::map &get_uuid_encryption_map() { +// return uuid_encryption_map_; +// } +// time_t get_starting_time() { +// return starting_time_; +// } +// time_t get_end_time() { +// return end_time_; +// } +// +// public: +// friend class boost::serialization::access; +// +// template +// void serialize(Archive &ar, const unsigned int version) { +// ar & device_count_; +// ar & uuid_encryption_map_; +// ar & starting_time_; +// ar & end_time_; +// } +// +// public: +// int device_count_ = 0; +// std::map uuid_encryption_map_; +// time_t starting_time_ = 0; +// time_t end_time_ = 0; +//}; +// +//class SerializedLicenseFile { +// public: +// ~SerializedLicenseFile() { +// if (license_file_ != nullptr) { +// delete (license_file_); +// license_file_ = nullptr; +// } +// } +// +// void +// set_license_file(LicenseFile *license_file) { +// license_file_ = license_file; +// } +// +// LicenseFile *get_license_file() { +// return license_file_; +// } +// private: +// friend class boost::serialization::access; +// +// template +// void serialize(Archive &ar, const unsigned int version) { +// ar & license_file_; +// } +// +// private: +// LicenseFile *license_file_ = nullptr; +//}; +// diff --git a/cpp/src/license/LicenseGenerator.cpp b/cpp/src/license/LicenseGenerator.cpp index 2a83b6b2d1..a5064eb4c3 100644 --- a/cpp/src/license/LicenseGenerator.cpp +++ b/cpp/src/license/LicenseGenerator.cpp @@ -1,121 +1,121 @@ - -#include -#include -#include - -#include "utils/Log.h" -#include "license/LicenseLibrary.h" -#include "utils/Error.h" - - -using namespace zilliz::milvus; -// Not provide path: current work path will be used and system.info. - -void -print_usage(const std::string &app_name) { - printf("\n Usage: %s [OPTIONS]\n\n", app_name.c_str()); - printf(" Options:\n"); - printf(" -h --help Print this help\n"); - printf(" -s --sysinfo filename sysinfo file location\n"); - printf(" -l --license filename Generate license file as given name\n"); - printf(" -b --starting time Set start time (format: YYYY-MM-DD)\n"); - printf(" -e --end time Set end time (format: YYYY-MM-DD)\n"); - printf("\n"); -} - -int main(int argc, char *argv[]) { - std::string app_name = argv[0]; -// if (argc != 1 && argc != 3) { -// print_usage(app_name); -// return EXIT_FAILURE; +// +//#include +//#include +//#include +// +//#include "utils/Log.h" +//#include "license/LicenseLibrary.h" +//#include "utils/Error.h" +// +// +//using namespace zilliz::milvus; +//// Not provide path: current work path will be used and system.info. +// +//void +//print_usage(const std::string &app_name) { +// printf("\n Usage: %s [OPTIONS]\n\n", app_name.c_str()); +// printf(" Options:\n"); +// printf(" -h --help Print this help\n"); +// printf(" -s --sysinfo filename sysinfo file location\n"); +// printf(" -l --license filename Generate license file as given name\n"); +// printf(" -b --starting time Set start time (format: YYYY-MM-DD)\n"); +// printf(" -e --end time Set end time (format: YYYY-MM-DD)\n"); +// printf("\n"); +//} +// +//int main(int argc, char *argv[]) { +// std::string app_name = argv[0]; +//// if (argc != 1 && argc != 3) { +//// print_usage(app_name); +//// return EXIT_FAILURE; +//// } +// static struct option long_options[] = {{"system_info", required_argument, 0, 's'}, +// {"license", optional_argument, 0, 'l'}, +// {"help", no_argument, 0, 'h'}, +// {"starting_time", required_argument, 0, 'b'}, +// {"end_time", required_argument, 0, 'e'}, +// {NULL, 0, 0, 0}}; +// server::ServerError err; +// int value = 0; +// int option_index = 0; +// std::string system_info_filename = "./system.info"; +// std::string license_filename = "./system.license"; +// char *string_starting_time = NULL; +// char *string_end_time = NULL; +// time_t starting_time = 0; +// time_t end_time = 0; +// int flag_s = 1; +// int flag_b = 1; +// int flag_e = 1; +// while ((value = getopt_long(argc, argv, "hl:s:b:e:", long_options, NULL)) != -1) { +// switch (value) { +// case 's': { +// flag_s = 0; +// system_info_filename = (std::string) (optarg); +// break; +// } +// case 'b': { +// flag_b = 0; +// string_starting_time = optarg; +// break; +// } +// case 'e': { +// flag_e = 0; +// string_end_time = optarg; +// break; +// } +// case 'l': { +// license_filename = (std::string) (optarg); +// break; +// } +// case 'h':print_usage(app_name); +// return EXIT_SUCCESS; +// case '?':print_usage(app_name); +// return EXIT_FAILURE; +// default:print_usage(app_name); +// break; +// } +// // } - static struct option long_options[] = {{"system_info", required_argument, 0, 's'}, - {"license", optional_argument, 0, 'l'}, - {"help", no_argument, 0, 'h'}, - {"starting_time", required_argument, 0, 'b'}, - {"end_time", required_argument, 0, 'e'}, - {NULL, 0, 0, 0}}; - server::ServerError err; - int value = 0; - int option_index = 0; - std::string system_info_filename = "./system.info"; - std::string license_filename = "./system.license"; - char *string_starting_time = NULL; - char *string_end_time = NULL; - time_t starting_time = 0; - time_t end_time = 0; - int flag_s = 1; - int flag_b = 1; - int flag_e = 1; - while ((value = getopt_long(argc, argv, "hl:s:b:e:", long_options, NULL)) != -1) { - switch (value) { - case 's': { - flag_s = 0; - system_info_filename = (std::string) (optarg); - break; - } - case 'b': { - flag_b = 0; - string_starting_time = optarg; - break; - } - case 'e': { - flag_e = 0; - string_end_time = optarg; - break; - } - case 'l': { - license_filename = (std::string) (optarg); - break; - } - case 'h':print_usage(app_name); - return EXIT_SUCCESS; - case '?':print_usage(app_name); - return EXIT_FAILURE; - default:print_usage(app_name); - break; - } - - } - if (flag_s) { - printf("Error: sysinfo file location must be entered\n"); - return 1; - } - if (flag_b) { - printf("Error: start time must be entered\n"); - return 1; - } - if (flag_e) { - printf("Error: end time must be entered\n"); - return 1; - } - - err = server::LicenseLibrary::GetDateTime(string_starting_time, starting_time); - if (err != server::SERVER_SUCCESS) return -1; - - err = server::LicenseLibrary::GetDateTime(string_end_time, end_time); - if (err != server::SERVER_SUCCESS) return -1; - - - int output_info_device_count = 0; - std::map output_info_uuid_encrption_map; - - - err = server::LicenseLibrary::GPUinfoFileDeserialization(system_info_filename, - output_info_device_count, - output_info_uuid_encrption_map); - if (err != server::SERVER_SUCCESS) return -1; - - - err = server::LicenseLibrary::LicenseFileSerialization(license_filename, - output_info_device_count, - output_info_uuid_encrption_map, - starting_time, - end_time); - if (err != server::SERVER_SUCCESS) return -1; - - - printf("Generate License File Success\n"); - - return 0; -} +// if (flag_s) { +// printf("Error: sysinfo file location must be entered\n"); +// return 1; +// } +// if (flag_b) { +// printf("Error: start time must be entered\n"); +// return 1; +// } +// if (flag_e) { +// printf("Error: end time must be entered\n"); +// return 1; +// } +// +// err = server::LicenseLibrary::GetDateTime(string_starting_time, starting_time); +// if (err != server::SERVER_SUCCESS) return -1; +// +// err = server::LicenseLibrary::GetDateTime(string_end_time, end_time); +// if (err != server::SERVER_SUCCESS) return -1; +// +// +// int output_info_device_count = 0; +// std::map output_info_uuid_encrption_map; +// +// +// err = server::LicenseLibrary::GPUinfoFileDeserialization(system_info_filename, +// output_info_device_count, +// output_info_uuid_encrption_map); +// if (err != server::SERVER_SUCCESS) return -1; +// +// +// err = server::LicenseLibrary::LicenseFileSerialization(license_filename, +// output_info_device_count, +// output_info_uuid_encrption_map, +// starting_time, +// end_time); +// if (err != server::SERVER_SUCCESS) return -1; +// +// +// printf("Generate License File Success\n"); +// +// return 0; +//} diff --git a/cpp/src/license/LicenseLibrary.cpp b/cpp/src/license/LicenseLibrary.cpp index 0bb616f5ea..bf377650a6 100644 --- a/cpp/src/license/LicenseLibrary.cpp +++ b/cpp/src/license/LicenseLibrary.cpp @@ -1,345 +1,345 @@ -//////////////////////////////////////////////////////////////////////////////// -// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved -// Unauthorized copying of this file, via any medium is strictly prohibited. -// Proprietary and confidential. -//////////////////////////////////////////////////////////////////////////////// - -#include "LicenseLibrary.h" -#include "utils/Log.h" -#include -#include -#include -#include - -#include -#include -//#include -//#include -#include -#include -#include - - -namespace zilliz { -namespace milvus { -namespace server { - -constexpr int LicenseLibrary::sha256_length_; - -// Part 0: File check -bool -LicenseLibrary::IsFileExistent(const std::string &path) { - - boost::system::error_code error; - auto file_status = boost::filesystem::status(path, error); - if (error) { - return false; - } - - if (!boost::filesystem::exists(file_status)) { - return false; - } - - return !boost::filesystem::is_directory(file_status); -} - -// Part 1: Get GPU Info -ServerError -LicenseLibrary::GetDeviceCount(int &device_count) { - nvmlReturn_t result = nvmlInit(); - if (NVML_SUCCESS != result) { - printf("Failed to initialize NVML: %s\n", nvmlErrorString(result)); - return SERVER_UNEXPECTED_ERROR; - } - cudaError_t error_id = cudaGetDeviceCount(&device_count); - if (error_id != cudaSuccess) { - printf("cudaGetDeviceCount returned %d\n-> %s\n", (int) error_id, cudaGetErrorString(error_id)); - printf("Result = FAIL\n"); - return SERVER_UNEXPECTED_ERROR; - } - return SERVER_SUCCESS; -} - -ServerError -LicenseLibrary::GetUUID(int device_count, std::vector &uuid_array) { - if (device_count == 0) { - printf("There are no available device(s) that support CUDA\n"); - return SERVER_UNEXPECTED_ERROR; - } - - for (int dev = 0; dev < device_count; ++dev) { - nvmlDevice_t device; - nvmlReturn_t result = nvmlDeviceGetHandleByIndex(dev, &device); - if (NVML_SUCCESS != result) { - printf("Failed to get handle for device %i: %s\n", dev, nvmlErrorString(result)); - return SERVER_UNEXPECTED_ERROR; - } - - char uuid[80]; - unsigned int length = 80; - nvmlReturn_t err = nvmlDeviceGetUUID(device, uuid, length); - if (err != NVML_SUCCESS) { - printf("nvmlDeviceGetUUID error: %d\n", err); - return SERVER_UNEXPECTED_ERROR; - } - - uuid_array.emplace_back(uuid); - } - return SERVER_SUCCESS; -} - -ServerError -LicenseLibrary::GetUUIDMD5(int device_count, - std::vector &uuid_array, - std::vector &md5_array) { - MD5_CTX ctx; - unsigned char outmd[16]; - char temp[2]; - std::string md5; - for (int dev = 0; dev < device_count; ++dev) { - md5.clear(); - memset(outmd, 0, sizeof(outmd)); - MD5_Init(&ctx); - MD5_Update(&ctx, uuid_array[dev].c_str(), uuid_array[dev].size()); - MD5_Final(outmd, &ctx); - for (int i = 0; i < 16; ++i) { - std::snprintf(temp, 2, "%02X", outmd[i]); - md5 += temp; - } - md5_array.push_back(md5); - } - return SERVER_SUCCESS; -} - -ServerError -LicenseLibrary::GetUUIDSHA256(const int &device_count, - std::vector &uuid_array, - std::vector &sha_array) { - SHA256_CTX ctx; - unsigned char outmd[sha256_length_]; - char temp[2]; - std::string sha; - for (int dev = 0; dev < device_count; ++dev) { - sha.clear(); - memset(outmd, 0, sizeof(outmd)); - SHA256_Init(&ctx); - SHA256_Update(&ctx, uuid_array[dev].c_str(), uuid_array[dev].size()); - SHA256_Final(outmd, &ctx); - for (int i = 0; i < sha256_length_; ++i) { - std::snprintf(temp, 2, "%02X", outmd[i]); - sha += temp; - } - sha_array.push_back(sha); - } - return SERVER_SUCCESS; -} - -ServerError -LicenseLibrary::GetSystemTime(time_t &system_time) { - system_time = time(NULL); - return SERVER_SUCCESS; -} - -// Part 2: Handle License File -ServerError -LicenseLibrary::LicenseFileSerialization(const std::string &path, - int device_count, - const std::map &uuid_encrption_map, - time_t starting_time, - time_t end_time) { - - std::ofstream file(path); - boost::archive::binary_oarchive oa(file); - oa.register_type(); - - SerializedLicenseFile serialized_license_file; - - serialized_license_file.set_license_file(new LicenseFile(device_count, - uuid_encrption_map, - starting_time, - end_time)); - oa << serialized_license_file; - - file.close(); - return SERVER_SUCCESS; -} - -ServerError -LicenseLibrary::LicenseFileDeserialization(const std::string &path, - int &device_count, - std::map &uuid_encrption_map, - time_t &starting_time, - time_t &end_time) { - if (!IsFileExistent(path)) return SERVER_LICENSE_FILE_NOT_EXIST; - std::ifstream file(path); - boost::archive::binary_iarchive ia(file); - ia.register_type(); - - SerializedLicenseFile serialized_license_file; - ia >> serialized_license_file; - - device_count = serialized_license_file.get_license_file()->get_device_count(); - uuid_encrption_map = serialized_license_file.get_license_file()->get_uuid_encryption_map(); - starting_time = serialized_license_file.get_license_file()->get_starting_time(); - end_time = serialized_license_file.get_license_file()->get_end_time(); - - file.close(); - return SERVER_SUCCESS; -} - +////////////////////////////////////////////////////////////////////////////////// +//// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved +//// Unauthorized copying of this file, via any medium is strictly prohibited. +//// Proprietary and confidential. +////////////////////////////////////////////////////////////////////////////////// +// +//#include "LicenseLibrary.h" +//#include "utils/Log.h" +//#include +//#include +//#include +//#include +// +//#include +//#include +////#include +////#include +//#include +//#include +//#include +// +// +//namespace zilliz { +//namespace milvus { +//namespace server { +// +//constexpr int LicenseLibrary::sha256_length_; +// +//// Part 0: File check +//bool +//LicenseLibrary::IsFileExistent(const std::string &path) { +// +// boost::system::error_code error; +// auto file_status = boost::filesystem::status(path, error); +// if (error) { +// return false; +// } +// +// if (!boost::filesystem::exists(file_status)) { +// return false; +// } +// +// return !boost::filesystem::is_directory(file_status); +//} +// +//// Part 1: Get GPU Info //ServerError -//LicenseLibrary::SecretFileSerialization(const std::string &path, -// const time_t &update_time, -// const off_t &file_size, -// const time_t &starting_time, -// const time_t &end_time, -// const std::string &file_md5) { +//LicenseLibrary::GetDeviceCount(int &device_count) { +// nvmlReturn_t result = nvmlInit(); +// if (NVML_SUCCESS != result) { +// printf("Failed to initialize NVML: %s\n", nvmlErrorString(result)); +// return SERVER_UNEXPECTED_ERROR; +// } +// cudaError_t error_id = cudaGetDeviceCount(&device_count); +// if (error_id != cudaSuccess) { +// printf("cudaGetDeviceCount returned %d\n-> %s\n", (int) error_id, cudaGetErrorString(error_id)); +// printf("Result = FAIL\n"); +// return SERVER_UNEXPECTED_ERROR; +// } +// return SERVER_SUCCESS; +//} +// +//ServerError +//LicenseLibrary::GetUUID(int device_count, std::vector &uuid_array) { +// if (device_count == 0) { +// printf("There are no available device(s) that support CUDA\n"); +// return SERVER_UNEXPECTED_ERROR; +// } +// +// for (int dev = 0; dev < device_count; ++dev) { +// nvmlDevice_t device; +// nvmlReturn_t result = nvmlDeviceGetHandleByIndex(dev, &device); +// if (NVML_SUCCESS != result) { +// printf("Failed to get handle for device %i: %s\n", dev, nvmlErrorString(result)); +// return SERVER_UNEXPECTED_ERROR; +// } +// +// char uuid[80]; +// unsigned int length = 80; +// nvmlReturn_t err = nvmlDeviceGetUUID(device, uuid, length); +// if (err != NVML_SUCCESS) { +// printf("nvmlDeviceGetUUID error: %d\n", err); +// return SERVER_UNEXPECTED_ERROR; +// } +// +// uuid_array.emplace_back(uuid); +// } +// return SERVER_SUCCESS; +//} +// +//ServerError +//LicenseLibrary::GetUUIDMD5(int device_count, +// std::vector &uuid_array, +// std::vector &md5_array) { +// MD5_CTX ctx; +// unsigned char outmd[16]; +// char temp[2]; +// std::string md5; +// for (int dev = 0; dev < device_count; ++dev) { +// md5.clear(); +// memset(outmd, 0, sizeof(outmd)); +// MD5_Init(&ctx); +// MD5_Update(&ctx, uuid_array[dev].c_str(), uuid_array[dev].size()); +// MD5_Final(outmd, &ctx); +// for (int i = 0; i < 16; ++i) { +// std::snprintf(temp, 2, "%02X", outmd[i]); +// md5 += temp; +// } +// md5_array.push_back(md5); +// } +// return SERVER_SUCCESS; +//} +// +//ServerError +//LicenseLibrary::GetUUIDSHA256(const int &device_count, +// std::vector &uuid_array, +// std::vector &sha_array) { +// SHA256_CTX ctx; +// unsigned char outmd[sha256_length_]; +// char temp[2]; +// std::string sha; +// for (int dev = 0; dev < device_count; ++dev) { +// sha.clear(); +// memset(outmd, 0, sizeof(outmd)); +// SHA256_Init(&ctx); +// SHA256_Update(&ctx, uuid_array[dev].c_str(), uuid_array[dev].size()); +// SHA256_Final(outmd, &ctx); +// for (int i = 0; i < sha256_length_; ++i) { +// std::snprintf(temp, 2, "%02X", outmd[i]); +// sha += temp; +// } +// sha_array.push_back(sha); +// } +// return SERVER_SUCCESS; +//} +// +//ServerError +//LicenseLibrary::GetSystemTime(time_t &system_time) { +// system_time = time(NULL); +// return SERVER_SUCCESS; +//} +// +//// Part 2: Handle License File +//ServerError +//LicenseLibrary::LicenseFileSerialization(const std::string &path, +// int device_count, +// const std::map &uuid_encrption_map, +// time_t starting_time, +// time_t end_time) { +// // std::ofstream file(path); // boost::archive::binary_oarchive oa(file); -// oa.register_type(); +// oa.register_type(); // -// SerializedSecretFile serialized_secret_file; +// SerializedLicenseFile serialized_license_file; // -// serialized_secret_file.set_secret_file(new SecretFile(update_time, file_size, starting_time, end_time, file_md5)); -// oa << serialized_secret_file; +// serialized_license_file.set_license_file(new LicenseFile(device_count, +// uuid_encrption_map, +// starting_time, +// end_time)); +// oa << serialized_license_file; // // file.close(); // return SERVER_SUCCESS; //} // //ServerError -//LicenseLibrary::SecretFileDeserialization(const std::string &path, -// time_t &update_time, -// off_t &file_size, -// time_t &starting_time, -// time_t &end_time, -// std::string &file_md5) { +//LicenseLibrary::LicenseFileDeserialization(const std::string &path, +// int &device_count, +// std::map &uuid_encrption_map, +// time_t &starting_time, +// time_t &end_time) { +// if (!IsFileExistent(path)) return SERVER_LICENSE_FILE_NOT_EXIST; +// std::ifstream file(path); +// boost::archive::binary_iarchive ia(file); +// ia.register_type(); +// +// SerializedLicenseFile serialized_license_file; +// ia >> serialized_license_file; +// +// device_count = serialized_license_file.get_license_file()->get_device_count(); +// uuid_encrption_map = serialized_license_file.get_license_file()->get_uuid_encryption_map(); +// starting_time = serialized_license_file.get_license_file()->get_starting_time(); +// end_time = serialized_license_file.get_license_file()->get_end_time(); +// +// file.close(); +// return SERVER_SUCCESS; +//} +// +////ServerError +////LicenseLibrary::SecretFileSerialization(const std::string &path, +//// const time_t &update_time, +//// const off_t &file_size, +//// const time_t &starting_time, +//// const time_t &end_time, +//// const std::string &file_md5) { +//// std::ofstream file(path); +//// boost::archive::binary_oarchive oa(file); +//// oa.register_type(); +//// +//// SerializedSecretFile serialized_secret_file; +//// +//// serialized_secret_file.set_secret_file(new SecretFile(update_time, file_size, starting_time, end_time, file_md5)); +//// oa << serialized_secret_file; +//// +//// file.close(); +//// return SERVER_SUCCESS; +////} +//// +////ServerError +////LicenseLibrary::SecretFileDeserialization(const std::string &path, +//// time_t &update_time, +//// off_t &file_size, +//// time_t &starting_time, +//// time_t &end_time, +//// std::string &file_md5) { +//// if (!IsFileExistent(path)) return SERVER_LICENSE_FILE_NOT_EXIST; +//// +//// std::ifstream file(path); +//// boost::archive::binary_iarchive ia(file); +//// ia.register_type(); +//// SerializedSecretFile serialized_secret_file; +//// +//// ia >> serialized_secret_file; +//// update_time = serialized_secret_file.get_secret_file()->get_update_time(); +//// file_size = serialized_secret_file.get_secret_file()->get_file_size(); +//// starting_time = serialized_secret_file.get_secret_file()->get_starting_time(); +//// end_time = serialized_secret_file.get_secret_file()->get_end_time(); +//// file_md5 = serialized_secret_file.get_secret_file()->get_file_md5(); +//// file.close(); +//// return SERVER_SUCCESS; +////} +// +// +// +//// Part 3: File attribute: UpdateTime Time/ Size/ MD5 +//ServerError +//LicenseLibrary::GetFileUpdateTimeAndSize(const std::string &path, time_t &update_time, off_t &file_size) { +// +// if (!IsFileExistent(path)) return SERVER_LICENSE_FILE_NOT_EXIST; +// +// struct stat buf; +// int err_no = stat(path.c_str(), &buf); +// if (err_no != 0) { +// std::cout << strerror(err_no) << std::endl; +// return SERVER_UNEXPECTED_ERROR; +// } +// +// update_time = buf.st_mtime; +// file_size = buf.st_size; +// +// return SERVER_SUCCESS; +//} +// +//ServerError +//LicenseLibrary::GetFileMD5(const std::string &path, std::string &filemd5) { +// +// if (!IsFileExistent(path)) return SERVER_LICENSE_FILE_NOT_EXIST; +// +// filemd5.clear(); +// +// std::ifstream file(path.c_str(), std::ifstream::binary); +// if (!file) { +// return -1; +// } +// +// MD5_CTX md5Context; +// MD5_Init(&md5Context); +// +// char buf[1024 * 16]; +// while (file.good()) { +// file.read(buf, sizeof(buf)); +// MD5_Update(&md5Context, buf, file.gcount()); +// } +// +// unsigned char result[MD5_DIGEST_LENGTH]; +// MD5_Final(result, &md5Context); +// +// char hex[35]; +// memset(hex, 0, sizeof(hex)); +// for (int i = 0; i < MD5_DIGEST_LENGTH; ++i) { +// sprintf(hex + i * 2, "%02X", result[i]); +// } +// hex[32] = '\0'; +// filemd5 = std::string(hex); +// +// return SERVER_SUCCESS; +//} +//// Part 4: GPU Info File Serialization/Deserialization +//ServerError +//LicenseLibrary::GPUinfoFileSerialization(const std::string &path, +// int device_count, +// const std::map &uuid_encrption_map) { +// std::ofstream file(path); +// boost::archive::binary_oarchive oa(file); +// oa.register_type(); +// +// SerializedGPUInfoFile serialized_gpu_info_file; +// +// serialized_gpu_info_file.set_gpu_info_file(new GPUInfoFile(device_count, uuid_encrption_map)); +// oa << serialized_gpu_info_file; +// +// file.close(); +// return SERVER_SUCCESS; +//} +//ServerError +//LicenseLibrary::GPUinfoFileDeserialization(const std::string &path, +// int &device_count, +// std::map &uuid_encrption_map) { // if (!IsFileExistent(path)) return SERVER_LICENSE_FILE_NOT_EXIST; // // std::ifstream file(path); // boost::archive::binary_iarchive ia(file); -// ia.register_type(); -// SerializedSecretFile serialized_secret_file; +// ia.register_type(); +// +// SerializedGPUInfoFile serialized_gpu_info_file; +// ia >> serialized_gpu_info_file; +// +// device_count = serialized_gpu_info_file.get_gpu_info_file()->get_device_count(); +// uuid_encrption_map = serialized_gpu_info_file.get_gpu_info_file()->get_uuid_encryption_map(); // -// ia >> serialized_secret_file; -// update_time = serialized_secret_file.get_secret_file()->get_update_time(); -// file_size = serialized_secret_file.get_secret_file()->get_file_size(); -// starting_time = serialized_secret_file.get_secret_file()->get_starting_time(); -// end_time = serialized_secret_file.get_secret_file()->get_end_time(); -// file_md5 = serialized_secret_file.get_secret_file()->get_file_md5(); // file.close(); // return SERVER_SUCCESS; //} - - - -// Part 3: File attribute: UpdateTime Time/ Size/ MD5 -ServerError -LicenseLibrary::GetFileUpdateTimeAndSize(const std::string &path, time_t &update_time, off_t &file_size) { - - if (!IsFileExistent(path)) return SERVER_LICENSE_FILE_NOT_EXIST; - - struct stat buf; - int err_no = stat(path.c_str(), &buf); - if (err_no != 0) { - std::cout << strerror(err_no) << std::endl; - return SERVER_UNEXPECTED_ERROR; - } - - update_time = buf.st_mtime; - file_size = buf.st_size; - - return SERVER_SUCCESS; -} - -ServerError -LicenseLibrary::GetFileMD5(const std::string &path, std::string &filemd5) { - - if (!IsFileExistent(path)) return SERVER_LICENSE_FILE_NOT_EXIST; - - filemd5.clear(); - - std::ifstream file(path.c_str(), std::ifstream::binary); - if (!file) { - return -1; - } - - MD5_CTX md5Context; - MD5_Init(&md5Context); - - char buf[1024 * 16]; - while (file.good()) { - file.read(buf, sizeof(buf)); - MD5_Update(&md5Context, buf, file.gcount()); - } - - unsigned char result[MD5_DIGEST_LENGTH]; - MD5_Final(result, &md5Context); - - char hex[35]; - memset(hex, 0, sizeof(hex)); - for (int i = 0; i < MD5_DIGEST_LENGTH; ++i) { - sprintf(hex + i * 2, "%02X", result[i]); - } - hex[32] = '\0'; - filemd5 = std::string(hex); - - return SERVER_SUCCESS; -} -// Part 4: GPU Info File Serialization/Deserialization -ServerError -LicenseLibrary::GPUinfoFileSerialization(const std::string &path, - int device_count, - const std::map &uuid_encrption_map) { - std::ofstream file(path); - boost::archive::binary_oarchive oa(file); - oa.register_type(); - - SerializedGPUInfoFile serialized_gpu_info_file; - - serialized_gpu_info_file.set_gpu_info_file(new GPUInfoFile(device_count, uuid_encrption_map)); - oa << serialized_gpu_info_file; - - file.close(); - return SERVER_SUCCESS; -} -ServerError -LicenseLibrary::GPUinfoFileDeserialization(const std::string &path, - int &device_count, - std::map &uuid_encrption_map) { - if (!IsFileExistent(path)) return SERVER_LICENSE_FILE_NOT_EXIST; - - std::ifstream file(path); - boost::archive::binary_iarchive ia(file); - ia.register_type(); - - SerializedGPUInfoFile serialized_gpu_info_file; - ia >> serialized_gpu_info_file; - - device_count = serialized_gpu_info_file.get_gpu_info_file()->get_device_count(); - uuid_encrption_map = serialized_gpu_info_file.get_gpu_info_file()->get_uuid_encryption_map(); - - file.close(); - return SERVER_SUCCESS; -} - -ServerError -LicenseLibrary::GetDateTime(const char *cha, time_t &data_time) { - tm tm_; - int year, month, day; - sscanf(cha, "%d-%d-%d", &year, &month, &day); - tm_.tm_year = year - 1900; - tm_.tm_mon = month - 1; - tm_.tm_mday = day; - tm_.tm_hour = 0; - tm_.tm_min = 0; - tm_.tm_sec = 0; - tm_.tm_isdst = 0; - data_time = mktime(&tm_); - return SERVER_SUCCESS; - -} - -} -} -} \ No newline at end of file +// +//ServerError +//LicenseLibrary::GetDateTime(const char *cha, time_t &data_time) { +// tm tm_; +// int year, month, day; +// sscanf(cha, "%d-%d-%d", &year, &month, &day); +// tm_.tm_year = year - 1900; +// tm_.tm_mon = month - 1; +// tm_.tm_mday = day; +// tm_.tm_hour = 0; +// tm_.tm_min = 0; +// tm_.tm_sec = 0; +// tm_.tm_isdst = 0; +// data_time = mktime(&tm_); +// return SERVER_SUCCESS; +// +//} +// +//} +//} +//} \ No newline at end of file diff --git a/cpp/src/license/LicenseLibrary.h b/cpp/src/license/LicenseLibrary.h index 3340e60caa..427dda8d47 100644 --- a/cpp/src/license/LicenseLibrary.h +++ b/cpp/src/license/LicenseLibrary.h @@ -1,105 +1,105 @@ -#pragma once - -#include "LicenseFile.h" -#include "GPUInfoFile.h" - -#include "utils/Error.h" - -#include -#include -#include - -#include -#include -#include - - -namespace zilliz { -namespace milvus { -namespace server { - -class LicenseLibrary { - public: - // Part 0: File check - static bool - IsFileExistent(const std::string &path); - - // Part 1: Get GPU Info - static ServerError - GetDeviceCount(int &device_count); - - static ServerError - GetUUID(int device_count, std::vector &uuid_array); - - static ServerError - GetUUIDMD5(int device_count, std::vector &uuid_array, std::vector &md5_array); - - - static ServerError - GetUUIDSHA256(const int &device_count, - std::vector &uuid_array, - std::vector &sha_array); - - static ServerError - GetSystemTime(time_t &system_time); - - // Part 2: Handle License File - static ServerError - LicenseFileSerialization(const std::string &path, - int device_count, - const std::map &uuid_encrption_map, - time_t starting_time, - time_t end_time); - - static ServerError - LicenseFileDeserialization(const std::string &path, - int &device_count, - std::map &uuid_encrption_map, - time_t &starting_time, - time_t &end_time); - +//#pragma once +// +//#include "LicenseFile.h" +//#include "GPUInfoFile.h" +// +//#include "utils/Error.h" +// +//#include +//#include +//#include +// +//#include +//#include +//#include +// +// +//namespace zilliz { +//namespace milvus { +//namespace server { +// +//class LicenseLibrary { +// public: +// // Part 0: File check +// static bool +// IsFileExistent(const std::string &path); +// +// // Part 1: Get GPU Info // static ServerError -// SecretFileSerialization(const std::string &path, -// const time_t &update_time, -// const off_t &file_size, -// const time_t &starting_time, -// const time_t &end_time, -// const std::string &file_md5); +// GetDeviceCount(int &device_count); // // static ServerError -// SecretFileDeserialization(const std::string &path, -// time_t &update_time, -// off_t &file_size, -// time_t &starting_time, -// time_t &end_time, -// std::string &file_md5); - - // Part 3: File attribute: UpdateTime Time/ Size/ MD5 - static ServerError - GetFileUpdateTimeAndSize(const std::string &path, time_t &update_time, off_t &file_size); - - static ServerError - GetFileMD5(const std::string &path, std::string &filemd5); - - // Part 4: GPU Info File Serialization/Deserialization - static ServerError - GPUinfoFileSerialization(const std::string &path, - int device_count, - const std::map &uuid_encrption_map); - static ServerError - GPUinfoFileDeserialization(const std::string &path, - int &device_count, - std::map &uuid_encrption_map); - - static ServerError - GetDateTime(const char *cha, time_t &data_time); - - - private: - static constexpr int sha256_length_ = 32; -}; - - -} -} -} +// GetUUID(int device_count, std::vector &uuid_array); +// +// static ServerError +// GetUUIDMD5(int device_count, std::vector &uuid_array, std::vector &md5_array); +// +// +// static ServerError +// GetUUIDSHA256(const int &device_count, +// std::vector &uuid_array, +// std::vector &sha_array); +// +// static ServerError +// GetSystemTime(time_t &system_time); +// +// // Part 2: Handle License File +// static ServerError +// LicenseFileSerialization(const std::string &path, +// int device_count, +// const std::map &uuid_encrption_map, +// time_t starting_time, +// time_t end_time); +// +// static ServerError +// LicenseFileDeserialization(const std::string &path, +// int &device_count, +// std::map &uuid_encrption_map, +// time_t &starting_time, +// time_t &end_time); +// +//// static ServerError +//// SecretFileSerialization(const std::string &path, +//// const time_t &update_time, +//// const off_t &file_size, +//// const time_t &starting_time, +//// const time_t &end_time, +//// const std::string &file_md5); +//// +//// static ServerError +//// SecretFileDeserialization(const std::string &path, +//// time_t &update_time, +//// off_t &file_size, +//// time_t &starting_time, +//// time_t &end_time, +//// std::string &file_md5); +// +// // Part 3: File attribute: UpdateTime Time/ Size/ MD5 +// static ServerError +// GetFileUpdateTimeAndSize(const std::string &path, time_t &update_time, off_t &file_size); +// +// static ServerError +// GetFileMD5(const std::string &path, std::string &filemd5); +// +// // Part 4: GPU Info File Serialization/Deserialization +// static ServerError +// GPUinfoFileSerialization(const std::string &path, +// int device_count, +// const std::map &uuid_encrption_map); +// static ServerError +// GPUinfoFileDeserialization(const std::string &path, +// int &device_count, +// std::map &uuid_encrption_map); +// +// static ServerError +// GetDateTime(const char *cha, time_t &data_time); +// +// +// private: +// static constexpr int sha256_length_ = 32; +//}; +// +// +//} +//} +//} diff --git a/cpp/src/metrics/MetricBase.h b/cpp/src/metrics/MetricBase.h index c18e857e4e..fe9b246503 100644 --- a/cpp/src/metrics/MetricBase.h +++ b/cpp/src/metrics/MetricBase.h @@ -22,40 +22,21 @@ class MetricsBase{ } virtual ServerError Init() {}; - virtual void AddGroupSuccessTotalIncrement(double value = 1) {}; - virtual void AddGroupFailTotalIncrement(double value = 1) {}; - virtual void HasGroupSuccessTotalIncrement(double value = 1) {}; - virtual void HasGroupFailTotalIncrement(double value = 1) {}; - virtual void GetGroupSuccessTotalIncrement(double value = 1) {}; - virtual void GetGroupFailTotalIncrement(double value = 1) {}; - virtual void GetGroupFilesSuccessTotalIncrement(double value = 1) {}; - virtual void GetGroupFilesFailTotalIncrement(double value = 1) {}; + virtual void AddVectorsSuccessTotalIncrement(double value = 1) {}; virtual void AddVectorsFailTotalIncrement(double value = 1) {}; virtual void AddVectorsDurationHistogramOberve(double value) {}; - virtual void SearchSuccessTotalIncrement(double value = 1) {}; - virtual void SearchFailTotalIncrement(double value = 1) {}; - virtual void SearchDurationHistogramObserve(double value) {}; + virtual void RawFileSizeHistogramObserve(double value) {}; virtual void IndexFileSizeHistogramObserve(double value) {}; virtual void BuildIndexDurationSecondsHistogramObserve(double value) {}; - virtual void AllBuildIndexDurationSecondsHistogramObserve(double value) {}; - virtual void CacheUsageGaugeIncrement(double value = 1) {}; - virtual void CacheUsageGaugeDecrement(double value = 1) {}; + virtual void CacheUsageGaugeSet(double value) {}; - virtual void MetaVisitTotalIncrement(double value = 1) {}; - virtual void MetaVisitDurationSecondsHistogramObserve(double value) {}; - virtual void MemUsagePercentGaugeSet(double value) {}; - virtual void MemUsagePercentGaugeIncrement(double value = 1) {}; - virtual void MemUsagePercentGaugeDecrement(double value = 1) {}; - virtual void MemUsageTotalGaugeSet(double value) {}; - virtual void MemUsageTotalGaugeIncrement(double value = 1) {}; - virtual void MemUsageTotalGaugeDecrement(double value = 1) {}; + virtual void MetaAccessTotalIncrement(double value = 1) {}; virtual void MetaAccessDurationSecondsHistogramObserve(double value) {}; virtual void FaissDiskLoadDurationSecondsHistogramObserve(double value) {}; virtual void FaissDiskLoadSizeBytesHistogramObserve(double value) {}; - virtual void FaissDiskLoadIOSpeedHistogramObserve(double value) {}; virtual void CacheAccessTotalIncrement(double value = 1) {}; virtual void MemTableMergeDurationSecondsHistogramObserve(double value) {}; virtual void SearchIndexDataDurationSecondsHistogramObserve(double value) {}; diff --git a/cpp/src/metrics/PrometheusMetrics.cpp b/cpp/src/metrics/PrometheusMetrics.cpp index d0d50800ad..df0516344e 100644 --- a/cpp/src/metrics/PrometheusMetrics.cpp +++ b/cpp/src/metrics/PrometheusMetrics.cpp @@ -60,9 +60,6 @@ PrometheusMetrics::GPUPercentGaugeSet() { if(!startup_) return; int numDevide = server::SystemInfo::GetInstance().num_device(); std::vector values = server::SystemInfo::GetInstance().GPUPercent(); -// for (int i = 0; i < numDevide; ++i) { -// GPU_percent_gauges_[i].Set(static_cast(values[i])); -// } if(numDevide >= 1) GPU0_percent_gauge_.Set(static_cast(values[0])); if(numDevide >= 2) GPU1_percent_gauge_.Set(static_cast(values[1])); if(numDevide >= 3) GPU2_percent_gauge_.Set(static_cast(values[2])); @@ -77,13 +74,10 @@ PrometheusMetrics::GPUPercentGaugeSet() { void PrometheusMetrics::GPUMemoryUsageGaugeSet() { if(!startup_) return; - int numDevide = server::SystemInfo::GetInstance().num_device(); std::vector values = server::SystemInfo::GetInstance().GPUMemoryUsed(); constexpr unsigned long long MtoB = 1024*1024; int numDevice = values.size(); -// for (int i = 0; i < numDevice; ++i) { -// GPU_memory_usage_gauges_[i].Set(values[i]/MtoB); -// } + if(numDevice >=1) GPU0_memory_usage_gauge_.Set(values[0]/MtoB); if(numDevice >=2) GPU1_memory_usage_gauge_.Set(values[1]/MtoB); if(numDevice >=3) GPU2_memory_usage_gauge_.Set(values[2]/MtoB); @@ -146,21 +140,6 @@ void PrometheusMetrics::OctetsSet() { outoctets_gauge_.Set((in_and_out_octets.second-old_outoctets)/total_second); } -//void PrometheusMetrics::GpuPercentInit() { -// int num_device = SystemInfo::GetInstance().num_device(); -// constexpr char device_number[] = "DeviceNum"; -// for(int i = 0; i < num_device; ++ i) { -// GPU_percent_gauges_.emplace_back(GPU_percent_.Add({{device_number,std::to_string(i)}})); -// } -// -//} -//void PrometheusMetrics::GpuMemoryInit() { -// int num_device = SystemInfo::GetInstance().num_device(); -// constexpr char device_number[] = "DeviceNum"; -// for(int i = 0; i < num_device; ++ i) { -// GPU_memory_usage_gauges_.emplace_back(GPU_memory_usage_.Add({{device_number,std::to_string(i)}})); -// } -//} } diff --git a/cpp/src/metrics/PrometheusMetrics.h b/cpp/src/metrics/PrometheusMetrics.h index 902e79b5a5..5b651ec14f 100644 --- a/cpp/src/metrics/PrometheusMetrics.h +++ b/cpp/src/metrics/PrometheusMetrics.h @@ -34,10 +34,6 @@ class PrometheusMetrics: public MetricsBase { public: static PrometheusMetrics & GetInstance() { -// switch(MetricCollectorType) { -// case: prometheus:: -// static -// } static PrometheusMetrics instance; return instance; } @@ -49,46 +45,21 @@ class PrometheusMetrics: public MetricsBase { std::shared_ptr exposer_ptr_; std::shared_ptr registry_ = std::make_shared(); bool startup_ = false; -// void GpuPercentInit(); -// void GpuMemoryInit(); public: - - void AddGroupSuccessTotalIncrement(double value = 1.0) override { if(startup_) add_group_success_total_.Increment(value);}; - void AddGroupFailTotalIncrement(double value = 1.0) override { if(startup_) add_group_fail_total_.Increment(value);}; - void HasGroupSuccessTotalIncrement(double value = 1.0) override { if(startup_) has_group_success_total_.Increment(value);}; - void HasGroupFailTotalIncrement(double value = 1.0) override { if(startup_) has_group_fail_total_.Increment(value);}; - void GetGroupSuccessTotalIncrement(double value = 1.0) override { if(startup_) get_group_success_total_.Increment(value);}; - void GetGroupFailTotalIncrement(double value = 1.0) override { if(startup_) get_group_fail_total_.Increment(value);}; - void GetGroupFilesSuccessTotalIncrement(double value = 1.0) override { if(startup_) get_group_files_success_total_.Increment(value);}; - void GetGroupFilesFailTotalIncrement(double value = 1.0) override { if(startup_) get_group_files_fail_total_.Increment(value);}; + void SetStartup(bool startup) {startup_ = startup;}; void AddVectorsSuccessTotalIncrement(double value = 1.0) override { if(startup_) add_vectors_success_total_.Increment(value);}; void AddVectorsFailTotalIncrement(double value = 1.0) override { if(startup_) add_vectors_fail_total_.Increment(value);}; void AddVectorsDurationHistogramOberve(double value) override { if(startup_) add_vectors_duration_histogram_.Observe(value);}; - void SearchSuccessTotalIncrement(double value = 1.0) override { if(startup_) search_success_total_.Increment(value);}; - void SearchFailTotalIncrement(double value = 1.0) override { if(startup_) search_fail_total_.Increment(value); }; - void SearchDurationHistogramObserve(double value) override { if(startup_) search_duration_histogram_.Observe(value);}; void RawFileSizeHistogramObserve(double value) override { if(startup_) raw_files_size_histogram_.Observe(value);}; void IndexFileSizeHistogramObserve(double value) override { if(startup_) index_files_size_histogram_.Observe(value);}; void BuildIndexDurationSecondsHistogramObserve(double value) override { if(startup_) build_index_duration_seconds_histogram_.Observe(value);}; - void AllBuildIndexDurationSecondsHistogramObserve(double value) override { if(startup_) all_build_index_duration_seconds_histogram_.Observe(value);}; - void CacheUsageGaugeIncrement(double value = 1.0) override { if(startup_) cache_usage_gauge_.Increment(value);}; - void CacheUsageGaugeDecrement(double value = 1.0) override { if(startup_) cache_usage_gauge_.Decrement(value);}; void CacheUsageGaugeSet(double value) override { if(startup_) cache_usage_gauge_.Set(value);}; -// void MetaVisitTotalIncrement(double value = 1.0) override { meta_visit_total_.Increment(value);}; -// void MetaVisitDurationSecondsHistogramObserve(double value) override { meta_visit_duration_seconds_histogram_.Observe(value);}; - void MemUsagePercentGaugeSet(double value) override { if(startup_) mem_usage_percent_gauge_.Set(value);}; - void MemUsagePercentGaugeIncrement(double value = 1.0) override { if(startup_) mem_usage_percent_gauge_.Increment(value);}; - void MemUsagePercentGaugeDecrement(double value = 1.0) override { if(startup_) mem_usage_percent_gauge_.Decrement(value);}; - void MemUsageTotalGaugeSet(double value) override { if(startup_) mem_usage_total_gauge_.Set(value);}; - void MemUsageTotalGaugeIncrement(double value = 1.0) override { if(startup_) mem_usage_total_gauge_.Increment(value);}; - void MemUsageTotalGaugeDecrement(double value = 1.0) override { if(startup_) mem_usage_total_gauge_.Decrement(value);}; void MetaAccessTotalIncrement(double value = 1) override { if(startup_) meta_access_total_.Increment(value);}; void MetaAccessDurationSecondsHistogramObserve(double value) override { if(startup_) meta_access_duration_seconds_histogram_.Observe(value);}; void FaissDiskLoadDurationSecondsHistogramObserve(double value) override { if(startup_) faiss_disk_load_duration_seconds_histogram_.Observe(value);}; void FaissDiskLoadSizeBytesHistogramObserve(double value) override { if(startup_) faiss_disk_load_size_bytes_histogram_.Observe(value);}; -// void FaissDiskLoadIOSpeedHistogramObserve(double value) { if(startup_) faiss_disk_load_IO_speed_histogram_.Observe(value);}; void FaissDiskLoadIOSpeedGaugeSet(double value) override { if(startup_) faiss_disk_load_IO_speed_gauge_.Set(value);}; void CacheAccessTotalIncrement(double value = 1) override { if(startup_) cache_access_total_.Increment(value);}; diff --git a/cpp/src/metrics/SystemInfo.cpp b/cpp/src/metrics/SystemInfo.cpp index d4ea2de575..a64cbc4992 100644 --- a/cpp/src/metrics/SystemInfo.cpp +++ b/cpp/src/metrics/SystemInfo.cpp @@ -135,46 +135,6 @@ SystemInfo::CPUPercent() { return percent; } -//std::unordered_map> -//SystemInfo::GetGPUMemPercent(){ -// // return GPUID: MEM% -// -// //write GPU info to a file -// system("nvidia-smi pmon -c 1 > GPUInfo.txt"); -// int pid = (int)getpid(); -// -// //parse line -// std::ifstream read_file; -// read_file.open("GPUInfo.txt"); -// std::string line; -// while(getline(read_file, line)){ -// std::vector words = split(line); -// // 0 1 2 3 4 5 6 7 -// //words stand for gpuindex, pid, type, sm, mem, enc, dec, command respectively -// if(std::stoi(words[1]) != pid) continue; -// int GPUindex = std::stoi(words[0]); -// double sm_percent = std::stod(words[3]); -// double mem_percent = std::stod(words[4]); -// -// } -// -//} - -//std::vector -//SystemInfo::split(std::string input) { -// std::vector words; -// input += " "; -// int word_start = 0; -// for (int i = 0; i < input.size(); ++i) { -// if(input[i] != ' ') continue; -// if(input[i] == ' ') { -// word_start = i + 1; -// continue; -// } -// words.push_back(input.substr(word_start,i-word_start)); -// } -// return words; -//} std::vector SystemInfo::GPUPercent() { diff --git a/cpp/src/metrics/SystemInfo.h b/cpp/src/metrics/SystemInfo.h index eeff4b61f5..2562e316e4 100644 --- a/cpp/src/metrics/SystemInfo.h +++ b/cpp/src/metrics/SystemInfo.h @@ -59,8 +59,6 @@ class SystemInfo { double MemoryPercent(); double CPUPercent(); std::pair Octets(); -// std::unordered_map> GetGPUMemPercent() {}; -// std::vector split(std::string input) {}; std::vector GPUPercent(); std::vector GPUMemoryUsed(); diff --git a/cpp/src/sdk/src/client/ClientProxy.cpp b/cpp/src/sdk/src/client/ClientProxy.cpp index 56a55459eb..d26007cd0b 100644 --- a/cpp/src/sdk/src/client/ClientProxy.cpp +++ b/cpp/src/sdk/src/client/ClientProxy.cpp @@ -4,7 +4,6 @@ * Proprietary and confidential. ******************************************************************************/ #include "ClientProxy.h" -#include "util/ConvertUtil.h" namespace milvus { diff --git a/cpp/src/sdk/src/util/ConvertUtil.cpp b/cpp/src/sdk/src/util/ConvertUtil.cpp deleted file mode 100644 index 867221dc89..0000000000 --- a/cpp/src/sdk/src/util/ConvertUtil.cpp +++ /dev/null @@ -1,44 +0,0 @@ -/******************************************************************************* - * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved - * Unauthorized copying of this file, via any medium is strictly prohibited. - * Proprietary and confidential. - ******************************************************************************/ -#include "ConvertUtil.h" -#include "Exception.h" - -#include - -namespace milvus { - -static const std::string INDEX_RAW = "raw"; -static const std::string INDEX_IVFFLAT = "ivfflat"; - -std::string ConvertUtil::IndexType2Str(IndexType index) { - static const std::map s_index2str = { - {IndexType::cpu_idmap, INDEX_RAW}, - {IndexType::gpu_ivfflat, INDEX_IVFFLAT} - }; - - const auto& iter = s_index2str.find(index); - if(iter == s_index2str.end()) { - throw Exception(StatusCode::InvalidAgument, "Invalid index type"); - } - - return iter->second; -} - -IndexType ConvertUtil::Str2IndexType(const std::string& type) { - static const std::map s_str2index = { - {INDEX_RAW, IndexType::cpu_idmap}, - {INDEX_IVFFLAT, IndexType::gpu_ivfflat} - }; - - const auto& iter = s_str2index.find(type); - if(iter == s_str2index.end()) { - throw Exception(StatusCode::InvalidAgument, "Invalid index type"); - } - - return iter->second; -} - -} \ No newline at end of file diff --git a/cpp/src/sdk/src/util/ConvertUtil.h b/cpp/src/sdk/src/util/ConvertUtil.h deleted file mode 100644 index 1411f3eb6d..0000000000 --- a/cpp/src/sdk/src/util/ConvertUtil.h +++ /dev/null @@ -1,18 +0,0 @@ -/******************************************************************************* - * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved - * Unauthorized copying of this file, via any medium is strictly prohibited. - * Proprietary and confidential. - ******************************************************************************/ -#pragma once - -#include "MilvusApi.h" - -namespace milvus { - -class ConvertUtil { -public: - static std::string IndexType2Str(IndexType index); - static IndexType Str2IndexType(const std::string& type); -}; - -} diff --git a/cpp/src/server/DBWrapper.cpp b/cpp/src/server/DBWrapper.cpp index 2a583a660d..7892a57f2b 100644 --- a/cpp/src/server/DBWrapper.cpp +++ b/cpp/src/server/DBWrapper.cpp @@ -37,12 +37,16 @@ DBWrapper::DBWrapper() { opt.meta.archive_conf.SetCriterias(criterial); //create db root folder - CommonUtil::CreateDirectory(opt.meta.path); + ServerError err = CommonUtil::CreateDirectory(opt.meta.path); + if(err != SERVER_SUCCESS) { + std::cout << "ERROR! Failed to create database root path: " << opt.meta.path << std::endl; + kill(0, SIGUSR1); + } zilliz::milvus::engine::DB::Open(opt, &db_); if(db_ == nullptr) { - SERVER_LOG_ERROR << "Failed to open db"; - throw ServerException(SERVER_NULL_POINTER, "Failed to open db"); + std::cout << "ERROR! Failed to open database" << std::endl; + kill(0, SIGUSR1); } } diff --git a/cpp/src/server/MilvusServer.cpp b/cpp/src/server/MilvusServer.cpp index 5e42eb3d7a..322460968f 100644 --- a/cpp/src/server/MilvusServer.cpp +++ b/cpp/src/server/MilvusServer.cpp @@ -23,6 +23,7 @@ #include #include +#include namespace zilliz { namespace milvus { @@ -92,7 +93,8 @@ MilvusServer::StartService() { return; } } catch (apache::thrift::TException& ex) { - //SERVER_LOG_ERROR << "Server encounter exception: " << ex.what(); + std::cout << "ERROR! " << ex.what() << std::endl; + kill(0, SIGUSR1); } } diff --git a/cpp/src/server/RequestScheduler.cpp b/cpp/src/server/RequestScheduler.cpp index 99ae36701e..36df155b62 100644 --- a/cpp/src/server/RequestScheduler.cpp +++ b/cpp/src/server/RequestScheduler.cpp @@ -18,35 +18,35 @@ using namespace ::milvus; namespace { const std::map &ErrorMap() { static const std::map code_map = { - {SERVER_UNEXPECTED_ERROR, thrift::ErrorCode::ILLEGAL_ARGUMENT}, - {SERVER_NULL_POINTER, thrift::ErrorCode::ILLEGAL_ARGUMENT}, + {SERVER_UNEXPECTED_ERROR, thrift::ErrorCode::UNEXPECTED_ERROR}, + {SERVER_UNSUPPORTED_ERROR, thrift::ErrorCode::UNEXPECTED_ERROR}, + {SERVER_NULL_POINTER, thrift::ErrorCode::UNEXPECTED_ERROR}, {SERVER_INVALID_ARGUMENT, thrift::ErrorCode::ILLEGAL_ARGUMENT}, - {SERVER_FILE_NOT_FOUND, thrift::ErrorCode::ILLEGAL_ARGUMENT}, - {SERVER_NOT_IMPLEMENT, thrift::ErrorCode::ILLEGAL_ARGUMENT}, - {SERVER_BLOCKING_QUEUE_EMPTY, thrift::ErrorCode::ILLEGAL_ARGUMENT}, + {SERVER_FILE_NOT_FOUND, thrift::ErrorCode::FILE_NOT_FOUND}, + {SERVER_NOT_IMPLEMENT, thrift::ErrorCode::UNEXPECTED_ERROR}, + {SERVER_BLOCKING_QUEUE_EMPTY, thrift::ErrorCode::UNEXPECTED_ERROR}, + {SERVER_CANNOT_CREATE_FOLDER, thrift::ErrorCode::CANNOT_CREATE_FOLDER}, + {SERVER_CANNOT_CREATE_FILE, thrift::ErrorCode::CANNOT_CREATE_FILE}, + {SERVER_CANNOT_DELETE_FOLDER, thrift::ErrorCode::CANNOT_DELETE_FOLDER}, + {SERVER_CANNOT_DELETE_FILE, thrift::ErrorCode::CANNOT_DELETE_FILE}, {SERVER_TABLE_NOT_EXIST, thrift::ErrorCode::TABLE_NOT_EXISTS}, + {SERVER_INVALID_TABLE_NAME, thrift::ErrorCode::ILLEGAL_TABLE_NAME}, + {SERVER_INVALID_TABLE_DIMENSION, thrift::ErrorCode::ILLEGAL_DIMENSION}, {SERVER_INVALID_TIME_RANGE, thrift::ErrorCode::ILLEGAL_RANGE}, {SERVER_INVALID_VECTOR_DIMENSION, thrift::ErrorCode::ILLEGAL_DIMENSION}, + + {SERVER_INVALID_INDEX_TYPE, thrift::ErrorCode::ILLEGAL_INDEX_TYPE}, + {SERVER_INVALID_ROWRECORD, thrift::ErrorCode::ILLEGAL_ROWRECORD}, + {SERVER_INVALID_ROWRECORD_ARRAY, thrift::ErrorCode::ILLEGAL_ROWRECORD}, + {SERVER_INVALID_TOPK, thrift::ErrorCode::ILLEGAL_TOPK}, + {SERVER_ILLEGAL_VECTOR_ID, thrift::ErrorCode::ILLEGAL_VECTOR_ID}, + {SERVER_ILLEGAL_SEARCH_RESULT, thrift::ErrorCode::ILLEGAL_SEARCH_RESULT}, + {SERVER_CACHE_ERROR, thrift::ErrorCode::CACHE_FAILED}, + {DB_META_TRANSACTION_FAILED, thrift::ErrorCode::META_FAILED}, }; return code_map; } - - const std::map &ErrorMessage() { - static const std::map msg_map = { - {SERVER_UNEXPECTED_ERROR, "unexpected error occurs"}, - {SERVER_NULL_POINTER, "null pointer error"}, - {SERVER_INVALID_ARGUMENT, "invalid argument"}, - {SERVER_FILE_NOT_FOUND, "file not found"}, - {SERVER_NOT_IMPLEMENT, "not implemented"}, - {SERVER_BLOCKING_QUEUE_EMPTY, "queue empty"}, - {SERVER_TABLE_NOT_EXIST, "table not exist"}, - {SERVER_INVALID_TIME_RANGE, "invalid time range"}, - {SERVER_INVALID_VECTOR_DIMENSION, "invalid vector dimension"}, - }; - - return msg_map; - } } //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -69,6 +69,14 @@ ServerError BaseTask::Execute() { return error_code_; } +ServerError BaseTask::SetError(ServerError error_code, const std::string& error_msg) { + error_code_ = error_code; + error_msg_ = error_msg; + + SERVER_LOG_ERROR << error_msg_; + return error_code_; +} + ServerError BaseTask::WaitToFinish() { std::unique_lock lock(finish_mtx_); finish_cond_.wait(lock, [this] { return done_; }); @@ -102,7 +110,7 @@ void RequestScheduler::ExecTask(BaseTaskPtr& task_ptr) { ex.__set_code(ErrorMap().at(err)); std::string msg = task_ptr->ErrorMsg(); if(msg.empty()){ - msg = ErrorMessage().at(err); + msg = "Error message not set"; } ex.__set_reason(msg); throw ex; diff --git a/cpp/src/server/RequestScheduler.h b/cpp/src/server/RequestScheduler.h index b7562cc367..d4b1e1c826 100644 --- a/cpp/src/server/RequestScheduler.h +++ b/cpp/src/server/RequestScheduler.h @@ -34,6 +34,8 @@ public: protected: virtual ServerError OnExecute() = 0; + ServerError SetError(ServerError error_code, const std::string& msg); + protected: mutable std::mutex finish_mtx_; std::condition_variable finish_cond_; diff --git a/cpp/src/server/RequestTask.cpp b/cpp/src/server/RequestTask.cpp index b645d15a4e..5bf2165cfe 100644 --- a/cpp/src/server/RequestTask.cpp +++ b/cpp/src/server/RequestTask.cpp @@ -53,26 +53,27 @@ namespace { return map_type[type]; } - ServerError + void ConvertRowRecordToFloatArray(const std::vector& record_array, uint64_t dimension, - std::vector& float_array) { - ServerError error_code; + std::vector& float_array, + ServerError& error_code, + std::string& error_msg) { uint64_t vec_count = record_array.size(); float_array.resize(vec_count*dimension);//allocate enough memory for(uint64_t i = 0; i < vec_count; i++) { const auto& record = record_array[i]; if(record.vector_data.empty()) { - error_code = SERVER_INVALID_ARGUMENT; - SERVER_LOG_ERROR << "No vector provided in record"; - return error_code; + error_code = SERVER_INVALID_ROWRECORD; + error_msg = "Rowrecord float array is empty"; + return; } uint64_t vec_dim = record.vector_data.size()/sizeof(double);//how many double value? if(vec_dim != dimension) { - SERVER_LOG_ERROR << "Invalid vector dimension: " << vec_dim - << " vs. table dimension:" << dimension; error_code = SERVER_INVALID_VECTOR_DIMENSION; - return error_code; + error_msg = "Invalid rowrecord dimension: " + std::to_string(vec_dim) + + " vs. table dimension:" + std::to_string(dimension); + return; } //convert double array to float array(thrift has no float type) @@ -81,30 +82,29 @@ namespace { float_array[i*vec_dim + d] = (float)(d_p[d]); } } - - return SERVER_SUCCESS; } static constexpr long DAY_SECONDS = 86400; - ServerError + void ConvertTimeRangeToDBDates(const std::vector &range_array, - std::vector& dates) { + std::vector& dates, + ServerError& error_code, + std::string& error_msg) { dates.clear(); - ServerError error_code; for(auto& range : range_array) { time_t tt_start, tt_end; tm tm_start, tm_end; if(!CommonUtil::TimeStrToTime(range.start_value, tt_start, tm_start)){ error_code = SERVER_INVALID_TIME_RANGE; - SERVER_LOG_ERROR << "Invalid time range: " << range.start_value; - return error_code; + error_msg = "Invalid time range: " + range.start_value; + return; } if(!CommonUtil::TimeStrToTime(range.end_value, tt_end, tm_end)){ error_code = SERVER_INVALID_TIME_RANGE; - SERVER_LOG_ERROR << "Invalid time range: " << range.end_value; - return error_code; + error_msg = "Invalid time range: " + range.start_value; + return; } long days = (tt_end > tt_start) ? (tt_end - tt_start)/DAY_SECONDS : (tt_start - tt_end)/DAY_SECONDS; @@ -117,8 +117,6 @@ namespace { dates.push_back(date); } } - - return SERVER_SUCCESS; } } @@ -138,19 +136,16 @@ ServerError CreateTableTask::OnExecute() { try { //step 1: check arguments - if(schema_.table_name.empty() || schema_.dimension <= 0) { - error_code_ = SERVER_INVALID_ARGUMENT; - error_msg_ = "Invalid table name or dimension"; - SERVER_LOG_ERROR << error_msg_; - return error_code_; + if(schema_.table_name.empty()) { + return SetError(SERVER_INVALID_TABLE_NAME, "Empty table name"); + } + if(schema_.dimension <= 0) { + return SetError(SERVER_INVALID_TABLE_DIMENSION, "Invalid table dimension: " + std::to_string(schema_.dimension)); } engine::EngineType engine_type = EngineType(schema_.index_type); if(engine_type == engine::EngineType::INVALID) { - error_code_ = SERVER_INVALID_ARGUMENT; - error_msg_ = "Invalid index type"; - SERVER_LOG_ERROR << error_msg_; - return error_code_; + return SetError(SERVER_INVALID_INDEX_TYPE, "Invalid index type: " + std::to_string(schema_.index_type)); } //step 2: construct table schema @@ -163,17 +158,11 @@ ServerError CreateTableTask::OnExecute() { //step 3: create table engine::Status stat = DBWrapper::DB()->CreateTable(table_info); if(!stat.ok()) {//table could exist - error_code_ = SERVER_UNEXPECTED_ERROR; - error_msg_ = "Engine failed: " + stat.ToString(); - SERVER_LOG_ERROR << error_msg_; - return error_code_; + return SetError(DB_META_TRANSACTION_FAILED, "Engine failed: " + stat.ToString()); } } catch (std::exception& ex) { - error_code_ = SERVER_UNEXPECTED_ERROR; - error_msg_ = ex.what(); - SERVER_LOG_ERROR << error_msg_; - return error_code_; + return SetError(SERVER_UNEXPECTED_ERROR, ex.what()); } rc.Record("done"); @@ -199,10 +188,7 @@ ServerError DescribeTableTask::OnExecute() { try { //step 1: check arguments if(table_name_.empty()) { - error_code_ = SERVER_INVALID_ARGUMENT; - error_msg_ = "Table name cannot be empty"; - SERVER_LOG_ERROR << error_msg_; - return error_code_; + return SetError(SERVER_INVALID_TABLE_NAME, "Empty table name"); } //step 2: get table info @@ -210,10 +196,7 @@ ServerError DescribeTableTask::OnExecute() { table_info.table_id_ = table_name_; engine::Status stat = DBWrapper::DB()->DescribeTable(table_info); if(!stat.ok()) { - error_code_ = SERVER_TABLE_NOT_EXIST; - error_msg_ = "Engine failed: " + stat.ToString(); - SERVER_LOG_ERROR << error_msg_; - return error_code_; + return SetError(DB_META_TRANSACTION_FAILED, "Engine failed: " + stat.ToString()); } schema_.table_name = table_info.table_id_; @@ -222,10 +205,7 @@ ServerError DescribeTableTask::OnExecute() { schema_.store_raw_vector = table_info.store_raw_data_; } catch (std::exception& ex) { - error_code_ = SERVER_UNEXPECTED_ERROR; - error_msg_ = ex.what(); - SERVER_LOG_ERROR << error_msg_; - return SERVER_UNEXPECTED_ERROR; + return SetError(SERVER_UNEXPECTED_ERROR, ex.what()); } rc.Record("done"); @@ -250,22 +230,19 @@ ServerError HasTableTask::OnExecute() { TimeRecorder rc("HasTableTask"); //step 1: check arguments - if (table_name_.empty()) { - error_code_ = SERVER_INVALID_ARGUMENT; - error_msg_ = "Table name cannot be empty"; - SERVER_LOG_ERROR << error_msg_; - return error_code_; + if(table_name_.empty()) { + return SetError(SERVER_INVALID_TABLE_NAME, "Empty table name"); } //step 2: check table existence engine::Status stat = DBWrapper::DB()->HasTable(table_name_, has_table_); + if(!stat.ok()) { + return SetError(DB_META_TRANSACTION_FAILED, "Engine failed: " + stat.ToString()); + } rc.Elapse("totally cost"); } catch (std::exception& ex) { - error_code_ = SERVER_UNEXPECTED_ERROR; - error_msg_ = ex.what(); - SERVER_LOG_ERROR << error_msg_; - return error_code_; + return SetError(SERVER_UNEXPECTED_ERROR, ex.what()); } return SERVER_SUCCESS; @@ -288,10 +265,7 @@ ServerError DeleteTableTask::OnExecute() { //step 1: check arguments if (table_name_.empty()) { - error_code_ = SERVER_INVALID_ARGUMENT; - error_msg_ = "Table name cannot be empty"; - SERVER_LOG_ERROR << error_msg_; - return error_code_; + return SetError(SERVER_INVALID_TABLE_NAME, "Empty table name"); } //step 2: check table existence @@ -299,10 +273,11 @@ ServerError DeleteTableTask::OnExecute() { table_info.table_id_ = table_name_; engine::Status stat = DBWrapper::DB()->DescribeTable(table_info); if(!stat.ok()) { - error_code_ = SERVER_TABLE_NOT_EXIST; - error_msg_ = "Engine failed: " + stat.ToString(); - SERVER_LOG_ERROR << error_msg_; - return error_code_; + if(stat.IsNotFound()) { + return SetError(SERVER_TABLE_NOT_EXIST, "Table " + table_name_ + " not exists"); + } else { + return SetError(DB_META_TRANSACTION_FAILED, "Engine failed: " + stat.ToString()); + } } rc.Record("check validation"); @@ -311,17 +286,13 @@ ServerError DeleteTableTask::OnExecute() { std::vector dates; stat = DBWrapper::DB()->DeleteTable(table_name_, dates); if(!stat.ok()) { - SERVER_LOG_ERROR << "Engine failed: " << stat.ToString(); - return SERVER_UNEXPECTED_ERROR; + return SetError(DB_META_TRANSACTION_FAILED, "Engine failed: " + stat.ToString()); } rc.Record("deleta table"); rc.Elapse("totally cost"); } catch (std::exception& ex) { - error_code_ = SERVER_UNEXPECTED_ERROR; - error_msg_ = ex.what(); - SERVER_LOG_ERROR << error_msg_; - return error_code_; + return SetError(SERVER_UNEXPECTED_ERROR, ex.what()); } return SERVER_SUCCESS; @@ -342,10 +313,7 @@ ServerError ShowTablesTask::OnExecute() { std::vector schema_array; engine::Status stat = DBWrapper::DB()->AllTables(schema_array); if(!stat.ok()) { - error_code_ = SERVER_UNEXPECTED_ERROR; - error_msg_ = "Engine failed: " + stat.ToString(); - SERVER_LOG_ERROR << error_msg_; - return error_code_; + return SetError(DB_META_TRANSACTION_FAILED, "Engine failed: " + stat.ToString()); } tables_.clear(); @@ -379,17 +347,11 @@ ServerError AddVectorTask::OnExecute() { //step 1: check arguments if (table_name_.empty()) { - error_code_ = SERVER_INVALID_ARGUMENT; - error_msg_ = "Table name cannot be empty"; - SERVER_LOG_ERROR << error_msg_; - return error_code_; + return SetError(SERVER_INVALID_TABLE_NAME, "Empty table name"); } if(record_array_.empty()) { - error_code_ = SERVER_INVALID_ARGUMENT; - error_msg_ = "Row record array is empty"; - SERVER_LOG_ERROR << error_msg_; - return error_code_; + return SetError(SERVER_INVALID_ROWRECORD_ARRAY, "Row record array is empty"); } //step 2: check table existence @@ -397,20 +359,22 @@ ServerError AddVectorTask::OnExecute() { table_info.table_id_ = table_name_; engine::Status stat = DBWrapper::DB()->DescribeTable(table_info); if(!stat.ok()) { - error_code_ = SERVER_TABLE_NOT_EXIST; - error_msg_ = "Engine failed: " + stat.ToString(); - SERVER_LOG_ERROR << error_msg_; - return error_code_; + if(stat.IsNotFound()) { + return SetError(SERVER_TABLE_NOT_EXIST, "Table " + table_name_ + " not exists"); + } else { + return SetError(DB_META_TRANSACTION_FAILED, "Engine failed: " + stat.ToString()); + } } rc.Record("check validation"); //step 3: prepare float data std::vector vec_f; - error_code_ = ConvertRowRecordToFloatArray(record_array_, table_info.dimension_, vec_f); - if(error_code_ != SERVER_SUCCESS) { - error_msg_ = "Invalid row record data"; - return error_code_; + ServerError error_code = SERVER_SUCCESS; + std::string error_msg; + ConvertRowRecordToFloatArray(record_array_, table_info.dimension_, vec_f, error_code, error_msg); + if(error_code != SERVER_SUCCESS) { + return SetError(error_code, error_msg); } rc.Record("prepare vectors data"); @@ -420,25 +384,20 @@ ServerError AddVectorTask::OnExecute() { stat = DBWrapper::DB()->InsertVectors(table_name_, vec_count, vec_f.data(), record_ids_); rc.Record("add vectors to engine"); if(!stat.ok()) { - error_code_ = SERVER_UNEXPECTED_ERROR; - error_msg_ = "Engine failed: " + stat.ToString(); - SERVER_LOG_ERROR << error_msg_; - return error_code_; + return SetError(SERVER_CACHE_ERROR, "Cache error: " + stat.ToString()); } if(record_ids_.size() != vec_count) { - SERVER_LOG_ERROR << "Vector ID not returned"; - return SERVER_UNEXPECTED_ERROR; + std::string msg = "Add " + std::to_string(vec_count) + " vectors but only return " + + std::to_string(record_ids_.size()) + " id"; + return SetError(SERVER_ILLEGAL_VECTOR_ID, msg); } rc.Record("do insert"); rc.Elapse("totally cost"); } catch (std::exception& ex) { - error_code_ = SERVER_UNEXPECTED_ERROR; - error_msg_ = ex.what(); - SERVER_LOG_ERROR << error_msg_; - return error_code_; + return SetError(SERVER_UNEXPECTED_ERROR, ex.what()); } return SERVER_SUCCESS; @@ -477,17 +436,14 @@ ServerError SearchVectorTask::OnExecute() { //step 1: check arguments if (table_name_.empty()) { - error_code_ = SERVER_INVALID_ARGUMENT; - error_msg_ = "Table name cannot be empty"; - SERVER_LOG_ERROR << error_msg_; - return error_code_; + return SetError(SERVER_INVALID_TABLE_NAME, "Empty table name"); } - if(top_k_ <= 0 || record_array_.empty()) { - error_code_ = SERVER_INVALID_ARGUMENT; - error_msg_ = "Invalid topk value, or query record array is empty"; - SERVER_LOG_ERROR << error_msg_; - return error_code_; + if(top_k_ <= 0) { + return SetError(SERVER_INVALID_TOPK, "Invalid topk: " + std::to_string(top_k_)); + } + if(record_array_.empty()) { + return SetError(SERVER_INVALID_ROWRECORD_ARRAY, "Row record array is empty"); } //step 2: check table existence @@ -495,28 +451,29 @@ ServerError SearchVectorTask::OnExecute() { table_info.table_id_ = table_name_; engine::Status stat = DBWrapper::DB()->DescribeTable(table_info); if(!stat.ok()) { - error_code_ = SERVER_TABLE_NOT_EXIST; - error_msg_ = "Engine failed: " + stat.ToString(); - SERVER_LOG_ERROR << error_msg_; - return error_code_; + if(stat.IsNotFound()) { + return SetError(SERVER_TABLE_NOT_EXIST, "Table " + table_name_ + " not exists"); + } else { + return SetError(DB_META_TRANSACTION_FAILED, "Engine failed: " + stat.ToString()); + } } //step 3: check date range, and convert to db dates std::vector dates; - error_code_ = ConvertTimeRangeToDBDates(range_array_, dates); - if(error_code_ != SERVER_SUCCESS) { - error_msg_ = "Invalid query range"; - return error_code_; + ServerError error_code = SERVER_SUCCESS; + std::string error_msg; + ConvertTimeRangeToDBDates(range_array_, dates, error_code, error_msg); + if(error_code != SERVER_SUCCESS) { + return SetError(error_code, error_msg); } rc.Record("check validation"); //step 3: prepare float data std::vector vec_f; - error_code_ = ConvertRowRecordToFloatArray(record_array_, table_info.dimension_, vec_f); - if(error_code_ != SERVER_SUCCESS) { - error_msg_ = "Invalid row record data"; - return error_code_; + ConvertRowRecordToFloatArray(record_array_, table_info.dimension_, vec_f, error_code, error_msg); + if(error_code != SERVER_SUCCESS) { + return SetError(error_code, error_msg); } rc.Record("prepare vector data"); @@ -533,13 +490,17 @@ ServerError SearchVectorTask::OnExecute() { rc.Record("search vectors from engine"); if(!stat.ok()) { - SERVER_LOG_ERROR << "Engine failed: " << stat.ToString(); - return SERVER_UNEXPECTED_ERROR; + return SetError(DB_META_TRANSACTION_FAILED, "Engine failed: " + stat.ToString()); + } + + if(results.empty()) { + return SERVER_SUCCESS; //empty table } if(results.size() != record_count) { - SERVER_LOG_ERROR << "Search result not returned"; - return SERVER_UNEXPECTED_ERROR; + std::string msg = "Search " + std::to_string(record_count) + " vectors but only return " + + std::to_string(results.size()) + " results"; + return SetError(SERVER_ILLEGAL_SEARCH_RESULT, msg); } rc.Record("do search"); @@ -564,10 +525,7 @@ ServerError SearchVectorTask::OnExecute() { rc.Elapse("totally cost"); } catch (std::exception& ex) { - error_code_ = SERVER_UNEXPECTED_ERROR; - error_msg_ = ex.what(); - SERVER_LOG_ERROR << error_msg_; - return error_code_; + return SetError(SERVER_UNEXPECTED_ERROR, ex.what()); } return SERVER_SUCCESS; @@ -591,20 +549,14 @@ ServerError GetTableRowCountTask::OnExecute() { //step 1: check arguments if (table_name_.empty()) { - error_code_ = SERVER_INVALID_ARGUMENT; - error_msg_ = "Table name cannot be empty"; - SERVER_LOG_ERROR << error_msg_; - return error_code_; + return SetError(SERVER_INVALID_TABLE_NAME, "Empty table name"); } //step 2: get row count uint64_t row_count = 0; engine::Status stat = DBWrapper::DB()->GetTableRowCount(table_name_, row_count); if (!stat.ok()) { - error_code_ = SERVER_UNEXPECTED_ERROR; - error_msg_ = "Engine failed: " + stat.ToString(); - SERVER_LOG_ERROR << error_msg_; - return error_code_; + return SetError(DB_META_TRANSACTION_FAILED, "Engine failed: " + stat.ToString()); } row_count_ = (int64_t) row_count; @@ -612,10 +564,7 @@ ServerError GetTableRowCountTask::OnExecute() { rc.Elapse("totally cost"); } catch (std::exception& ex) { - error_code_ = SERVER_UNEXPECTED_ERROR; - error_msg_ = ex.what(); - SERVER_LOG_ERROR << error_msg_; - return error_code_; + return SetError(SERVER_UNEXPECTED_ERROR, ex.what()); } return SERVER_SUCCESS; diff --git a/cpp/src/server/RequestTask.h b/cpp/src/server/RequestTask.h index 3061b3b75d..4bf9f964e0 100644 --- a/cpp/src/server/RequestTask.h +++ b/cpp/src/server/RequestTask.h @@ -7,7 +7,6 @@ #include "RequestScheduler.h" #include "utils/Error.h" -#include "utils/AttributeSerializer.h" #include "db/Types.h" #include "milvus_types.h" diff --git a/cpp/src/storage/s3/S3ClientWrapper.cpp b/cpp/src/storage/s3/S3ClientWrapper.cpp index fcbab4f9bd..b739bc1e28 100644 --- a/cpp/src/storage/s3/S3ClientWrapper.cpp +++ b/cpp/src/storage/s3/S3ClientWrapper.cpp @@ -1,159 +1,159 @@ -#include "S3ClientWrapper.h" - -#include -#include -#include -#include -#include - -#include -#include - - -namespace zilliz { -namespace milvus { -namespace engine { -namespace storage { - -Status -S3ClientWrapper::Create(const std::string &ip_address, - const std::string &port, - const std::string &access_key, - const std::string &secret_key) { - Aws::InitAPI(options_); - Aws::Client::ClientConfiguration cfg; - - // TODO: ip_address need to be validated. - - cfg.endpointOverride = ip_address + ":" + port; // S3 server ip address and port - cfg.scheme = Aws::Http::Scheme::HTTP; - cfg.verifySSL = - false; //Aws::Auth::AWSCredentials cred("RPW421T9GSIO4A45Y9ZR", "2owKYy9emSS90Q0pXuyqpX1OxBCyEDYodsiBemcq"); // 认证的Key - client_ = - new S3Client(Aws::Auth::AWSCredentials(access_key, secret_key), - cfg, - Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Always, - false); - if (client_ == nullptr) { - std::string error = "Can't connect server."; - return Status::Error(error); - } else { - return Status::OK(); - } -} - - -Status -S3ClientWrapper::Close() { - if (client_ != nullptr) { - delete client_; - client_ = nullptr; - } - Aws::ShutdownAPI(options_); - return Status::OK(); -} - -Status -S3ClientWrapper::CreateBucket(std::string& bucket_name) { - Aws::S3::Model::CreateBucketRequest request; - request.SetBucket(bucket_name); - - auto outcome = client_->CreateBucket(request); - - if (outcome.IsSuccess()) - { - return Status::OK(); - } - else - { - std::cout << "CreateBucket error: " - << outcome.GetError().GetExceptionName() << std::endl - << outcome.GetError().GetMessage() << std::endl; - switch(outcome.GetError().GetErrorType()) { - case Aws::S3::S3Errors::BUCKET_ALREADY_EXISTS: - case Aws::S3::S3Errors::BUCKET_ALREADY_OWNED_BY_YOU: - return Status::AlreadyExist(outcome.GetError().GetMessage()); - default: - return Status::Error(outcome.GetError().GetMessage()); - } - } -} - -Status -S3ClientWrapper::DeleteBucket(std::string& bucket_name) { - Aws::S3::Model::DeleteBucketRequest bucket_request; - bucket_request.SetBucket(bucket_name); - - auto outcome = client_->DeleteBucket(bucket_request); - - if (outcome.IsSuccess()) - { - return Status::OK(); - } - else - { - std::cout << "DeleteBucket error: " - << outcome.GetError().GetExceptionName() << " - " - << outcome.GetError().GetMessage() << std::endl; - return Status::Error(outcome.GetError().GetMessage()); - } -} - -Status -S3ClientWrapper::UploadFile(std::string &BucketName, std::string &objectKey, std::string &pathkey) { - - PutObjectRequest putObjectRequest; - putObjectRequest.WithBucket(BucketName.c_str()).WithKey(objectKey.c_str()); - - auto input_data = Aws::MakeShared("PutObjectInputStream", - pathkey.c_str(), - std::ios_base::in | std::ios_base::binary); - putObjectRequest.SetBody(input_data); - auto put_object_result = client_->PutObject(putObjectRequest); - if (put_object_result.IsSuccess()) { - return Status::OK(); - } else { - std::cout << "PutObject error: " << put_object_result.GetError().GetExceptionName() << " " - << put_object_result.GetError().GetMessage() << std::endl; - return Status::Error(put_object_result.GetError().GetMessage()); - } -} - -Status -S3ClientWrapper::DownloadFile(std::string &BucketName, std::string &objectKey, std::string &pathkey) { - GetObjectRequest object_request; - object_request.WithBucket(BucketName.c_str()).WithKey(objectKey.c_str()); - auto get_object_outcome = client_->GetObject(object_request); - if (get_object_outcome.IsSuccess()) { - Aws::OFStream local_file(pathkey.c_str(), std::ios::out | std::ios::binary); - local_file << get_object_outcome.GetResult().GetBody().rdbuf(); - return Status::OK(); - } else { - std::cout << "GetObject error: " << get_object_outcome.GetError().GetExceptionName() << " " - << get_object_outcome.GetError().GetMessage() << std::endl; - return Status::Error(get_object_outcome.GetError().GetMessage()); - } -} - -Status -S3ClientWrapper::DeleteFile(std::string &bucket_name, std::string &object_key) { - Aws::S3::Model::DeleteObjectRequest object_request; - object_request.WithBucket(bucket_name).WithKey(object_key); - - auto delete_object_outcome = client_->DeleteObject(object_request); - - if (delete_object_outcome.IsSuccess()) { - return Status::OK(); - } else { - std::cout << "DeleteObject error: " << - delete_object_outcome.GetError().GetExceptionName() << " " << - delete_object_outcome.GetError().GetMessage() << std::endl; - - return Status::Error(delete_object_outcome.GetError().GetMessage()); - } -} - -} -} -} -} \ No newline at end of file +//#include "S3ClientWrapper.h" +// +//#include +//#include +//#include +//#include +//#include +// +//#include +//#include +// +// +//namespace zilliz { +//namespace milvus { +//namespace engine { +//namespace storage { +// +//Status +//S3ClientWrapper::Create(const std::string &ip_address, +// const std::string &port, +// const std::string &access_key, +// const std::string &secret_key) { +// Aws::InitAPI(options_); +// Aws::Client::ClientConfiguration cfg; +// +// // TODO: ip_address need to be validated. +// +// cfg.endpointOverride = ip_address + ":" + port; // S3 server ip address and port +// cfg.scheme = Aws::Http::Scheme::HTTP; +// cfg.verifySSL = +// false; //Aws::Auth::AWSCredentials cred("RPW421T9GSIO4A45Y9ZR", "2owKYy9emSS90Q0pXuyqpX1OxBCyEDYodsiBemcq"); // 认证的Key +// client_ = +// new S3Client(Aws::Auth::AWSCredentials(access_key, secret_key), +// cfg, +// Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Always, +// false); +// if (client_ == nullptr) { +// std::string error = "Can't connect server."; +// return Status::Error(error); +// } else { +// return Status::OK(); +// } +//} +// +// +//Status +//S3ClientWrapper::Close() { +// if (client_ != nullptr) { +// delete client_; +// client_ = nullptr; +// } +// Aws::ShutdownAPI(options_); +// return Status::OK(); +//} +// +//Status +//S3ClientWrapper::CreateBucket(std::string& bucket_name) { +// Aws::S3::Model::CreateBucketRequest request; +// request.SetBucket(bucket_name); +// +// auto outcome = client_->CreateBucket(request); +// +// if (outcome.IsSuccess()) +// { +// return Status::OK(); +// } +// else +// { +// std::cout << "CreateBucket error: " +// << outcome.GetError().GetExceptionName() << std::endl +// << outcome.GetError().GetMessage() << std::endl; +// switch(outcome.GetError().GetErrorType()) { +// case Aws::S3::S3Errors::BUCKET_ALREADY_EXISTS: +// case Aws::S3::S3Errors::BUCKET_ALREADY_OWNED_BY_YOU: +// return Status::AlreadyExist(outcome.GetError().GetMessage()); +// default: +// return Status::Error(outcome.GetError().GetMessage()); +// } +// } +//} +// +//Status +//S3ClientWrapper::DeleteBucket(std::string& bucket_name) { +// Aws::S3::Model::DeleteBucketRequest bucket_request; +// bucket_request.SetBucket(bucket_name); +// +// auto outcome = client_->DeleteBucket(bucket_request); +// +// if (outcome.IsSuccess()) +// { +// return Status::OK(); +// } +// else +// { +// std::cout << "DeleteBucket error: " +// << outcome.GetError().GetExceptionName() << " - " +// << outcome.GetError().GetMessage() << std::endl; +// return Status::Error(outcome.GetError().GetMessage()); +// } +//} +// +//Status +//S3ClientWrapper::UploadFile(std::string &BucketName, std::string &objectKey, std::string &pathkey) { +// +// PutObjectRequest putObjectRequest; +// putObjectRequest.WithBucket(BucketName.c_str()).WithKey(objectKey.c_str()); +// +// auto input_data = Aws::MakeShared("PutObjectInputStream", +// pathkey.c_str(), +// std::ios_base::in | std::ios_base::binary); +// putObjectRequest.SetBody(input_data); +// auto put_object_result = client_->PutObject(putObjectRequest); +// if (put_object_result.IsSuccess()) { +// return Status::OK(); +// } else { +// std::cout << "PutObject error: " << put_object_result.GetError().GetExceptionName() << " " +// << put_object_result.GetError().GetMessage() << std::endl; +// return Status::Error(put_object_result.GetError().GetMessage()); +// } +//} +// +//Status +//S3ClientWrapper::DownloadFile(std::string &BucketName, std::string &objectKey, std::string &pathkey) { +// GetObjectRequest object_request; +// object_request.WithBucket(BucketName.c_str()).WithKey(objectKey.c_str()); +// auto get_object_outcome = client_->GetObject(object_request); +// if (get_object_outcome.IsSuccess()) { +// Aws::OFStream local_file(pathkey.c_str(), std::ios::out | std::ios::binary); +// local_file << get_object_outcome.GetResult().GetBody().rdbuf(); +// return Status::OK(); +// } else { +// std::cout << "GetObject error: " << get_object_outcome.GetError().GetExceptionName() << " " +// << get_object_outcome.GetError().GetMessage() << std::endl; +// return Status::Error(get_object_outcome.GetError().GetMessage()); +// } +//} +// +//Status +//S3ClientWrapper::DeleteFile(std::string &bucket_name, std::string &object_key) { +// Aws::S3::Model::DeleteObjectRequest object_request; +// object_request.WithBucket(bucket_name).WithKey(object_key); +// +// auto delete_object_outcome = client_->DeleteObject(object_request); +// +// if (delete_object_outcome.IsSuccess()) { +// return Status::OK(); +// } else { +// std::cout << "DeleteObject error: " << +// delete_object_outcome.GetError().GetExceptionName() << " " << +// delete_object_outcome.GetError().GetMessage() << std::endl; +// +// return Status::Error(delete_object_outcome.GetError().GetMessage()); +// } +//} +// +//} +//} +//} +//} \ No newline at end of file diff --git a/cpp/src/storage/s3/S3ClientWrapper.h b/cpp/src/storage/s3/S3ClientWrapper.h index 9ef60174cf..fc66304a5a 100644 --- a/cpp/src/storage/s3/S3ClientWrapper.h +++ b/cpp/src/storage/s3/S3ClientWrapper.h @@ -1,45 +1,45 @@ -#pragma once - -#include "storage/IStorage.h" - - -#include -#include -#include - - -using namespace Aws::S3; -using namespace Aws::S3::Model; - -namespace zilliz { -namespace milvus { -namespace engine { -namespace storage { - -class S3ClientWrapper : public IStorage { - public: - - S3ClientWrapper() = default; - ~S3ClientWrapper() = default; - - Status Create(const std::string &ip_address, - const std::string &port, - const std::string &access_key, - const std::string &secret_key) override; - Status Close() override; - - Status CreateBucket(std::string& bucket_name) override; - Status DeleteBucket(std::string& bucket_name) override; - Status UploadFile(std::string &BucketName, std::string &objectKey, std::string &pathkey) override; - Status DownloadFile(std::string &BucketName, std::string &objectKey, std::string &pathkey) override; - Status DeleteFile(std::string &bucket_name, std::string &object_key) override; - - private: - S3Client *client_ = nullptr; - Aws::SDKOptions options_; -}; - -} -} -} -} \ No newline at end of file +//#pragma once +// +//#include "storage/IStorage.h" +// +// +//#include +//#include +//#include +// +// +//using namespace Aws::S3; +//using namespace Aws::S3::Model; +// +//namespace zilliz { +//namespace milvus { +//namespace engine { +//namespace storage { +// +//class S3ClientWrapper : public IStorage { +// public: +// +// S3ClientWrapper() = default; +// ~S3ClientWrapper() = default; +// +// Status Create(const std::string &ip_address, +// const std::string &port, +// const std::string &access_key, +// const std::string &secret_key) override; +// Status Close() override; +// +// Status CreateBucket(std::string& bucket_name) override; +// Status DeleteBucket(std::string& bucket_name) override; +// Status UploadFile(std::string &BucketName, std::string &objectKey, std::string &pathkey) override; +// Status DownloadFile(std::string &BucketName, std::string &objectKey, std::string &pathkey) override; +// Status DeleteFile(std::string &bucket_name, std::string &object_key) override; +// +// private: +// S3Client *client_ = nullptr; +// Aws::SDKOptions options_; +//}; +// +//} +//} +//} +//} \ No newline at end of file diff --git a/cpp/src/thrift/gen-cpp/milvus_types.cpp b/cpp/src/thrift/gen-cpp/milvus_types.cpp index 0b87156bb3..09aedec0bf 100644 --- a/cpp/src/thrift/gen-cpp/milvus_types.cpp +++ b/cpp/src/thrift/gen-cpp/milvus_types.cpp @@ -15,23 +15,51 @@ namespace milvus { namespace thrift { int _kErrorCodeValues[] = { ErrorCode::SUCCESS, + ErrorCode::UNEXPECTED_ERROR, ErrorCode::CONNECT_FAILED, ErrorCode::PERMISSION_DENIED, ErrorCode::TABLE_NOT_EXISTS, ErrorCode::ILLEGAL_ARGUMENT, ErrorCode::ILLEGAL_RANGE, - ErrorCode::ILLEGAL_DIMENSION + ErrorCode::ILLEGAL_DIMENSION, + ErrorCode::ILLEGAL_INDEX_TYPE, + ErrorCode::ILLEGAL_TABLE_NAME, + ErrorCode::ILLEGAL_TOPK, + ErrorCode::ILLEGAL_ROWRECORD, + ErrorCode::ILLEGAL_VECTOR_ID, + ErrorCode::ILLEGAL_SEARCH_RESULT, + ErrorCode::FILE_NOT_FOUND, + ErrorCode::META_FAILED, + ErrorCode::CACHE_FAILED, + ErrorCode::CANNOT_CREATE_FOLDER, + ErrorCode::CANNOT_CREATE_FILE, + ErrorCode::CANNOT_DELETE_FOLDER, + ErrorCode::CANNOT_DELETE_FILE }; const char* _kErrorCodeNames[] = { "SUCCESS", + "UNEXPECTED_ERROR", "CONNECT_FAILED", "PERMISSION_DENIED", "TABLE_NOT_EXISTS", "ILLEGAL_ARGUMENT", "ILLEGAL_RANGE", - "ILLEGAL_DIMENSION" + "ILLEGAL_DIMENSION", + "ILLEGAL_INDEX_TYPE", + "ILLEGAL_TABLE_NAME", + "ILLEGAL_TOPK", + "ILLEGAL_ROWRECORD", + "ILLEGAL_VECTOR_ID", + "ILLEGAL_SEARCH_RESULT", + "FILE_NOT_FOUND", + "META_FAILED", + "CACHE_FAILED", + "CANNOT_CREATE_FOLDER", + "CANNOT_CREATE_FILE", + "CANNOT_DELETE_FOLDER", + "CANNOT_DELETE_FILE" }; -const std::map _ErrorCode_VALUES_TO_NAMES(::apache::thrift::TEnumIterator(7, _kErrorCodeValues, _kErrorCodeNames), ::apache::thrift::TEnumIterator(-1, NULL, NULL)); +const std::map _ErrorCode_VALUES_TO_NAMES(::apache::thrift::TEnumIterator(21, _kErrorCodeValues, _kErrorCodeNames), ::apache::thrift::TEnumIterator(-1, NULL, NULL)); std::ostream& operator<<(std::ostream& out, const ErrorCode::type& val) { std::map::const_iterator it = _ErrorCode_VALUES_TO_NAMES.find(val); diff --git a/cpp/src/thrift/gen-cpp/milvus_types.h b/cpp/src/thrift/gen-cpp/milvus_types.h index 0a7db0f5ca..34c5e084ce 100644 --- a/cpp/src/thrift/gen-cpp/milvus_types.h +++ b/cpp/src/thrift/gen-cpp/milvus_types.h @@ -23,12 +23,26 @@ namespace milvus { namespace thrift { struct ErrorCode { enum type { SUCCESS = 0, - CONNECT_FAILED = 1, - PERMISSION_DENIED = 2, - TABLE_NOT_EXISTS = 3, - ILLEGAL_ARGUMENT = 4, - ILLEGAL_RANGE = 5, - ILLEGAL_DIMENSION = 6 + UNEXPECTED_ERROR = 1, + CONNECT_FAILED = 2, + PERMISSION_DENIED = 3, + TABLE_NOT_EXISTS = 4, + ILLEGAL_ARGUMENT = 5, + ILLEGAL_RANGE = 6, + ILLEGAL_DIMENSION = 7, + ILLEGAL_INDEX_TYPE = 8, + ILLEGAL_TABLE_NAME = 9, + ILLEGAL_TOPK = 10, + ILLEGAL_ROWRECORD = 11, + ILLEGAL_VECTOR_ID = 12, + ILLEGAL_SEARCH_RESULT = 13, + FILE_NOT_FOUND = 14, + META_FAILED = 15, + CACHE_FAILED = 16, + CANNOT_CREATE_FOLDER = 17, + CANNOT_CREATE_FILE = 18, + CANNOT_DELETE_FOLDER = 19, + CANNOT_DELETE_FILE = 20 }; }; diff --git a/cpp/src/thrift/milvus.thrift b/cpp/src/thrift/milvus.thrift index d72c77884d..813026fdd8 100644 --- a/cpp/src/thrift/milvus.thrift +++ b/cpp/src/thrift/milvus.thrift @@ -15,12 +15,26 @@ namespace netcore milvus.thrift enum ErrorCode { SUCCESS = 0, + UNEXPECTED_ERROR, CONNECT_FAILED, PERMISSION_DENIED, TABLE_NOT_EXISTS, ILLEGAL_ARGUMENT, ILLEGAL_RANGE, ILLEGAL_DIMENSION, + ILLEGAL_INDEX_TYPE, + ILLEGAL_TABLE_NAME, + ILLEGAL_TOPK, + ILLEGAL_ROWRECORD, + ILLEGAL_VECTOR_ID, + ILLEGAL_SEARCH_RESULT, + FILE_NOT_FOUND, + META_FAILED, + CACHE_FAILED, + CANNOT_CREATE_FOLDER, + CANNOT_CREATE_FILE, + CANNOT_DELETE_FOLDER, + CANNOT_DELETE_FILE, } exception Exception { diff --git a/cpp/src/utils/AttributeSerializer.cpp b/cpp/src/utils/AttributeSerializer.cpp deleted file mode 100644 index b351262e60..0000000000 --- a/cpp/src/utils/AttributeSerializer.cpp +++ /dev/null @@ -1,50 +0,0 @@ -/******************************************************************************* - * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved - * Unauthorized copying of this file, via any medium is strictly prohibited. - * Proprietary and confidential. - ******************************************************************************/ - -#include "AttributeSerializer.h" -#include "StringHelpFunctions.h" - -namespace zilliz { -namespace milvus { -namespace server { - - -ServerError AttributeSerializer::Encode(const AttribMap& attrib_map, std::string& attrib_str) { - attrib_str = ""; - for(auto iter : attrib_map) { - attrib_str += iter.first; - attrib_str += ":\""; - attrib_str += iter.second; - attrib_str += "\";"; - } - - return SERVER_SUCCESS; -} - -ServerError AttributeSerializer::Decode(const std::string& attrib_str, AttribMap& attrib_map) { - attrib_map.clear(); - - std::vector kv_pairs; - StringHelpFunctions::SplitStringByQuote(attrib_str, ";", "\"", kv_pairs); - for(std::string& str : kv_pairs) { - std::string key, val; - size_t index = str.find_first_of(":", 0); - if (index != std::string::npos) { - key = str.substr(0, index); - val = str.substr(index + 1); - } else { - key = str; - } - - attrib_map.insert(std::make_pair(key, val)); - } - - return SERVER_SUCCESS; -} - -} -} -} diff --git a/cpp/src/utils/AttributeSerializer.h b/cpp/src/utils/AttributeSerializer.h deleted file mode 100644 index 8d0341aa2e..0000000000 --- a/cpp/src/utils/AttributeSerializer.h +++ /dev/null @@ -1,27 +0,0 @@ -/******************************************************************************* - * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved - * Unauthorized copying of this file, via any medium is strictly prohibited. - * Proprietary and confidential. - ******************************************************************************/ -#pragma once - -#include - -#include "Error.h" - -namespace zilliz { -namespace milvus { -namespace server { - -using AttribMap = std::map; - -class AttributeSerializer { -public: - static ServerError Encode(const AttribMap& attrib_map, std::string& attrib_str); - static ServerError Decode(const std::string& attrib_str, AttribMap& attrib_map); -}; - - -} -} -} diff --git a/cpp/src/utils/CommonUtil.cpp b/cpp/src/utils/CommonUtil.cpp index 0583150d86..19157af9f2 100644 --- a/cpp/src/utils/CommonUtil.cpp +++ b/cpp/src/utils/CommonUtil.cpp @@ -51,7 +51,7 @@ bool CommonUtil::GetSystemAvailableThreads(unsigned int &threadCnt) { return true; } -bool CommonUtil::IsDirectoryExit(const std::string &path) +bool CommonUtil::IsDirectoryExist(const std::string &path) { DIR *dp = nullptr; if ((dp = opendir(path.c_str())) == nullptr) { @@ -182,7 +182,7 @@ void CommonUtil::ConvertTime(time_t time_integer, tm &time_struct) { memcpy(&time_struct, t_m, sizeof(tm)); } -void ConvertTime(tm time_struct, time_t &time_integer) { +void CommonUtil::ConvertTime(tm time_struct, time_t &time_integer) { time_integer = mktime(&time_struct); } diff --git a/cpp/src/utils/CommonUtil.h b/cpp/src/utils/CommonUtil.h index 4de87badbc..f1db3c059d 100755 --- a/cpp/src/utils/CommonUtil.h +++ b/cpp/src/utils/CommonUtil.h @@ -20,7 +20,7 @@ class CommonUtil { static bool GetSystemAvailableThreads(unsigned int &threadCnt); static bool IsFileExist(const std::string &path); - static bool IsDirectoryExit(const std::string &path); + static bool IsDirectoryExist(const std::string &path); static ServerError CreateDirectory(const std::string &path); static ServerError DeleteDirectory(const std::string &path); diff --git a/cpp/src/utils/Error.h b/cpp/src/utils/Error.h index bfb19f47a9..8c4da70339 100644 --- a/cpp/src/utils/Error.h +++ b/cpp/src/utils/Error.h @@ -24,18 +24,35 @@ ToGlobalServerErrorCode(const ServerError error_code) { return SERVER_ERROR_CODE_BASE + error_code; } -constexpr ServerError SERVER_UNEXPECTED_ERROR = ToGlobalServerErrorCode(0x001); -constexpr ServerError SERVER_UNSUPPORTED_ERROR = ToGlobalServerErrorCode(0x002); -constexpr ServerError SERVER_NULL_POINTER = ToGlobalServerErrorCode(0x003); -constexpr ServerError SERVER_INVALID_ARGUMENT = ToGlobalServerErrorCode(0x004); -constexpr ServerError SERVER_FILE_NOT_FOUND = ToGlobalServerErrorCode(0x005); -constexpr ServerError SERVER_NOT_IMPLEMENT = ToGlobalServerErrorCode(0x006); -constexpr ServerError SERVER_BLOCKING_QUEUE_EMPTY = ToGlobalServerErrorCode(0x007); -constexpr ServerError SERVER_TABLE_NOT_EXIST = ToGlobalServerErrorCode(0x008); -constexpr ServerError SERVER_INVALID_TIME_RANGE = ToGlobalServerErrorCode(0x009); -constexpr ServerError SERVER_INVALID_VECTOR_DIMENSION = ToGlobalServerErrorCode(0x00a); -constexpr ServerError SERVER_LICENSE_VALIDATION_FAIL = ToGlobalServerErrorCode(0x00b); -constexpr ServerError SERVER_LICENSE_FILE_NOT_EXIST = ToGlobalServerErrorCode(0x00c); +constexpr ServerError SERVER_UNEXPECTED_ERROR = ToGlobalServerErrorCode(1); +constexpr ServerError SERVER_UNSUPPORTED_ERROR = ToGlobalServerErrorCode(2); +constexpr ServerError SERVER_NULL_POINTER = ToGlobalServerErrorCode(3); +constexpr ServerError SERVER_INVALID_ARGUMENT = ToGlobalServerErrorCode(4); +constexpr ServerError SERVER_FILE_NOT_FOUND = ToGlobalServerErrorCode(5); +constexpr ServerError SERVER_NOT_IMPLEMENT = ToGlobalServerErrorCode(6); +constexpr ServerError SERVER_BLOCKING_QUEUE_EMPTY = ToGlobalServerErrorCode(7); +constexpr ServerError SERVER_CANNOT_CREATE_FOLDER = ToGlobalServerErrorCode(8); +constexpr ServerError SERVER_CANNOT_CREATE_FILE = ToGlobalServerErrorCode(9); +constexpr ServerError SERVER_CANNOT_DELETE_FOLDER = ToGlobalServerErrorCode(10); +constexpr ServerError SERVER_CANNOT_DELETE_FILE = ToGlobalServerErrorCode(11); + +constexpr ServerError SERVER_TABLE_NOT_EXIST = ToGlobalServerErrorCode(100); +constexpr ServerError SERVER_INVALID_TABLE_NAME = ToGlobalServerErrorCode(101); +constexpr ServerError SERVER_INVALID_TABLE_DIMENSION = ToGlobalServerErrorCode(102); +constexpr ServerError SERVER_INVALID_TIME_RANGE = ToGlobalServerErrorCode(103); +constexpr ServerError SERVER_INVALID_VECTOR_DIMENSION = ToGlobalServerErrorCode(104); +constexpr ServerError SERVER_INVALID_INDEX_TYPE = ToGlobalServerErrorCode(105); +constexpr ServerError SERVER_INVALID_ROWRECORD = ToGlobalServerErrorCode(106); +constexpr ServerError SERVER_INVALID_ROWRECORD_ARRAY = ToGlobalServerErrorCode(107); +constexpr ServerError SERVER_INVALID_TOPK = ToGlobalServerErrorCode(108); +constexpr ServerError SERVER_ILLEGAL_VECTOR_ID = ToGlobalServerErrorCode(109); +constexpr ServerError SERVER_ILLEGAL_SEARCH_RESULT = ToGlobalServerErrorCode(110); +constexpr ServerError SERVER_CACHE_ERROR = ToGlobalServerErrorCode(111); + +constexpr ServerError SERVER_LICENSE_FILE_NOT_EXIST = ToGlobalServerErrorCode(500); +constexpr ServerError SERVER_LICENSE_VALIDATION_FAIL = ToGlobalServerErrorCode(501); + +constexpr ServerError DB_META_TRANSACTION_FAILED = ToGlobalServerErrorCode(1000); class ServerException : public std::exception { public: diff --git a/cpp/src/utils/StringHelpFunctions.cpp b/cpp/src/utils/StringHelpFunctions.cpp index aaefd7236a..36ba017beb 100644 --- a/cpp/src/utils/StringHelpFunctions.cpp +++ b/cpp/src/utils/StringHelpFunctions.cpp @@ -9,13 +9,6 @@ namespace zilliz { namespace milvus { namespace server { -void StringHelpFunctions::TrimStringLineBreak(std::string &string) { - if (!string.empty()) { - static std::string s_format("\n\r"); - string.erase(string.find_last_not_of(s_format) + 1); - } -} - void StringHelpFunctions::TrimStringBlank(std::string &string) { if (!string.empty()) { static std::string s_format(" \n\r\t"); diff --git a/cpp/src/utils/StringHelpFunctions.h b/cpp/src/utils/StringHelpFunctions.h index 2521c48e22..b5d8ce22b7 100644 --- a/cpp/src/utils/StringHelpFunctions.h +++ b/cpp/src/utils/StringHelpFunctions.h @@ -18,8 +18,6 @@ private: StringHelpFunctions() = default; public: - static void TrimStringLineBreak(std::string &string); - static void TrimStringBlank(std::string &string); static void TrimStringQuote(std::string &string, const std::string &qoute); diff --git a/cpp/unittest/CMakeLists.txt b/cpp/unittest/CMakeLists.txt index 2e2b1f91b6..b868d0f1f1 100644 --- a/cpp/unittest/CMakeLists.txt +++ b/cpp/unittest/CMakeLists.txt @@ -46,6 +46,6 @@ set(unittest_libs add_subdirectory(server) add_subdirectory(db) add_subdirectory(faiss_wrapper) -add_subdirectory(license) +#add_subdirectory(license) add_subdirectory(metrics) add_subdirectory(storage) \ No newline at end of file diff --git a/cpp/unittest/db/meta_tests.cpp b/cpp/unittest/db/meta_tests.cpp index d71a942d3e..9baef712ab 100644 --- a/cpp/unittest/db/meta_tests.cpp +++ b/cpp/unittest/db/meta_tests.cpp @@ -17,39 +17,39 @@ using namespace zilliz::milvus::engine; -TEST_F(MetaTest, GROUP_TEST) { - auto table_id = "meta_test_group"; +TEST_F(MetaTest, TABLE_TEST) { + auto table_id = "meta_test_table"; - meta::TableSchema group; - group.table_id_ = table_id; - auto status = impl_->CreateTable(group); + meta::TableSchema table; + table.table_id_ = table_id; + auto status = impl_->CreateTable(table); ASSERT_TRUE(status.ok()); - auto gid = group.id_; - group.id_ = -1; - status = impl_->DescribeTable(group); + auto gid = table.id_; + table.id_ = -1; + status = impl_->DescribeTable(table); ASSERT_TRUE(status.ok()); - ASSERT_EQ(group.id_, gid); - ASSERT_EQ(group.table_id_, table_id); + ASSERT_EQ(table.id_, gid); + ASSERT_EQ(table.table_id_, table_id); - group.table_id_ = "not_found"; - status = impl_->DescribeTable(group); + table.table_id_ = "not_found"; + status = impl_->DescribeTable(table); ASSERT_TRUE(!status.ok()); - group.table_id_ = table_id; - status = impl_->CreateTable(group); - ASSERT_TRUE(!status.ok()); + table.table_id_ = table_id; + status = impl_->CreateTable(table); + ASSERT_TRUE(status.ok()); } -TEST_F(MetaTest, table_file_TEST) { - auto table_id = "meta_test_group"; +TEST_F(MetaTest, TABLE_FILE_TEST) { + auto table_id = "meta_test_table"; - meta::TableSchema group; - group.table_id_ = table_id; - auto status = impl_->CreateTable(group); + meta::TableSchema table; + table.table_id_ = table_id; + auto status = impl_->CreateTable(table); meta::TableFileSchema table_file; - table_file.table_id_ = group.table_id_; + table_file.table_id_ = table.table_id_; status = impl_->CreateTableFile(table_file); ASSERT_TRUE(status.ok()); ASSERT_EQ(table_file.file_type_, meta::TableFileSchema::NEW); @@ -104,15 +104,15 @@ TEST_F(MetaTest, ARCHIVE_TEST_DAYS) { options.archive_conf = ArchiveConf("delete", ss.str()); auto impl = meta::DBMetaImpl(options); - auto table_id = "meta_test_group"; + auto table_id = "meta_test_table"; - meta::TableSchema group; - group.table_id_ = table_id; - auto status = impl.CreateTable(group); + meta::TableSchema table; + table.table_id_ = table_id; + auto status = impl.CreateTable(table); meta::TableFilesSchema files; meta::TableFileSchema table_file; - table_file.table_id_ = group.table_id_; + table_file.table_id_ = table.table_id_; auto cnt = 100; long ts = utils::GetMicroSecTimeStamp(); @@ -156,13 +156,13 @@ TEST_F(MetaTest, ARCHIVE_TEST_DISK) { auto impl = meta::DBMetaImpl(options); auto table_id = "meta_test_group"; - meta::TableSchema group; - group.table_id_ = table_id; - auto status = impl.CreateTable(group); + meta::TableSchema table; + table.table_id_ = table_id; + auto status = impl.CreateTable(table); meta::TableFilesSchema files; meta::TableFileSchema table_file; - table_file.table_id_ = group.table_id_; + table_file.table_id_ = table.table_id_; auto cnt = 10; auto each_size = 2UL; @@ -198,9 +198,9 @@ TEST_F(MetaTest, ARCHIVE_TEST_DISK) { TEST_F(MetaTest, TABLE_FILES_TEST) { auto table_id = "meta_test_group"; - meta::TableSchema group; - group.table_id_ = table_id; - auto status = impl_->CreateTable(group); + meta::TableSchema table; + table.table_id_ = table_id; + auto status = impl_->CreateTable(table); int new_files_cnt = 4; int raw_files_cnt = 5; @@ -208,7 +208,7 @@ TEST_F(MetaTest, TABLE_FILES_TEST) { int index_files_cnt = 7; meta::TableFileSchema table_file; - table_file.table_id_ = group.table_id_; + table_file.table_id_ = table.table_id_; for (auto i=0; iCreateTableFile(table_file); @@ -241,7 +241,7 @@ TEST_F(MetaTest, TABLE_FILES_TEST) { ASSERT_EQ(files.size(), to_index_files_cnt); meta::DatePartionedTableFilesSchema dated_files; - status = impl_->FilesToMerge(group.table_id_, dated_files); + status = impl_->FilesToMerge(table.table_id_, dated_files); ASSERT_TRUE(status.ok()); ASSERT_EQ(dated_files[table_file.date_].size(), raw_files_cnt); diff --git a/cpp/unittest/license/license_check_test.cpp b/cpp/unittest/license/license_check_test.cpp index 2621ff58ae..0ad719bb60 100644 --- a/cpp/unittest/license/license_check_test.cpp +++ b/cpp/unittest/license/license_check_test.cpp @@ -1,184 +1,184 @@ +//// +//// Created by zilliz on 19-5-13. +//// // -// Created by zilliz on 19-5-13. +//#include "utils/Log.h" +//#include "license/LicenseCheck.h" +//#include "utils/Error.h" +// +//#include +//#include +// +// +//using namespace zilliz::milvus; +// +//TEST(LicenseLibraryTest, CHECK_TEST) { +// +// +// server::ServerError err; +// // 1. Set license file name +// std::string license_file_path("/tmp/milvus/abc.license"); +// +// // 2. Legality check +// err = server::LicenseCheck::LegalityCheck(license_file_path); +// ASSERT_EQ(err, server::SERVER_SUCCESS); +// +//} +// +//TEST(LicenseLibraryTest, CHECK_ERROR1_TEST){ +// +// server::ServerError err; +// // 1. Set license file name +// std::string license_file_path("/tmp/milvus/abc"); +// +// // 2. Legality check +// err = server::LicenseCheck::LegalityCheck(license_file_path); +// ASSERT_EQ(err, server::SERVER_SUCCESS); +//} +// +//TEST(LicenseLibraryTest, CHECK_ERROR2_TEST){ +// +// server::ServerError err; +// // 1. Set license file name +// std::string license_file_path("/tmp/milvus/abc.license"); +// +// // 2. Define output var +// int device_count; +// std::map uuid_encryption_map; +// time_t starting_time; +// time_t end_time; +// +// // 3. Read License File +// err = server::LicenseLibrary::LicenseFileDeserialization(license_file_path, +// device_count, +// uuid_encryption_map, +// starting_time, +// end_time); +// ASSERT_EQ(err, server::SERVER_SUCCESS); +// +// // 4. Change device count +// ++device_count; +// err = server::LicenseLibrary::LicenseFileSerialization(license_file_path, +// device_count, +// uuid_encryption_map, +// starting_time, +// end_time); +// ASSERT_EQ(err, server::SERVER_SUCCESS); +// +// // 5. Legality check +// err = server::LicenseCheck::LegalityCheck(license_file_path); +// ASSERT_EQ(err, server::SERVER_SUCCESS); +//} +// +//TEST(LicenseLibraryTest, CHECK_ERROR3_TEST){ +// +// server::ServerError err; +// // 1. Set license file name +// std::string license_file_path("/tmp/milvus/abc.license"); +// +// // 2. Define output var +// int device_count; +// std::map uuid_encryption_map; +// time_t starting_time; +// time_t end_time; +// +// // 3. Read License File +// err = server::LicenseLibrary::LicenseFileDeserialization(license_file_path, +// device_count, +// uuid_encryption_map, +// starting_time, +// end_time); +// ASSERT_EQ(err, server::SERVER_SUCCESS); +// +// // 4. Change device count +// if(device_count) uuid_encryption_map[0]+="u"; +// err = server::LicenseLibrary::LicenseFileSerialization(license_file_path, +// device_count, +// uuid_encryption_map, +// starting_time, +// end_time); +// ASSERT_EQ(err, server::SERVER_SUCCESS); +// +// // 5. Legality check +// err = server::LicenseCheck::LegalityCheck(license_file_path); +// ASSERT_EQ(err, server::SERVER_SUCCESS); +//} +// +//TEST(LicenseLibraryTest, CHECK_ERROR4_1_TEST){ +// +// server::ServerError err; +// // 1. Set license file name +// std::string license_file_path("/tmp/milvus/abc.license"); +// +// // 2. Define output var +// int device_count; +// std::map uuid_encryption_map; +// time_t starting_time; +// time_t end_time; +// +// // 3. Read License File +// err = server::LicenseLibrary::LicenseFileDeserialization(license_file_path, +// device_count, +// uuid_encryption_map, +// starting_time, +// end_time); +// ASSERT_EQ(err, server::SERVER_SUCCESS); +// +// // 4. Change starting time +// time_t system_time; +// err = server::LicenseLibrary::GetSystemTime(system_time); +// ASSERT_EQ(err, server::SERVER_SUCCESS); +// +// system_time+=60*60*24; +// +// err = server::LicenseLibrary::LicenseFileSerialization(license_file_path, +// device_count, +// uuid_encryption_map, +// system_time, +// end_time); +// ASSERT_EQ(err, server::SERVER_SUCCESS); +// +// // 5. Legality check +// err = server::LicenseCheck::LegalityCheck(license_file_path); +// ASSERT_EQ(err, server::SERVER_SUCCESS); +//} +// +//TEST(LicenseLibraryTest, CHECK_ERROR4_2_TEST){ +// +// server::ServerError err; +// // 1. Set license file name +// std::string license_file_path("/tmp/milvus/abc.license"); +// +// // 2. Define output var +// int device_count; +// std::map uuid_encryption_map; +// time_t starting_time; +// time_t end_time; +// +// // 3. Read License File +// err = server::LicenseLibrary::LicenseFileDeserialization(license_file_path, +// device_count, +// uuid_encryption_map, +// starting_time, +// end_time); +// ASSERT_EQ(err, server::SERVER_SUCCESS); +// +// // 4. Change end time +// time_t system_time; +// err = server::LicenseLibrary::GetSystemTime(system_time); +// ASSERT_EQ(err, server::SERVER_SUCCESS); +// +// system_time-=100; +// +// err = server::LicenseLibrary::LicenseFileSerialization(license_file_path, +// device_count, +// uuid_encryption_map, +// starting_time, +// system_time); +// ASSERT_EQ(err, server::SERVER_SUCCESS); +// +// // 5. Legality check +// err = server::LicenseCheck::LegalityCheck(license_file_path); +// ASSERT_EQ(err, server::SERVER_SUCCESS); +//} // - -#include "utils/Log.h" -#include "license/LicenseCheck.h" -#include "utils/Error.h" - -#include -#include - - -using namespace zilliz::milvus; - -TEST(LicenseLibraryTest, CHECK_TEST) { - - - server::ServerError err; - // 1. Set license file name - std::string license_file_path("/tmp/milvus/abc.license"); - - // 2. Legality check - err = server::LicenseCheck::LegalityCheck(license_file_path); - ASSERT_EQ(err, server::SERVER_SUCCESS); - -} - -TEST(LicenseLibraryTest, CHECK_ERROR1_TEST){ - - server::ServerError err; - // 1. Set license file name - std::string license_file_path("/tmp/milvus/abc"); - - // 2. Legality check - err = server::LicenseCheck::LegalityCheck(license_file_path); - ASSERT_EQ(err, server::SERVER_SUCCESS); -} - -TEST(LicenseLibraryTest, CHECK_ERROR2_TEST){ - - server::ServerError err; - // 1. Set license file name - std::string license_file_path("/tmp/milvus/abc.license"); - - // 2. Define output var - int device_count; - std::map uuid_encryption_map; - time_t starting_time; - time_t end_time; - - // 3. Read License File - err = server::LicenseLibrary::LicenseFileDeserialization(license_file_path, - device_count, - uuid_encryption_map, - starting_time, - end_time); - ASSERT_EQ(err, server::SERVER_SUCCESS); - - // 4. Change device count - ++device_count; - err = server::LicenseLibrary::LicenseFileSerialization(license_file_path, - device_count, - uuid_encryption_map, - starting_time, - end_time); - ASSERT_EQ(err, server::SERVER_SUCCESS); - - // 5. Legality check - err = server::LicenseCheck::LegalityCheck(license_file_path); - ASSERT_EQ(err, server::SERVER_SUCCESS); -} - -TEST(LicenseLibraryTest, CHECK_ERROR3_TEST){ - - server::ServerError err; - // 1. Set license file name - std::string license_file_path("/tmp/milvus/abc.license"); - - // 2. Define output var - int device_count; - std::map uuid_encryption_map; - time_t starting_time; - time_t end_time; - - // 3. Read License File - err = server::LicenseLibrary::LicenseFileDeserialization(license_file_path, - device_count, - uuid_encryption_map, - starting_time, - end_time); - ASSERT_EQ(err, server::SERVER_SUCCESS); - - // 4. Change device count - if(device_count) uuid_encryption_map[0]+="u"; - err = server::LicenseLibrary::LicenseFileSerialization(license_file_path, - device_count, - uuid_encryption_map, - starting_time, - end_time); - ASSERT_EQ(err, server::SERVER_SUCCESS); - - // 5. Legality check - err = server::LicenseCheck::LegalityCheck(license_file_path); - ASSERT_EQ(err, server::SERVER_SUCCESS); -} - -TEST(LicenseLibraryTest, CHECK_ERROR4_1_TEST){ - - server::ServerError err; - // 1. Set license file name - std::string license_file_path("/tmp/milvus/abc.license"); - - // 2. Define output var - int device_count; - std::map uuid_encryption_map; - time_t starting_time; - time_t end_time; - - // 3. Read License File - err = server::LicenseLibrary::LicenseFileDeserialization(license_file_path, - device_count, - uuid_encryption_map, - starting_time, - end_time); - ASSERT_EQ(err, server::SERVER_SUCCESS); - - // 4. Change starting time - time_t system_time; - err = server::LicenseLibrary::GetSystemTime(system_time); - ASSERT_EQ(err, server::SERVER_SUCCESS); - - system_time+=60*60*24; - - err = server::LicenseLibrary::LicenseFileSerialization(license_file_path, - device_count, - uuid_encryption_map, - system_time, - end_time); - ASSERT_EQ(err, server::SERVER_SUCCESS); - - // 5. Legality check - err = server::LicenseCheck::LegalityCheck(license_file_path); - ASSERT_EQ(err, server::SERVER_SUCCESS); -} - -TEST(LicenseLibraryTest, CHECK_ERROR4_2_TEST){ - - server::ServerError err; - // 1. Set license file name - std::string license_file_path("/tmp/milvus/abc.license"); - - // 2. Define output var - int device_count; - std::map uuid_encryption_map; - time_t starting_time; - time_t end_time; - - // 3. Read License File - err = server::LicenseLibrary::LicenseFileDeserialization(license_file_path, - device_count, - uuid_encryption_map, - starting_time, - end_time); - ASSERT_EQ(err, server::SERVER_SUCCESS); - - // 4. Change end time - time_t system_time; - err = server::LicenseLibrary::GetSystemTime(system_time); - ASSERT_EQ(err, server::SERVER_SUCCESS); - - system_time-=100; - - err = server::LicenseLibrary::LicenseFileSerialization(license_file_path, - device_count, - uuid_encryption_map, - starting_time, - system_time); - ASSERT_EQ(err, server::SERVER_SUCCESS); - - // 5. Legality check - err = server::LicenseCheck::LegalityCheck(license_file_path); - ASSERT_EQ(err, server::SERVER_SUCCESS); -} - diff --git a/cpp/unittest/license/license_library_tests.cpp b/cpp/unittest/license/license_library_tests.cpp index 406fd863c1..e96a1ecb2b 100644 --- a/cpp/unittest/license/license_library_tests.cpp +++ b/cpp/unittest/license/license_library_tests.cpp @@ -1,214 +1,214 @@ -/******************************************************************************* - * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved - * Unauthorized copying of this file, via any medium is strictly prohibited. - * Proprietary and confidential. - ******************************************************************************/ - - -#include "utils/Log.h" -#include "license/LicenseLibrary.h" -#include "utils/Error.h" - -#include -#include - - -using namespace zilliz::milvus; - -TEST(LicenseLibraryTest, FILE_EXISTENT_TEST) { - - std::string hosts_file = "/etc/hosts"; - ASSERT_EQ(server::LicenseLibrary::IsFileExistent(hosts_file), true); - - std::string no_exist_file = "/temp/asdaasd"; - ASSERT_EQ(server::LicenseLibrary::IsFileExistent(no_exist_file), false); - - std::string directory = "/tmp"; - ASSERT_EQ(server::LicenseLibrary::IsFileExistent(directory), false); -} - -TEST(LicenseLibraryTest, GPU_INFO_TEST) { - - int device_count = 0; - server::ServerError err = server::LicenseLibrary::GetDeviceCount(device_count); - ASSERT_EQ(err, server::SERVER_SUCCESS); - std::cout << "Device Count: " << device_count << std::endl; - - std::vector uuid_array; - err = server::LicenseLibrary::GetUUID(device_count, uuid_array); - ASSERT_EQ(err, server::SERVER_SUCCESS); - for (long i = 0; i < device_count; ++i) { - std::cout << "Device Id: " << i << ", UUID: " << uuid_array[i] << std::endl; - } - - std::vector uuid_md5_array; - err = server::LicenseLibrary::GetUUIDMD5(device_count, uuid_array, uuid_md5_array); - ASSERT_EQ(err, server::SERVER_SUCCESS); - for (long i = 0; i < device_count; ++i) { - std::cout << "Device Id: " << i << ", UUID: " << uuid_array[i] << ", UUID_MD5: " << uuid_md5_array[i] - << std::endl; - } - - std::vector uuid_sha256_array; - err = server::LicenseLibrary::GetUUIDSHA256(device_count, uuid_array, uuid_sha256_array); - ASSERT_EQ(err, server::SERVER_SUCCESS); - for (long i = 0; i < device_count; ++i) { - std::cout << "Device Id: " << i << ", UUID: " << uuid_array[i] << ", UUID_SHA256: " - << uuid_sha256_array[i] << std::endl; - } - - time_t systemtime; - err = server::LicenseLibrary::GetSystemTime(systemtime); - ASSERT_EQ(err, server::SERVER_SUCCESS); - std::cout << "System Time: " << systemtime << std::endl; - -} - -TEST(LicenseLibraryTest, LICENSE_FILE_TEST) { - - // 0. File check - std::string test("/tmp/a.test"); - bool is = server::LicenseLibrary::IsFileExistent(test); - ASSERT_EQ(is, false); - - // 1. Get Device Count - int device_count = 0; - server::ServerError err = server::LicenseLibrary::GetDeviceCount(device_count); - ASSERT_EQ(err, server::SERVER_SUCCESS); - - // 2. Get All GPU UUID - std::vector uuid_array; - err = server::LicenseLibrary::GetUUID(device_count, uuid_array); - ASSERT_EQ(err, server::SERVER_SUCCESS); - - // 3. Get UUID SHA256 - std::vector uuid_sha256_array; - err = server::LicenseLibrary::GetUUIDSHA256(device_count, uuid_array, uuid_sha256_array); - ASSERT_EQ(err, server::SERVER_SUCCESS); - - // 4. Generate GPU ID map with GPU UUID - std::map uuid_encrption_map; - for (int i = 0; i < device_count; ++i) { - uuid_encrption_map[i] = uuid_sha256_array[i]; - } - - // 5.GPU_info File - std::string GPU_info_file_path("/tmp/milvus.info"); - - - // 6. Generate GPU_info File - err = server::LicenseLibrary::GPUinfoFileSerialization(GPU_info_file_path, - device_count, - uuid_encrption_map); - ASSERT_EQ(err, server::SERVER_SUCCESS); - - // 7. Define output var - int output_info_device_count = 0; - std::map output_info_uuid_encrption_map; - - // 8. Read GPU_info File - err = server::LicenseLibrary::GPUinfoFileDeserialization(GPU_info_file_path, - output_info_device_count, - output_info_uuid_encrption_map); - ASSERT_EQ(err, server::SERVER_SUCCESS); - - ASSERT_EQ(device_count, output_info_device_count); - for (int i = 0; i < device_count; ++i) { - ASSERT_EQ(uuid_encrption_map[i], output_info_uuid_encrption_map[i]); - } - - // 9. Set license file name - std::string license_file_path("/tmp/milvus.license"); - - // 10. Get System Time/starting_time ans End Time - time_t sysyem_time; - err = server::LicenseLibrary::GetSystemTime(sysyem_time); - ASSERT_EQ(err, server::SERVER_SUCCESS); - - // 11.GetDateTime - time_t starting_time; - time_t end_time; - const char *string_starting_time = "2019-05-10"; - const char *string_end_time = "2022-05-10"; - err = server::LicenseLibrary::GetDateTime(string_starting_time, starting_time); - ASSERT_EQ(err, server::SERVER_SUCCESS); - err = server::LicenseLibrary::GetDateTime(string_end_time, end_time); - ASSERT_EQ(err, server::SERVER_SUCCESS); - - // 12. Generate License File - err = server::LicenseLibrary::LicenseFileSerialization(license_file_path, - device_count, - uuid_encrption_map, - starting_time, - end_time); - ASSERT_EQ(err, server::SERVER_SUCCESS); - - // 13. Define output var - int output_device_count = 0; - std::map output_uuid_encrption_map; - time_t output_starting_time; - time_t output_end_time; - - // 14. Read License File - err = server::LicenseLibrary::LicenseFileDeserialization(license_file_path, - output_device_count, - output_uuid_encrption_map, - output_starting_time, - output_end_time); - ASSERT_EQ(err, server::SERVER_SUCCESS); - - ASSERT_EQ(device_count, output_device_count); - ASSERT_EQ(starting_time, output_starting_time); - ASSERT_EQ(end_time, output_end_time); - - for (int i = 0; i < device_count; ++i) { - ASSERT_EQ(uuid_encrption_map[i], output_uuid_encrption_map[i]); - } - -// // 15. Get License File Attribute -// time_t update_time; -// off_t file_size; -// err = server::LicenseLibrary::GetFileUpdateTimeAndSize(license_file_path, update_time, file_size); +///******************************************************************************* +// * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved +// * Unauthorized copying of this file, via any medium is strictly prohibited. +// * Proprietary and confidential. +// ******************************************************************************/ +// +// +//#include "utils/Log.h" +//#include "license/LicenseLibrary.h" +//#include "utils/Error.h" +// +//#include +//#include +// +// +//using namespace zilliz::milvus; +// +//TEST(LicenseLibraryTest, FILE_EXISTENT_TEST) { +// +// std::string hosts_file = "/etc/hosts"; +// ASSERT_EQ(server::LicenseLibrary::IsFileExistent(hosts_file), true); +// +// std::string no_exist_file = "/temp/asdaasd"; +// ASSERT_EQ(server::LicenseLibrary::IsFileExistent(no_exist_file), false); +// +// std::string directory = "/tmp"; +// ASSERT_EQ(server::LicenseLibrary::IsFileExistent(directory), false); +//} +// +//TEST(LicenseLibraryTest, GPU_INFO_TEST) { +// +// int device_count = 0; +// server::ServerError err = server::LicenseLibrary::GetDeviceCount(device_count); +// ASSERT_EQ(err, server::SERVER_SUCCESS); +// std::cout << "Device Count: " << device_count << std::endl; +// +// std::vector uuid_array; +// err = server::LicenseLibrary::GetUUID(device_count, uuid_array); +// ASSERT_EQ(err, server::SERVER_SUCCESS); +// for (long i = 0; i < device_count; ++i) { +// std::cout << "Device Id: " << i << ", UUID: " << uuid_array[i] << std::endl; +// } +// +// std::vector uuid_md5_array; +// err = server::LicenseLibrary::GetUUIDMD5(device_count, uuid_array, uuid_md5_array); +// ASSERT_EQ(err, server::SERVER_SUCCESS); +// for (long i = 0; i < device_count; ++i) { +// std::cout << "Device Id: " << i << ", UUID: " << uuid_array[i] << ", UUID_MD5: " << uuid_md5_array[i] +// << std::endl; +// } +// +// std::vector uuid_sha256_array; +// err = server::LicenseLibrary::GetUUIDSHA256(device_count, uuid_array, uuid_sha256_array); +// ASSERT_EQ(err, server::SERVER_SUCCESS); +// for (long i = 0; i < device_count; ++i) { +// std::cout << "Device Id: " << i << ", UUID: " << uuid_array[i] << ", UUID_SHA256: " +// << uuid_sha256_array[i] << std::endl; +// } +// +// time_t systemtime; +// err = server::LicenseLibrary::GetSystemTime(systemtime); +// ASSERT_EQ(err, server::SERVER_SUCCESS); +// std::cout << "System Time: " << systemtime << std::endl; +// +//} +// +//TEST(LicenseLibraryTest, LICENSE_FILE_TEST) { +// +// // 0. File check +// std::string test("/tmp/a.test"); +// bool is = server::LicenseLibrary::IsFileExistent(test); +// ASSERT_EQ(is, false); +// +// // 1. Get Device Count +// int device_count = 0; +// server::ServerError err = server::LicenseLibrary::GetDeviceCount(device_count); // ASSERT_EQ(err, server::SERVER_SUCCESS); // -// // 16. Get License File MD5 -// std::string file_md5; -// err = server::LicenseLibrary::GetFileMD5(license_file_path, file_md5); +// // 2. Get All GPU UUID +// std::vector uuid_array; +// err = server::LicenseLibrary::GetUUID(device_count, uuid_array); // ASSERT_EQ(err, server::SERVER_SUCCESS); - - - -// // 17. Generate Secret File -// std::string secret_file_path("/tmp/milvus.secret"); -// err = server::LicenseLibrary::SecretFileSerialization(secret_file_path, -// update_time, -// file_size, -// starting_time, -// end_time, -// file_md5); +// +// // 3. Get UUID SHA256 +// std::vector uuid_sha256_array; +// err = server::LicenseLibrary::GetUUIDSHA256(device_count, uuid_array, uuid_sha256_array); // ASSERT_EQ(err, server::SERVER_SUCCESS); - -// // 18. Define output var -// time_t output_update_time; -// off_t output_file_size; +// +// // 4. Generate GPU ID map with GPU UUID +// std::map uuid_encrption_map; +// for (int i = 0; i < device_count; ++i) { +// uuid_encrption_map[i] = uuid_sha256_array[i]; +// } +// +// // 5.GPU_info File +// std::string GPU_info_file_path("/tmp/milvus.info"); +// +// +// // 6. Generate GPU_info File +// err = server::LicenseLibrary::GPUinfoFileSerialization(GPU_info_file_path, +// device_count, +// uuid_encrption_map); +// ASSERT_EQ(err, server::SERVER_SUCCESS); +// +// // 7. Define output var +// int output_info_device_count = 0; +// std::map output_info_uuid_encrption_map; +// +// // 8. Read GPU_info File +// err = server::LicenseLibrary::GPUinfoFileDeserialization(GPU_info_file_path, +// output_info_device_count, +// output_info_uuid_encrption_map); +// ASSERT_EQ(err, server::SERVER_SUCCESS); +// +// ASSERT_EQ(device_count, output_info_device_count); +// for (int i = 0; i < device_count; ++i) { +// ASSERT_EQ(uuid_encrption_map[i], output_info_uuid_encrption_map[i]); +// } +// +// // 9. Set license file name +// std::string license_file_path("/tmp/milvus.license"); +// +// // 10. Get System Time/starting_time ans End Time +// time_t sysyem_time; +// err = server::LicenseLibrary::GetSystemTime(sysyem_time); +// ASSERT_EQ(err, server::SERVER_SUCCESS); +// +// // 11.GetDateTime +// time_t starting_time; +// time_t end_time; +// const char *string_starting_time = "2019-05-10"; +// const char *string_end_time = "2022-05-10"; +// err = server::LicenseLibrary::GetDateTime(string_starting_time, starting_time); +// ASSERT_EQ(err, server::SERVER_SUCCESS); +// err = server::LicenseLibrary::GetDateTime(string_end_time, end_time); +// ASSERT_EQ(err, server::SERVER_SUCCESS); +// +// // 12. Generate License File +// err = server::LicenseLibrary::LicenseFileSerialization(license_file_path, +// device_count, +// uuid_encrption_map, +// starting_time, +// end_time); +// ASSERT_EQ(err, server::SERVER_SUCCESS); +// +// // 13. Define output var +// int output_device_count = 0; +// std::map output_uuid_encrption_map; // time_t output_starting_time; // time_t output_end_time; -// std::string output_file_md5; - -// // 19. Read License File -// err = server::LicenseLibrary::SecretFileDeserialization(secret_file_path, -// output_update_time, -// output_file_size, -// output_starting_time, -// output_end_time, -// output_file_md5); +// +// // 14. Read License File +// err = server::LicenseLibrary::LicenseFileDeserialization(license_file_path, +// output_device_count, +// output_uuid_encrption_map, +// output_starting_time, +// output_end_time); // ASSERT_EQ(err, server::SERVER_SUCCESS); // -// ASSERT_EQ(update_time, output_update_time); -// ASSERT_EQ(file_size, output_file_size); +// ASSERT_EQ(device_count, output_device_count); // ASSERT_EQ(starting_time, output_starting_time); // ASSERT_EQ(end_time, output_end_time); -// ASSERT_EQ(file_md5, output_file_md5); - - -} +// +// for (int i = 0; i < device_count; ++i) { +// ASSERT_EQ(uuid_encrption_map[i], output_uuid_encrption_map[i]); +// } +// +//// // 15. Get License File Attribute +//// time_t update_time; +//// off_t file_size; +//// err = server::LicenseLibrary::GetFileUpdateTimeAndSize(license_file_path, update_time, file_size); +//// ASSERT_EQ(err, server::SERVER_SUCCESS); +//// +//// // 16. Get License File MD5 +//// std::string file_md5; +//// err = server::LicenseLibrary::GetFileMD5(license_file_path, file_md5); +//// ASSERT_EQ(err, server::SERVER_SUCCESS); +// +// +// +//// // 17. Generate Secret File +//// std::string secret_file_path("/tmp/milvus.secret"); +//// err = server::LicenseLibrary::SecretFileSerialization(secret_file_path, +//// update_time, +//// file_size, +//// starting_time, +//// end_time, +//// file_md5); +//// ASSERT_EQ(err, server::SERVER_SUCCESS); +// +//// // 18. Define output var +//// time_t output_update_time; +//// off_t output_file_size; +//// time_t output_starting_time; +//// time_t output_end_time; +//// std::string output_file_md5; +// +//// // 19. Read License File +//// err = server::LicenseLibrary::SecretFileDeserialization(secret_file_path, +//// output_update_time, +//// output_file_size, +//// output_starting_time, +//// output_end_time, +//// output_file_md5); +//// ASSERT_EQ(err, server::SERVER_SUCCESS); +//// +//// ASSERT_EQ(update_time, output_update_time); +//// ASSERT_EQ(file_size, output_file_size); +//// ASSERT_EQ(starting_time, output_starting_time); +//// ASSERT_EQ(end_time, output_end_time); +//// ASSERT_EQ(file_md5, output_file_md5); +// +// +//} diff --git a/cpp/unittest/metrics/CMakeLists.txt b/cpp/unittest/metrics/CMakeLists.txt index 72e1b43867..80210772a8 100644 --- a/cpp/unittest/metrics/CMakeLists.txt +++ b/cpp/unittest/metrics/CMakeLists.txt @@ -55,8 +55,9 @@ set(count_test_src ${wrapper_src} ${metrics_src} metrics_test.cpp + prometheus_test.cpp ../db/utils.cpp - ) + metricbase_test.cpp) add_executable(metrics_test ${count_test_src} ${require_files} ) diff --git a/cpp/unittest/metrics/metricbase_test.cpp b/cpp/unittest/metrics/metricbase_test.cpp new file mode 100644 index 0000000000..ac850c7b48 --- /dev/null +++ b/cpp/unittest/metrics/metricbase_test.cpp @@ -0,0 +1,56 @@ +/******************************************************************************* + * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved + * Unauthorized copying of this file, via any medium is strictly prohibited. + * Proprietary and confidential. + ******************************************************************************/ + +#include "metrics/Metrics.h" + +#include +#include + +using namespace zilliz::milvus; + +TEST(MetricbaseTest, Metricbase_Test){ + server::MetricsBase instance = server::MetricsBase::GetInstance(); + instance.Init(); + server::SystemInfo::GetInstance().Init(); + instance.AddVectorsSuccessTotalIncrement(); + instance.AddVectorsFailTotalIncrement(); + instance.AddVectorsDurationHistogramOberve(1.0); + instance.RawFileSizeHistogramObserve(1.0); + instance.IndexFileSizeHistogramObserve(1.0); + instance.BuildIndexDurationSecondsHistogramObserve(1.0); + instance.CacheUsageGaugeSet(1.0); + instance.MetaAccessTotalIncrement(); + instance.MetaAccessDurationSecondsHistogramObserve(1.0); + instance.FaissDiskLoadDurationSecondsHistogramObserve(1.0); + instance.FaissDiskLoadSizeBytesHistogramObserve(1.0); + instance.FaissDiskLoadIOSpeedGaugeSet(1.0); + instance.CacheAccessTotalIncrement(); + instance.MemTableMergeDurationSecondsHistogramObserve(1.0); + instance.SearchIndexDataDurationSecondsHistogramObserve(1.0); + instance.SearchRawDataDurationSecondsHistogramObserve(1.0); + instance.IndexFileSizeTotalIncrement(); + instance.RawFileSizeTotalIncrement(); + instance.IndexFileSizeGaugeSet(1.0); + instance.RawFileSizeGaugeSet(1.0); + instance.QueryResponseSummaryObserve(1.0); + instance.DiskStoreIOSpeedGaugeSet(1.0); + instance.DataFileSizeGaugeSet(1.0); + instance.AddVectorsSuccessGaugeSet(1.0); + instance.AddVectorsFailGaugeSet(1.0); + instance.QueryVectorResponseSummaryObserve(1.0, 1); + instance.QueryVectorResponsePerSecondGaugeSet(1.0); + instance.CPUUsagePercentSet(); + instance.RAMUsagePercentSet(); + instance.QueryResponsePerSecondGaugeSet(1.0); + instance.GPUPercentGaugeSet(); + instance.GPUMemoryUsageGaugeSet(); + instance.AddVectorsPerSecondGaugeSet(1,1,1); + instance.QueryIndexTypePerSecondSet("IVF", 1.0); + instance.ConnectionGaugeIncrement(); + instance.ConnectionGaugeDecrement(); + instance.KeepingAliveCounterIncrement(); + instance.OctetsSet(); +} \ No newline at end of file diff --git a/cpp/unittest/metrics/metrics_test.cpp b/cpp/unittest/metrics/metrics_test.cpp index 72596dc79e..923c7b717b 100644 --- a/cpp/unittest/metrics/metrics_test.cpp +++ b/cpp/unittest/metrics/metrics_test.cpp @@ -24,7 +24,6 @@ using namespace zilliz::milvus; - TEST_F(DBTest, Metric_Tes) { server::SystemInfo::GetInstance().Init(); diff --git a/cpp/unittest/metrics/prometheus_test.cpp b/cpp/unittest/metrics/prometheus_test.cpp new file mode 100644 index 0000000000..885abed566 --- /dev/null +++ b/cpp/unittest/metrics/prometheus_test.cpp @@ -0,0 +1,57 @@ +/******************************************************************************* + * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved + * Unauthorized copying of this file, via any medium is strictly prohibited. + * Proprietary and confidential. + ******************************************************************************/ + +#include "metrics/PrometheusMetrics.h" + +#include +#include + +using namespace zilliz::milvus; + +TEST(PrometheusTest, Prometheus_Test){ + server::PrometheusMetrics instance = server::PrometheusMetrics::GetInstance(); + instance.Init(); + instance.SetStartup(true); + server::SystemInfo::GetInstance().Init(); + instance.AddVectorsSuccessTotalIncrement(); + instance.AddVectorsFailTotalIncrement(); + instance.AddVectorsDurationHistogramOberve(1.0); + instance.RawFileSizeHistogramObserve(1.0); + instance.IndexFileSizeHistogramObserve(1.0); + instance.BuildIndexDurationSecondsHistogramObserve(1.0); + instance.CacheUsageGaugeSet(1.0); + instance.MetaAccessTotalIncrement(); + instance.MetaAccessDurationSecondsHistogramObserve(1.0); + instance.FaissDiskLoadDurationSecondsHistogramObserve(1.0); + instance.FaissDiskLoadSizeBytesHistogramObserve(1.0); + instance.FaissDiskLoadIOSpeedGaugeSet(1.0); + instance.CacheAccessTotalIncrement(); + instance.MemTableMergeDurationSecondsHistogramObserve(1.0); + instance.SearchIndexDataDurationSecondsHistogramObserve(1.0); + instance.SearchRawDataDurationSecondsHistogramObserve(1.0); + instance.IndexFileSizeTotalIncrement(); + instance.RawFileSizeTotalIncrement(); + instance.IndexFileSizeGaugeSet(1.0); + instance.RawFileSizeGaugeSet(1.0); + instance.QueryResponseSummaryObserve(1.0); + instance.DiskStoreIOSpeedGaugeSet(1.0); + instance.DataFileSizeGaugeSet(1.0); + instance.AddVectorsSuccessGaugeSet(1.0); + instance.AddVectorsFailGaugeSet(1.0); + instance.QueryVectorResponseSummaryObserve(1.0, 1); + instance.QueryVectorResponsePerSecondGaugeSet(1.0); + instance.CPUUsagePercentSet(); + instance.RAMUsagePercentSet(); + instance.QueryResponsePerSecondGaugeSet(1.0); + instance.GPUPercentGaugeSet(); + instance.GPUMemoryUsageGaugeSet(); + instance.AddVectorsPerSecondGaugeSet(1,1,1); + instance.QueryIndexTypePerSecondSet("IVF", 1.0); + instance.ConnectionGaugeIncrement(); + instance.ConnectionGaugeDecrement(); + instance.KeepingAliveCounterIncrement(); + instance.OctetsSet(); +} \ No newline at end of file diff --git a/cpp/unittest/server/CMakeLists.txt b/cpp/unittest/server/CMakeLists.txt index d844b7a9a3..c4112cda9e 100644 --- a/cpp/unittest/server/CMakeLists.txt +++ b/cpp/unittest/server/CMakeLists.txt @@ -14,9 +14,11 @@ aux_source_directory(../../src/cache cache_srcs) aux_source_directory(../../src/wrapper wrapper_src) aux_source_directory(./ test_srcs) -set(server_src_files +set(utils_srcs ${MILVUS_ENGINE_SRC}/utils/StringHelpFunctions.cpp - ${MILVUS_ENGINE_SRC}/utils/AttributeSerializer.cpp + ${MILVUS_ENGINE_SRC}/utils/TimeRecorder.cpp + ${MILVUS_ENGINE_SRC}/utils/CommonUtil.cpp + ${MILVUS_ENGINE_SRC}/utils/LogUtil.cpp ) cuda_add_executable(server_test @@ -25,7 +27,7 @@ cuda_add_executable(server_test ${cache_srcs} ${wrapper_src} ${test_srcs} - ${server_src_files} + ${utils_srcs} ${require_files} ) @@ -53,3 +55,11 @@ target_link_libraries(server_test ) install(TARGETS server_test DESTINATION bin) + +configure_file(appendix/server_config.yaml + "${CMAKE_CURRENT_BINARY_DIR}/milvus/conf/server_config.yaml" + COPYONLY) + +configure_file(appendix/log_config.conf + "${CMAKE_CURRENT_BINARY_DIR}/milvus/conf/log_config.conf" + COPYONLY) diff --git a/cpp/conf/log_config.conf b/cpp/unittest/server/appendix/log_config.conf similarity index 91% rename from cpp/conf/log_config.conf rename to cpp/unittest/server/appendix/log_config.conf index 80710b570e..29d46a7fe5 100644 --- a/cpp/conf/log_config.conf +++ b/cpp/unittest/server/appendix/log_config.conf @@ -20,8 +20,8 @@ TO_STANDARD_OUTPUT = false ## Error logs * ERROR: - ENABLED = false + ENABLED = true FILENAME = "/tmp/milvus/logs/milvus-%datetime{%H:%m}-error.log" * FATAL: - ENABLED = false - FILENAME = "/tmp/milvus/logs/milvus-%datetime{%H:%m}-fatal.log" \ No newline at end of file + ENABLED = true + FILENAME = "/tmp/milvus/logs/milvus-%datetime{%H:%m}-fatal.log" diff --git a/cpp/conf/server_config.yaml b/cpp/unittest/server/appendix/server_config.yaml similarity index 92% rename from cpp/conf/server_config.yaml rename to cpp/unittest/server/appendix/server_config.yaml index 1c26c0aa52..9019461940 100644 --- a/cpp/conf/server_config.yaml +++ b/cpp/unittest/server/appendix/server_config.yaml @@ -22,7 +22,7 @@ metric_config: license_config: # license configure - license_path: "/tmp/system.license" # license file path + license_path: "/tmp/milvus/system.license" # license file path cache_config: # cache configure - cpu_cache_capacity: 16 # how many memory are used as cache, unit: GB, range: 0 ~ less than total memory \ No newline at end of file + cpu_cache_capacity: 16 # how many memory are used as cache, unit: GB, range: 0 ~ less than total memory diff --git a/cpp/unittest/server/cache_test.cpp b/cpp/unittest/server/cache_test.cpp index 98c06f788c..8c323d81d3 100644 --- a/cpp/unittest/server/cache_test.cpp +++ b/cpp/unittest/server/cache_test.cpp @@ -11,18 +11,35 @@ using namespace zilliz::milvus; -TEST(CacheTest, CACHE_TEST) { - cache::CacheMgr* cpu_mgr = cache::CpuCacheMgr::GetInstance(); +namespace { + +class InvalidCacheMgr : public cache::CacheMgr { +public: + InvalidCacheMgr() { + } +}; + +class LessItemCacheMgr : public cache::CacheMgr { +public: + LessItemCacheMgr() { + cache_ = std::make_shared(1UL << 12, 10); + } +}; + +} + +TEST(CacheTest, CPU_CACHE_TEST) { + cache::CacheMgr *cpu_mgr = cache::CpuCacheMgr::GetInstance(); const int64_t gbyte = 1 << 30; int64_t g_num = 16; - int64_t cap = g_num*gbyte; + int64_t cap = g_num * gbyte; cpu_mgr->SetCapacity(cap); ASSERT_EQ(cpu_mgr->CacheCapacity(), cap); const int dim = 256; - for(int i = 0; i < 20; i++) { + for (int i = 0; i < 20; i++) { std::shared_ptr raw_index(faiss::index_factory(dim, "IDMap,Flat")); engine::Index_ptr index = std::make_shared(raw_index); index->ntotal = 1000000;//less 1G per index @@ -31,6 +48,12 @@ TEST(CacheTest, CACHE_TEST) { } ASSERT_LT(cpu_mgr->ItemCount(), g_num); + auto obj = cpu_mgr->GetIndex("index_0"); + ASSERT_TRUE(obj == nullptr); + + obj = cpu_mgr->GetIndex("index_19"); + ASSERT_TRUE(obj != nullptr); + { std::string item = "index_15"; ASSERT_TRUE(cpu_mgr->ItemExists(item)); @@ -49,4 +72,58 @@ TEST(CacheTest, CACHE_TEST) { cpu_mgr->InsertItem("index_6g", index); ASSERT_EQ(cpu_mgr->ItemCount(), 0);//data greater than capacity can not be inserted sucessfully } + + cpu_mgr->PrintInfo(); +} + +TEST(CacheTest, GPU_CACHE_TEST) { + cache::CacheMgr* gpu_mgr = cache::GpuCacheMgr::GetInstance(); + + const int dim = 256; + + for(int i = 0; i < 20; i++) { + std::shared_ptr raw_index(faiss::index_factory(dim, "IDMap,Flat")); + engine::Index_ptr index = std::make_shared(raw_index); + index->ntotal = 1000; + + cache::DataObjPtr obj = std::make_shared(index); + + gpu_mgr->InsertItem("index_" + std::to_string(i), obj); + } + + auto obj = gpu_mgr->GetItem("index_0"); + + gpu_mgr->ClearCache(); + ASSERT_EQ(gpu_mgr->ItemCount(), 0); +} + +TEST(CacheTest, INVALID_TEST) { + { + InvalidCacheMgr mgr; + ASSERT_EQ(mgr.ItemCount(), 0); + ASSERT_FALSE(mgr.ItemExists("test")); + ASSERT_EQ(mgr.GetItem("test"), nullptr); + + mgr.InsertItem("test", cache::DataObjPtr()); + mgr.InsertItem("test", engine::Index_ptr(nullptr)); + mgr.EraseItem("test"); + mgr.PrintInfo(); + mgr.ClearCache(); + mgr.SetCapacity(100); + ASSERT_EQ(mgr.CacheCapacity(), 0); + ASSERT_EQ(mgr.CacheUsage(), 0); + } + + { + LessItemCacheMgr mgr; + for(int i = 0; i < 20; i++) { + std::shared_ptr raw_index(faiss::index_factory(2, "IDMap,Flat")); + engine::Index_ptr index = std::make_shared(raw_index); + index->ntotal = 2; + + cache::DataObjPtr obj = std::make_shared(index); + mgr.InsertItem("index_" + std::to_string(i), obj); + } + ASSERT_EQ(mgr.GetItem("index_0"), nullptr); + } } \ No newline at end of file diff --git a/cpp/unittest/server/common_test.cpp b/cpp/unittest/server/common_test.cpp deleted file mode 100644 index 27d8c522e5..0000000000 --- a/cpp/unittest/server/common_test.cpp +++ /dev/null @@ -1,26 +0,0 @@ -//////////////////////////////////////////////////////////////////////////////// -// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved -// Unauthorized copying of this file, via any medium is strictly prohibited. -// Proprietary and confidential. -//////////////////////////////////////////////////////////////////////////////// -#include -#include "utils/CommonUtil.h" -#include "utils/Error.h" - -using namespace zilliz::milvus; - - -TEST(CommonTest, COMMON_TEST) { - std::string path1 = "/tmp/milvus_test/"; - std::string path2 = path1 + "common_test_12345/"; - std::string path3 = path2 + "abcdef"; - server::ServerError err = server::CommonUtil::CreateDirectory(path3); - ASSERT_EQ(err, server::SERVER_SUCCESS); - - ASSERT_TRUE(server::CommonUtil::IsDirectoryExit(path3)); - - err = server::CommonUtil::DeleteDirectory(path1); - ASSERT_EQ(err, server::SERVER_SUCCESS); - - ASSERT_FALSE(server::CommonUtil::IsDirectoryExit(path1)); -} diff --git a/cpp/unittest/server/config_test.cpp b/cpp/unittest/server/config_test.cpp new file mode 100644 index 0000000000..4b10327cfb --- /dev/null +++ b/cpp/unittest/server/config_test.cpp @@ -0,0 +1,105 @@ +//////////////////////////////////////////////////////////////////////////////// +// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved +// Unauthorized copying of this file, via any medium is strictly prohibited. +// Proprietary and confidential. +//////////////////////////////////////////////////////////////////////////////// +#include + +#include "config/IConfigMgr.h" +#include "server/ServerConfig.h" + +using namespace zilliz::milvus; + +namespace { + +static const std::string CONFIG_FILE_PATH = "./milvus/conf/server_config.yaml"; +static const std::string LOG_FILE_PATH = "./milvus/conf/log_config.conf"; + +} + +TEST(ConfigTest, CONFIG_TEST) { + server::IConfigMgr* config_mgr = server::IConfigMgr::GetInstance(); + + server::ServerError err = config_mgr->LoadConfigFile(""); + ASSERT_EQ(err, server::SERVER_UNEXPECTED_ERROR); + + err = config_mgr->LoadConfigFile(LOG_FILE_PATH); + ASSERT_EQ(err, server::SERVER_UNEXPECTED_ERROR); + + err = config_mgr->LoadConfigFile(CONFIG_FILE_PATH); + ASSERT_EQ(err, server::SERVER_SUCCESS); + + config_mgr->Print(); + + server::ConfigNode& root_config = config_mgr->GetRootNode(); + server::ConfigNode& server_config = root_config.GetChild("server_config"); + server::ConfigNode& db_config = root_config.GetChild("db_config"); + server::ConfigNode& metric_config = root_config.GetChild("metric_config"); + server::ConfigNode& cache_config = root_config.GetChild("cache_config"); + + std::string address = server_config.GetValue("address"); + ASSERT_TRUE(!address.empty()); + int64_t port = server_config.GetInt64Value("port"); + ASSERT_TRUE(port != 0); + + server_config.SetValue("test", "2.5"); + double test = server_config.GetDoubleValue("test"); + ASSERT_EQ(test, 2.5); + + server::ConfigNode fake; + server_config.AddChild("fake", fake); + fake = server_config.GetChild("fake"); + server::ConfigNodeArr arr; + server_config.GetChildren(arr); + ASSERT_EQ(arr.size(), 1UL); + + server_config.ClearChildren(); + auto children = server_config.GetChildren(); + ASSERT_TRUE(children.empty()); + + server_config.ClearConfig(); + auto configs = server_config.GetConfig(); + ASSERT_TRUE(configs.empty()); + + server_config.AddSequenceItem("seq", "aaa"); + server_config.AddSequenceItem("seq", "bbb"); + auto seq = server_config.GetSequence("seq"); + ASSERT_EQ(seq.size(), 2UL); + + server::ConfigNode combine; + combine.Combine(server_config); + + combine.PrintAll(); + std::string all = combine.DumpString(); + ASSERT_TRUE(!all.empty()); + + server_config.ClearSequences(); + auto seqs = server_config.GetSequences(); + ASSERT_TRUE(seqs.empty()); + + const server::ConfigNode const_node = root_config.GetChild("cache_config"); + float flt = const_node.GetFloatValue("cpu_cache_capacity"); + ASSERT_GT(flt, 0.0); +} + +TEST(ConfigTest, SERVER_CONFIG_TEST) { + server::ServerConfig& config = server::ServerConfig::GetInstance(); + server::ServerError err = config.LoadConfigFile(CONFIG_FILE_PATH); + ASSERT_EQ(err, server::SERVER_SUCCESS); + + server::ConfigNode node1 = config.GetConfig("server_config"); + server::ConfigNode& node2 = config.GetConfig("cache_config"); + node1.Combine(node2); + + int32_t cap = node1.GetInt32Value("cpu_cache_capacity"); + ASSERT_GT(cap, 0); + + node1.SetValue("bool", "true"); + bool bt = node1.GetBoolValue("bool"); + ASSERT_TRUE(bt); + + config.PrintAll(); + + const server::ServerConfig const_config = config; + server::ConfigNode node = const_config.GetConfig("aaa"); +} \ No newline at end of file diff --git a/cpp/unittest/server/util_test.cpp b/cpp/unittest/server/util_test.cpp index 0bc951d88e..200bdc9776 100644 --- a/cpp/unittest/server/util_test.cpp +++ b/cpp/unittest/server/util_test.cpp @@ -4,31 +4,176 @@ // Proprietary and confidential. //////////////////////////////////////////////////////////////////////////////// #include +#include -#include "utils/AttributeSerializer.h" +#include "utils/CommonUtil.h" +#include "utils/Error.h" #include "utils/StringHelpFunctions.h" +#include "utils/TimeRecorder.h" +#include "utils/BlockingQueue.h" +#include "utils/LogUtil.h" using namespace zilliz::milvus; -TEST(AttribSerializeTest, ATTRIBSERIAL_TEST) { - std::map attrib; - attrib["uid"] = "ABCDEF"; - attrib["color"] = "red"; - attrib["number"] = "9900"; - attrib["comment"] = "please note: it is a car, not a ship"; - attrib["address"] = " china;shanghai "; +namespace { - std::string attri_str; - server::AttributeSerializer::Encode(attrib, attri_str); +static const std::string LOG_FILE_PATH = "./milvus/conf/log_config.conf"; - std::map attrib_out; - server::ServerError err = server::AttributeSerializer::Decode(attri_str, attrib_out); - ASSERT_EQ(err, server::SERVER_SUCCESS); - - ASSERT_EQ(attrib_out.size(), attrib.size()); - for(auto iter : attrib) { - ASSERT_EQ(attrib_out[iter.first], attrib_out[iter.first]); - } +using TimeUnit = server::TimeRecorder::TimeDisplayUnit; +double TestTimeRecorder(TimeUnit unit, int64_t log_level, int64_t sleep_ms) { + server::TimeRecorder rc("test rc", unit, log_level); + rc.Record("begin"); + std::this_thread::sleep_for(std::chrono::milliseconds(sleep_ms)); + rc.Elapse("end"); + return rc.Span(); +} } +TEST(UtilTest, EXCEPTION_TEST) { + std::string err_msg = "failed"; + server::ServerException ex(server::SERVER_UNEXPECTED_ERROR, err_msg); + ASSERT_EQ(ex.error_code(), server::SERVER_UNEXPECTED_ERROR); + std::string msg = ex.what(); + ASSERT_EQ(msg, err_msg); +} + +TEST(UtilTest, COMMON_TEST) { + unsigned long total_mem = 0, free_mem = 0; + server::CommonUtil::GetSystemMemInfo(total_mem, free_mem); + ASSERT_GT(total_mem, 0); + ASSERT_GT(free_mem, 0); + + unsigned int thread_cnt = 0; + server::CommonUtil::GetSystemAvailableThreads(thread_cnt); + ASSERT_GT(thread_cnt, 0); + + std::string path1 = "/tmp/milvus_test/"; + std::string path2 = path1 + "common_test_12345/"; + std::string path3 = path2 + "abcdef"; + server::ServerError err = server::CommonUtil::CreateDirectory(path3); + ASSERT_EQ(err, server::SERVER_SUCCESS); + //test again + err = server::CommonUtil::CreateDirectory(path3); + ASSERT_EQ(err, server::SERVER_SUCCESS); + + ASSERT_TRUE(server::CommonUtil::IsDirectoryExist(path3)); + + err = server::CommonUtil::DeleteDirectory(path1); + ASSERT_EQ(err, server::SERVER_SUCCESS); + //test again + err = server::CommonUtil::DeleteDirectory(path1); + ASSERT_EQ(err, server::SERVER_SUCCESS); + + ASSERT_FALSE(server::CommonUtil::IsDirectoryExist(path1)); + ASSERT_FALSE(server::CommonUtil::IsFileExist(path1)); + + std::string exe_path = server::CommonUtil::GetExePath(); + ASSERT_FALSE(exe_path.empty()); + + time_t tt; + time( &tt ); + tm time_struct; + memset(&time_struct, 0, sizeof(tm)); + server::CommonUtil::ConvertTime(tt, time_struct); + ASSERT_GT(time_struct.tm_year, 0); + ASSERT_GT(time_struct.tm_mon, 0); + ASSERT_GT(time_struct.tm_mday, 0); + server::CommonUtil::ConvertTime(time_struct, tt); + ASSERT_GT(tt, 0); + + bool res = server::CommonUtil::TimeStrToTime("2019-03-23", tt, time_struct); + ASSERT_EQ(time_struct.tm_year, 119); + ASSERT_EQ(time_struct.tm_mon, 2); + ASSERT_EQ(time_struct.tm_mday, 23); + ASSERT_GT(tt, 0); + ASSERT_TRUE(res); +} + +TEST(UtilTest, STRINGFUNCTIONS_TEST) { + std::string str = " test zilliz"; + server::StringHelpFunctions::TrimStringBlank(str); + ASSERT_EQ(str, "test zilliz"); + + str = "\"test zilliz\""; + server::StringHelpFunctions::TrimStringQuote(str, "\""); + ASSERT_EQ(str, "test zilliz"); + + str = "a,b,c"; + std::vector result; + server::ServerError err = server::StringHelpFunctions::SplitStringByDelimeter(str , ",", result); + ASSERT_EQ(err, server::SERVER_SUCCESS); + ASSERT_EQ(result.size(), 3UL); + + result.clear(); + err = server::StringHelpFunctions::SplitStringByQuote(str , ",", "\"", result); + ASSERT_EQ(err, server::SERVER_SUCCESS); + ASSERT_EQ(result.size(), 3UL); + + result.clear(); + err = server::StringHelpFunctions::SplitStringByQuote(str , ",", "", result); + ASSERT_EQ(err, server::SERVER_SUCCESS); + ASSERT_EQ(result.size(), 3UL); + + str = "55,\"aa,gg,yy\",b"; + result.clear(); + err = server::StringHelpFunctions::SplitStringByQuote(str , ",", "\"", result); + ASSERT_EQ(err, server::SERVER_SUCCESS); + ASSERT_EQ(result.size(), 3UL); + + +} + +TEST(UtilTest, TIMERECORDER_TEST) { + double span = TestTimeRecorder(TimeUnit::eTimeAutoUnit, 0, 1001); + ASSERT_GT(span, 0.0); + span = TestTimeRecorder(TimeUnit::eTimeAutoUnit, 0, 101); + ASSERT_GT(span, 0.0); + span = TestTimeRecorder(TimeUnit::eTimeHourUnit, 1, 10); + ASSERT_GT(span, 0.0); + span = TestTimeRecorder(TimeUnit::eTimeMinuteUnit, 2, 10); + ASSERT_GT(span, 0.0); + span = TestTimeRecorder(TimeUnit::eTimeSecondUnit, 3, 10); + ASSERT_GT(span, 0.0); + span = TestTimeRecorder(TimeUnit::eTimeMilliSecUnit, 4, 10); + ASSERT_GT(span, 0.0); + span = TestTimeRecorder(TimeUnit::eTimeMicroSecUnit, -1, 10); + ASSERT_GT(span, 0.0); +} + +TEST(UtilTest, BLOCKINGQUEUE_TEST) { + server::BlockingQueue bq; + + static const size_t count = 10; + bq.SetCapacity(count); + + for(size_t i = 1; i <= count; i++) { + std::string id = "No." + std::to_string(i); + bq.Put(id); + } + + ASSERT_EQ(bq.Size(), count); + ASSERT_FALSE(bq.Empty()); + + std::string str = bq.Front(); + ASSERT_EQ(str, "No.1"); + + str = bq.Back(); + ASSERT_EQ(str, "No." + std::to_string(count)); + + for(size_t i = 1; i <= count; i++) { + std::string id = "No." + std::to_string(i); + str = bq.Take(); + ASSERT_EQ(id, str); + } + + ASSERT_EQ(bq.Size(), 0); +} + +TEST(UtilTest, LOG_TEST) { + int32_t res = server::InitLog(LOG_FILE_PATH); + ASSERT_EQ(res, 0); + + std::string fname = server::GetFileName(LOG_FILE_PATH); + ASSERT_EQ(fname, "log_config.conf"); +} diff --git a/cpp/unittest/storage/S3ClientTest.cpp b/cpp/unittest/storage/S3ClientTest.cpp index c83209ecf6..5147f701ae 100644 --- a/cpp/unittest/storage/S3ClientTest.cpp +++ b/cpp/unittest/storage/S3ClientTest.cpp @@ -1,66 +1,66 @@ -//////////////////////////////////////////////////////////////////////////////// -// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved -// Unauthorized copying of this file, via any medium is strictly prohibited. -// Proprietary and confidential. -//////////////////////////////////////////////////////////////////////////////// - -#include "storage/IStorage.h" -#include "storage/s3/S3ClientWrapper.h" -#include -#include -#include - - -using namespace zilliz::milvus::engine; - -TEST(s3_client_wrapper, CLIENT_WRAPPER_TEST) { - - std::shared_ptr storage_ptr = std::make_shared(); - - std::string ip_address = "127.0.0.1"; - std::string port = "9000"; - std::string access_key = "AKIAIOSFODNN7EXAMPLE"; - std::string secret_key = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"; - - Status status = storage_ptr->Create(ip_address, port, access_key, secret_key); - ASSERT_TRUE(status.ok()); - - std::string filename = "/tmp/s3_test_file"; - std::string bucket_name = "bucktname"; - std::string object_name = "test_file"; - - status = storage_ptr->CreateBucket(bucket_name); - std::cout << status.IsAlreadyExist() << std::endl; - if (status.IsAlreadyExist()) { - status = storage_ptr->DeleteBucket(bucket_name); - status = storage_ptr->CreateBucket(bucket_name); - } - - ASSERT_TRUE(status.ok()); - - std::ofstream ofile(filename); - std::stringstream ss; - for (int i = 0; i < 1024; ++i) { - ss << i; - } - ofile << ss.str() << std::endl; - ofile.close(); - status = storage_ptr->UploadFile(bucket_name, object_name, filename); - ASSERT_TRUE(status.ok()); - - status = storage_ptr->DownloadFile(bucket_name, object_name, filename); - std::ifstream infile(filename); - std::string in_buffer; - infile >> in_buffer; - ASSERT_STREQ(in_buffer.c_str(), ss.str().c_str()); - - status = storage_ptr->DeleteFile(bucket_name, object_name); - ASSERT_TRUE(status.ok()); - - status = storage_ptr->DeleteBucket(bucket_name); - ASSERT_TRUE(status.ok()); - - status = storage_ptr->Close(); - ASSERT_TRUE(status.ok()); -} - +////////////////////////////////////////////////////////////////////////////////// +//// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved +//// Unauthorized copying of this file, via any medium is strictly prohibited. +//// Proprietary and confidential. +////////////////////////////////////////////////////////////////////////////////// +// +//#include "storage/IStorage.h" +//#include "storage/s3/S3ClientWrapper.h" +//#include +//#include +//#include +// +// +//using namespace zilliz::milvus::engine; +// +//TEST(s3_client_wrapper, CLIENT_WRAPPER_TEST) { +// +// std::shared_ptr storage_ptr = std::make_shared(); +// +// std::string ip_address = "127.0.0.1"; +// std::string port = "9000"; +// std::string access_key = "AKIAIOSFODNN7EXAMPLE"; +// std::string secret_key = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"; +// +// Status status = storage_ptr->Create(ip_address, port, access_key, secret_key); +// ASSERT_TRUE(status.ok()); +// +// std::string filename = "/tmp/s3_test_file"; +// std::string bucket_name = "bucktname"; +// std::string object_name = "test_file"; +// +// status = storage_ptr->CreateBucket(bucket_name); +// std::cout << status.IsAlreadyExist() << std::endl; +// if (status.IsAlreadyExist()) { +// status = storage_ptr->DeleteBucket(bucket_name); +// status = storage_ptr->CreateBucket(bucket_name); +// } +// +// ASSERT_TRUE(status.ok()); +// +// std::ofstream ofile(filename); +// std::stringstream ss; +// for (int i = 0; i < 1024; ++i) { +// ss << i; +// } +// ofile << ss.str() << std::endl; +// ofile.close(); +// status = storage_ptr->UploadFile(bucket_name, object_name, filename); +// ASSERT_TRUE(status.ok()); +// +// status = storage_ptr->DownloadFile(bucket_name, object_name, filename); +// std::ifstream infile(filename); +// std::string in_buffer; +// infile >> in_buffer; +// ASSERT_STREQ(in_buffer.c_str(), ss.str().c_str()); +// +// status = storage_ptr->DeleteFile(bucket_name, object_name); +// ASSERT_TRUE(status.ok()); +// +// status = storage_ptr->DeleteBucket(bucket_name); +// ASSERT_TRUE(status.ok()); +// +// status = storage_ptr->Close(); +// ASSERT_TRUE(status.ok()); +//} +//