diff --git a/cpp/.gitignore b/cpp/.gitignore index b9ba83ac8a..d8368bd79b 100644 --- a/cpp/.gitignore +++ b/cpp/.gitignore @@ -1,4 +1,5 @@ milvus/ conf/server_config.yaml +conf/log_config.conf version.h megasearch/ diff --git a/cpp/CHANGELOG.md b/cpp/CHANGELOG.md index 6c33a9eb2f..8ef9aca97a 100644 --- a/cpp/CHANGELOG.md +++ b/cpp/CHANGELOG.md @@ -24,6 +24,7 @@ Please mark all change in change log and use the ticket from JIRA. - MS-122 - Archive criteria config - MS-124 - HasTable interface - MS-126 - Add more error code +- MS-128 - Change default db path ## New Feature diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 05cbc1db64..f176ef7b27 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -135,12 +135,19 @@ if (BUILD_COVERAGE STREQUAL "ON") endif() -if (BUILD_UNIT_TEST) +if ("${BUILD_UNIT_TEST}" STREQUAL "ON") add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/unittest) endif(BUILD_UNIT_TEST) add_custom_target(Clean-All COMMAND ${CMAKE_BUILD_TOOL} clean) + +if("${MILVUS_DB_PATH}" STREQUAL "") + set(MILVUS_DB_PATH "/tmp/milvus") +endif() +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/conf/server_config.template ${CMAKE_CURRENT_SOURCE_DIR}/conf/server_config.yaml) +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/conf/log_config.template ${CMAKE_CURRENT_SOURCE_DIR}/conf/log_config.conf) + #install install(FILES scripts/start_server.sh @@ -153,4 +160,6 @@ install(FILES DESTINATION conf) + + config_summary() diff --git a/cpp/build.sh b/cpp/build.sh index a7ece5fee6..80be1d7ddb 100755 --- a/cpp/build.sh +++ b/cpp/build.sh @@ -1,13 +1,14 @@ #!/bin/bash BUILD_TYPE="Debug" -BUILD_UNITTEST="off" +BUILD_UNITTEST="OFF" LICENSE_CHECK="OFF" INSTALL_PREFIX=$(pwd)/milvus MAKE_CLEAN="OFF" BUILD_COVERAGE="OFF" +DB_PATH="/opt/milvus" -while getopts "p:t:uhlrc" arg +while getopts "p:d:t:uhlrc" arg do case $arg in t) @@ -15,11 +16,14 @@ do ;; u) echo "Build and run unittest cases" ; - BUILD_UNITTEST="on"; + BUILD_UNITTEST="ON"; ;; p) INSTALL_PREFIX=$OPTARG ;; + d) + DB_PATH=$OPTARG + ;; l) LICENSE_CHECK="ON" ;; @@ -36,12 +40,13 @@ do echo " parameter: --t: build type --u: building unit test options --p: install prefix --l: build license version --r: remove previous build directory --c: code coverage +-t: build type(default: Debug) +-u: building unit test options(default: OFF) +-p: install prefix(default: $(pwd)/milvus) +-d: db path(default: /opt/milvus) +-l: build license version(default: OFF) +-r: remove previous build directory(default: OFF) +-c: code coverage(default: OFF) usage: ./build.sh -t \${BUILD_TYPE} [-u] [-h] [-g] [-r] [-c] @@ -71,6 +76,7 @@ if [[ ${MAKE_CLEAN} == "ON" ]]; then -DCMAKE_CUDA_COMPILER=${CUDA_COMPILER} \ -DCMAKE_LICENSE_CHECK=${LICENSE_CHECK} \ -DBUILD_COVERAGE=${BUILD_COVERAGE} \ + -DMILVUS_DB_PATH=${DB_PATH} \ $@ ../" echo ${CMAKE_CMD} diff --git a/cpp/conf/log_config.conf b/cpp/conf/log_config.conf index 80710b570e..29d46a7fe5 100644 --- a/cpp/conf/log_config.conf +++ b/cpp/conf/log_config.conf @@ -20,8 +20,8 @@ TO_STANDARD_OUTPUT = false ## Error logs * ERROR: - ENABLED = false + ENABLED = true FILENAME = "/tmp/milvus/logs/milvus-%datetime{%H:%m}-error.log" * FATAL: - ENABLED = false - FILENAME = "/tmp/milvus/logs/milvus-%datetime{%H:%m}-fatal.log" \ No newline at end of file + ENABLED = true + FILENAME = "/tmp/milvus/logs/milvus-%datetime{%H:%m}-fatal.log" diff --git a/cpp/conf/log_config.template b/cpp/conf/log_config.template new file mode 100644 index 0000000000..f4f3d3684c --- /dev/null +++ b/cpp/conf/log_config.template @@ -0,0 +1,27 @@ +* GLOBAL: + FORMAT = "%datetime | %level | %logger | %msg" + FILENAME = "@MILVUS_DB_PATH@/logs/milvus-%datetime{%H:%m}-global.log" + ENABLED = true + TO_FILE = true + TO_STANDARD_OUTPUT = false + SUBSECOND_PRECISION = 3 + PERFORMANCE_TRACKING = false + MAX_LOG_FILE_SIZE = 2097152 ## Throw log files away after 2MB +* DEBUG: + FILENAME = "@MILVUS_DB_PATH@/logs/milvus-%datetime{%H:%m}-debug.log" + ENABLED = true +* WARNING: + FILENAME = "@MILVUS_DB_PATH@/logs/milvus-%datetime{%H:%m}-warning.log" +* TRACE: + FILENAME = "@MILVUS_DB_PATH@/logs/milvus-%datetime{%H:%m}-trace.log" +* VERBOSE: + FORMAT = "%datetime{%d/%M/%y} | %level-%vlevel | %msg" + TO_FILE = false + TO_STANDARD_OUTPUT = false +## Error logs +* ERROR: + ENABLED = true + FILENAME = "@MILVUS_DB_PATH@/logs/milvus-%datetime{%H:%m}-error.log" +* FATAL: + ENABLED = true + FILENAME = "@MILVUS_DB_PATH@/logs/milvus-%datetime{%H:%m}-fatal.log" \ No newline at end of file diff --git a/cpp/conf/server_config.template b/cpp/conf/server_config.template new file mode 100644 index 0000000000..1a1c8303f2 --- /dev/null +++ b/cpp/conf/server_config.template @@ -0,0 +1,28 @@ +server_config: + address: 0.0.0.0 + port: 19530 # the port milvus listen to, default: 19530, range: 1025 ~ 65534 + gpu_index: 0 # the gpu milvus use, default: 0, range: 0 ~ gpu number - 1 + mode: single # milvus deployment type: single, cluster + +db_config: + db_path: @MILVUS_DB_PATH@ # milvus data storage path + db_backend_url: http://127.0.0.1 # meta database uri + index_building_threshold: 1024 # index building trigger threshold, default: 1024, unit: MB + archive_disk_threshold: 512 # triger archive action if storage size exceed this value, unit: GB + archive_days_threshold: 30 # files older than x days will be archived, unit: day + +metric_config: + is_startup: off # if monitoring start: on, off + collector: prometheus # metrics collector: prometheus + prometheus_config: # following are prometheus configure + collect_type: pull # prometheus collect data method + port: 8080 # the port prometheus use to fetch metrics + push_gateway_ip_address: 127.0.0.1 # push method configure: push gateway ip address + push_gateway_port: 9091 # push method configure: push gateway port + + +license_config: # license configure + license_path: "@MILVUS_DB_PATH@/system.license" # license file path + +cache_config: # cache configure + cpu_cache_capacity: 16 # how many memory are used as cache, unit: GB, range: 0 ~ less than total memory \ No newline at end of file diff --git a/cpp/conf/server_config.yaml b/cpp/conf/server_config.yaml index 1c26c0aa52..9019461940 100644 --- a/cpp/conf/server_config.yaml +++ b/cpp/conf/server_config.yaml @@ -22,7 +22,7 @@ metric_config: license_config: # license configure - license_path: "/tmp/system.license" # license file path + license_path: "/tmp/milvus/system.license" # license file path cache_config: # cache configure - cpu_cache_capacity: 16 # how many memory are used as cache, unit: GB, range: 0 ~ less than total memory \ No newline at end of file + cpu_cache_capacity: 16 # how many memory are used as cache, unit: GB, range: 0 ~ less than total memory diff --git a/cpp/src/sdk/src/client/ClientProxy.cpp b/cpp/src/sdk/src/client/ClientProxy.cpp index 1185e4988c..35c774b52b 100644 --- a/cpp/src/sdk/src/client/ClientProxy.cpp +++ b/cpp/src/sdk/src/client/ClientProxy.cpp @@ -4,7 +4,6 @@ * Proprietary and confidential. ******************************************************************************/ #include "ClientProxy.h" -#include "util/ConvertUtil.h" namespace milvus { diff --git a/cpp/src/sdk/src/util/ConvertUtil.cpp b/cpp/src/sdk/src/util/ConvertUtil.cpp deleted file mode 100644 index 867221dc89..0000000000 --- a/cpp/src/sdk/src/util/ConvertUtil.cpp +++ /dev/null @@ -1,44 +0,0 @@ -/******************************************************************************* - * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved - * Unauthorized copying of this file, via any medium is strictly prohibited. - * Proprietary and confidential. - ******************************************************************************/ -#include "ConvertUtil.h" -#include "Exception.h" - -#include - -namespace milvus { - -static const std::string INDEX_RAW = "raw"; -static const std::string INDEX_IVFFLAT = "ivfflat"; - -std::string ConvertUtil::IndexType2Str(IndexType index) { - static const std::map s_index2str = { - {IndexType::cpu_idmap, INDEX_RAW}, - {IndexType::gpu_ivfflat, INDEX_IVFFLAT} - }; - - const auto& iter = s_index2str.find(index); - if(iter == s_index2str.end()) { - throw Exception(StatusCode::InvalidAgument, "Invalid index type"); - } - - return iter->second; -} - -IndexType ConvertUtil::Str2IndexType(const std::string& type) { - static const std::map s_str2index = { - {INDEX_RAW, IndexType::cpu_idmap}, - {INDEX_IVFFLAT, IndexType::gpu_ivfflat} - }; - - const auto& iter = s_str2index.find(type); - if(iter == s_str2index.end()) { - throw Exception(StatusCode::InvalidAgument, "Invalid index type"); - } - - return iter->second; -} - -} \ No newline at end of file diff --git a/cpp/src/sdk/src/util/ConvertUtil.h b/cpp/src/sdk/src/util/ConvertUtil.h deleted file mode 100644 index 1411f3eb6d..0000000000 --- a/cpp/src/sdk/src/util/ConvertUtil.h +++ /dev/null @@ -1,18 +0,0 @@ -/******************************************************************************* - * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved - * Unauthorized copying of this file, via any medium is strictly prohibited. - * Proprietary and confidential. - ******************************************************************************/ -#pragma once - -#include "MilvusApi.h" - -namespace milvus { - -class ConvertUtil { -public: - static std::string IndexType2Str(IndexType index); - static IndexType Str2IndexType(const std::string& type); -}; - -} diff --git a/cpp/src/server/DBWrapper.cpp b/cpp/src/server/DBWrapper.cpp index 2a583a660d..7892a57f2b 100644 --- a/cpp/src/server/DBWrapper.cpp +++ b/cpp/src/server/DBWrapper.cpp @@ -37,12 +37,16 @@ DBWrapper::DBWrapper() { opt.meta.archive_conf.SetCriterias(criterial); //create db root folder - CommonUtil::CreateDirectory(opt.meta.path); + ServerError err = CommonUtil::CreateDirectory(opt.meta.path); + if(err != SERVER_SUCCESS) { + std::cout << "ERROR! Failed to create database root path: " << opt.meta.path << std::endl; + kill(0, SIGUSR1); + } zilliz::milvus::engine::DB::Open(opt, &db_); if(db_ == nullptr) { - SERVER_LOG_ERROR << "Failed to open db"; - throw ServerException(SERVER_NULL_POINTER, "Failed to open db"); + std::cout << "ERROR! Failed to open database" << std::endl; + kill(0, SIGUSR1); } } diff --git a/cpp/src/server/MilvusServer.cpp b/cpp/src/server/MilvusServer.cpp index 5e42eb3d7a..322460968f 100644 --- a/cpp/src/server/MilvusServer.cpp +++ b/cpp/src/server/MilvusServer.cpp @@ -23,6 +23,7 @@ #include #include +#include namespace zilliz { namespace milvus { @@ -92,7 +93,8 @@ MilvusServer::StartService() { return; } } catch (apache::thrift::TException& ex) { - //SERVER_LOG_ERROR << "Server encounter exception: " << ex.what(); + std::cout << "ERROR! " << ex.what() << std::endl; + kill(0, SIGUSR1); } } diff --git a/cpp/src/server/RequestTask.h b/cpp/src/server/RequestTask.h index 3061b3b75d..4bf9f964e0 100644 --- a/cpp/src/server/RequestTask.h +++ b/cpp/src/server/RequestTask.h @@ -7,7 +7,6 @@ #include "RequestScheduler.h" #include "utils/Error.h" -#include "utils/AttributeSerializer.h" #include "db/Types.h" #include "milvus_types.h" diff --git a/cpp/src/storage/s3/S3ClientWrapper.cpp b/cpp/src/storage/s3/S3ClientWrapper.cpp index fcbab4f9bd..b739bc1e28 100644 --- a/cpp/src/storage/s3/S3ClientWrapper.cpp +++ b/cpp/src/storage/s3/S3ClientWrapper.cpp @@ -1,159 +1,159 @@ -#include "S3ClientWrapper.h" - -#include -#include -#include -#include -#include - -#include -#include - - -namespace zilliz { -namespace milvus { -namespace engine { -namespace storage { - -Status -S3ClientWrapper::Create(const std::string &ip_address, - const std::string &port, - const std::string &access_key, - const std::string &secret_key) { - Aws::InitAPI(options_); - Aws::Client::ClientConfiguration cfg; - - // TODO: ip_address need to be validated. - - cfg.endpointOverride = ip_address + ":" + port; // S3 server ip address and port - cfg.scheme = Aws::Http::Scheme::HTTP; - cfg.verifySSL = - false; //Aws::Auth::AWSCredentials cred("RPW421T9GSIO4A45Y9ZR", "2owKYy9emSS90Q0pXuyqpX1OxBCyEDYodsiBemcq"); // 认证的Key - client_ = - new S3Client(Aws::Auth::AWSCredentials(access_key, secret_key), - cfg, - Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Always, - false); - if (client_ == nullptr) { - std::string error = "Can't connect server."; - return Status::Error(error); - } else { - return Status::OK(); - } -} - - -Status -S3ClientWrapper::Close() { - if (client_ != nullptr) { - delete client_; - client_ = nullptr; - } - Aws::ShutdownAPI(options_); - return Status::OK(); -} - -Status -S3ClientWrapper::CreateBucket(std::string& bucket_name) { - Aws::S3::Model::CreateBucketRequest request; - request.SetBucket(bucket_name); - - auto outcome = client_->CreateBucket(request); - - if (outcome.IsSuccess()) - { - return Status::OK(); - } - else - { - std::cout << "CreateBucket error: " - << outcome.GetError().GetExceptionName() << std::endl - << outcome.GetError().GetMessage() << std::endl; - switch(outcome.GetError().GetErrorType()) { - case Aws::S3::S3Errors::BUCKET_ALREADY_EXISTS: - case Aws::S3::S3Errors::BUCKET_ALREADY_OWNED_BY_YOU: - return Status::AlreadyExist(outcome.GetError().GetMessage()); - default: - return Status::Error(outcome.GetError().GetMessage()); - } - } -} - -Status -S3ClientWrapper::DeleteBucket(std::string& bucket_name) { - Aws::S3::Model::DeleteBucketRequest bucket_request; - bucket_request.SetBucket(bucket_name); - - auto outcome = client_->DeleteBucket(bucket_request); - - if (outcome.IsSuccess()) - { - return Status::OK(); - } - else - { - std::cout << "DeleteBucket error: " - << outcome.GetError().GetExceptionName() << " - " - << outcome.GetError().GetMessage() << std::endl; - return Status::Error(outcome.GetError().GetMessage()); - } -} - -Status -S3ClientWrapper::UploadFile(std::string &BucketName, std::string &objectKey, std::string &pathkey) { - - PutObjectRequest putObjectRequest; - putObjectRequest.WithBucket(BucketName.c_str()).WithKey(objectKey.c_str()); - - auto input_data = Aws::MakeShared("PutObjectInputStream", - pathkey.c_str(), - std::ios_base::in | std::ios_base::binary); - putObjectRequest.SetBody(input_data); - auto put_object_result = client_->PutObject(putObjectRequest); - if (put_object_result.IsSuccess()) { - return Status::OK(); - } else { - std::cout << "PutObject error: " << put_object_result.GetError().GetExceptionName() << " " - << put_object_result.GetError().GetMessage() << std::endl; - return Status::Error(put_object_result.GetError().GetMessage()); - } -} - -Status -S3ClientWrapper::DownloadFile(std::string &BucketName, std::string &objectKey, std::string &pathkey) { - GetObjectRequest object_request; - object_request.WithBucket(BucketName.c_str()).WithKey(objectKey.c_str()); - auto get_object_outcome = client_->GetObject(object_request); - if (get_object_outcome.IsSuccess()) { - Aws::OFStream local_file(pathkey.c_str(), std::ios::out | std::ios::binary); - local_file << get_object_outcome.GetResult().GetBody().rdbuf(); - return Status::OK(); - } else { - std::cout << "GetObject error: " << get_object_outcome.GetError().GetExceptionName() << " " - << get_object_outcome.GetError().GetMessage() << std::endl; - return Status::Error(get_object_outcome.GetError().GetMessage()); - } -} - -Status -S3ClientWrapper::DeleteFile(std::string &bucket_name, std::string &object_key) { - Aws::S3::Model::DeleteObjectRequest object_request; - object_request.WithBucket(bucket_name).WithKey(object_key); - - auto delete_object_outcome = client_->DeleteObject(object_request); - - if (delete_object_outcome.IsSuccess()) { - return Status::OK(); - } else { - std::cout << "DeleteObject error: " << - delete_object_outcome.GetError().GetExceptionName() << " " << - delete_object_outcome.GetError().GetMessage() << std::endl; - - return Status::Error(delete_object_outcome.GetError().GetMessage()); - } -} - -} -} -} -} \ No newline at end of file +//#include "S3ClientWrapper.h" +// +//#include +//#include +//#include +//#include +//#include +// +//#include +//#include +// +// +//namespace zilliz { +//namespace milvus { +//namespace engine { +//namespace storage { +// +//Status +//S3ClientWrapper::Create(const std::string &ip_address, +// const std::string &port, +// const std::string &access_key, +// const std::string &secret_key) { +// Aws::InitAPI(options_); +// Aws::Client::ClientConfiguration cfg; +// +// // TODO: ip_address need to be validated. +// +// cfg.endpointOverride = ip_address + ":" + port; // S3 server ip address and port +// cfg.scheme = Aws::Http::Scheme::HTTP; +// cfg.verifySSL = +// false; //Aws::Auth::AWSCredentials cred("RPW421T9GSIO4A45Y9ZR", "2owKYy9emSS90Q0pXuyqpX1OxBCyEDYodsiBemcq"); // 认证的Key +// client_ = +// new S3Client(Aws::Auth::AWSCredentials(access_key, secret_key), +// cfg, +// Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Always, +// false); +// if (client_ == nullptr) { +// std::string error = "Can't connect server."; +// return Status::Error(error); +// } else { +// return Status::OK(); +// } +//} +// +// +//Status +//S3ClientWrapper::Close() { +// if (client_ != nullptr) { +// delete client_; +// client_ = nullptr; +// } +// Aws::ShutdownAPI(options_); +// return Status::OK(); +//} +// +//Status +//S3ClientWrapper::CreateBucket(std::string& bucket_name) { +// Aws::S3::Model::CreateBucketRequest request; +// request.SetBucket(bucket_name); +// +// auto outcome = client_->CreateBucket(request); +// +// if (outcome.IsSuccess()) +// { +// return Status::OK(); +// } +// else +// { +// std::cout << "CreateBucket error: " +// << outcome.GetError().GetExceptionName() << std::endl +// << outcome.GetError().GetMessage() << std::endl; +// switch(outcome.GetError().GetErrorType()) { +// case Aws::S3::S3Errors::BUCKET_ALREADY_EXISTS: +// case Aws::S3::S3Errors::BUCKET_ALREADY_OWNED_BY_YOU: +// return Status::AlreadyExist(outcome.GetError().GetMessage()); +// default: +// return Status::Error(outcome.GetError().GetMessage()); +// } +// } +//} +// +//Status +//S3ClientWrapper::DeleteBucket(std::string& bucket_name) { +// Aws::S3::Model::DeleteBucketRequest bucket_request; +// bucket_request.SetBucket(bucket_name); +// +// auto outcome = client_->DeleteBucket(bucket_request); +// +// if (outcome.IsSuccess()) +// { +// return Status::OK(); +// } +// else +// { +// std::cout << "DeleteBucket error: " +// << outcome.GetError().GetExceptionName() << " - " +// << outcome.GetError().GetMessage() << std::endl; +// return Status::Error(outcome.GetError().GetMessage()); +// } +//} +// +//Status +//S3ClientWrapper::UploadFile(std::string &BucketName, std::string &objectKey, std::string &pathkey) { +// +// PutObjectRequest putObjectRequest; +// putObjectRequest.WithBucket(BucketName.c_str()).WithKey(objectKey.c_str()); +// +// auto input_data = Aws::MakeShared("PutObjectInputStream", +// pathkey.c_str(), +// std::ios_base::in | std::ios_base::binary); +// putObjectRequest.SetBody(input_data); +// auto put_object_result = client_->PutObject(putObjectRequest); +// if (put_object_result.IsSuccess()) { +// return Status::OK(); +// } else { +// std::cout << "PutObject error: " << put_object_result.GetError().GetExceptionName() << " " +// << put_object_result.GetError().GetMessage() << std::endl; +// return Status::Error(put_object_result.GetError().GetMessage()); +// } +//} +// +//Status +//S3ClientWrapper::DownloadFile(std::string &BucketName, std::string &objectKey, std::string &pathkey) { +// GetObjectRequest object_request; +// object_request.WithBucket(BucketName.c_str()).WithKey(objectKey.c_str()); +// auto get_object_outcome = client_->GetObject(object_request); +// if (get_object_outcome.IsSuccess()) { +// Aws::OFStream local_file(pathkey.c_str(), std::ios::out | std::ios::binary); +// local_file << get_object_outcome.GetResult().GetBody().rdbuf(); +// return Status::OK(); +// } else { +// std::cout << "GetObject error: " << get_object_outcome.GetError().GetExceptionName() << " " +// << get_object_outcome.GetError().GetMessage() << std::endl; +// return Status::Error(get_object_outcome.GetError().GetMessage()); +// } +//} +// +//Status +//S3ClientWrapper::DeleteFile(std::string &bucket_name, std::string &object_key) { +// Aws::S3::Model::DeleteObjectRequest object_request; +// object_request.WithBucket(bucket_name).WithKey(object_key); +// +// auto delete_object_outcome = client_->DeleteObject(object_request); +// +// if (delete_object_outcome.IsSuccess()) { +// return Status::OK(); +// } else { +// std::cout << "DeleteObject error: " << +// delete_object_outcome.GetError().GetExceptionName() << " " << +// delete_object_outcome.GetError().GetMessage() << std::endl; +// +// return Status::Error(delete_object_outcome.GetError().GetMessage()); +// } +//} +// +//} +//} +//} +//} \ No newline at end of file diff --git a/cpp/src/storage/s3/S3ClientWrapper.h b/cpp/src/storage/s3/S3ClientWrapper.h index 9ef60174cf..fc66304a5a 100644 --- a/cpp/src/storage/s3/S3ClientWrapper.h +++ b/cpp/src/storage/s3/S3ClientWrapper.h @@ -1,45 +1,45 @@ -#pragma once - -#include "storage/IStorage.h" - - -#include -#include -#include - - -using namespace Aws::S3; -using namespace Aws::S3::Model; - -namespace zilliz { -namespace milvus { -namespace engine { -namespace storage { - -class S3ClientWrapper : public IStorage { - public: - - S3ClientWrapper() = default; - ~S3ClientWrapper() = default; - - Status Create(const std::string &ip_address, - const std::string &port, - const std::string &access_key, - const std::string &secret_key) override; - Status Close() override; - - Status CreateBucket(std::string& bucket_name) override; - Status DeleteBucket(std::string& bucket_name) override; - Status UploadFile(std::string &BucketName, std::string &objectKey, std::string &pathkey) override; - Status DownloadFile(std::string &BucketName, std::string &objectKey, std::string &pathkey) override; - Status DeleteFile(std::string &bucket_name, std::string &object_key) override; - - private: - S3Client *client_ = nullptr; - Aws::SDKOptions options_; -}; - -} -} -} -} \ No newline at end of file +//#pragma once +// +//#include "storage/IStorage.h" +// +// +//#include +//#include +//#include +// +// +//using namespace Aws::S3; +//using namespace Aws::S3::Model; +// +//namespace zilliz { +//namespace milvus { +//namespace engine { +//namespace storage { +// +//class S3ClientWrapper : public IStorage { +// public: +// +// S3ClientWrapper() = default; +// ~S3ClientWrapper() = default; +// +// Status Create(const std::string &ip_address, +// const std::string &port, +// const std::string &access_key, +// const std::string &secret_key) override; +// Status Close() override; +// +// Status CreateBucket(std::string& bucket_name) override; +// Status DeleteBucket(std::string& bucket_name) override; +// Status UploadFile(std::string &BucketName, std::string &objectKey, std::string &pathkey) override; +// Status DownloadFile(std::string &BucketName, std::string &objectKey, std::string &pathkey) override; +// Status DeleteFile(std::string &bucket_name, std::string &object_key) override; +// +// private: +// S3Client *client_ = nullptr; +// Aws::SDKOptions options_; +//}; +// +//} +//} +//} +//} \ No newline at end of file diff --git a/cpp/src/utils/AttributeSerializer.cpp b/cpp/src/utils/AttributeSerializer.cpp deleted file mode 100644 index b351262e60..0000000000 --- a/cpp/src/utils/AttributeSerializer.cpp +++ /dev/null @@ -1,50 +0,0 @@ -/******************************************************************************* - * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved - * Unauthorized copying of this file, via any medium is strictly prohibited. - * Proprietary and confidential. - ******************************************************************************/ - -#include "AttributeSerializer.h" -#include "StringHelpFunctions.h" - -namespace zilliz { -namespace milvus { -namespace server { - - -ServerError AttributeSerializer::Encode(const AttribMap& attrib_map, std::string& attrib_str) { - attrib_str = ""; - for(auto iter : attrib_map) { - attrib_str += iter.first; - attrib_str += ":\""; - attrib_str += iter.second; - attrib_str += "\";"; - } - - return SERVER_SUCCESS; -} - -ServerError AttributeSerializer::Decode(const std::string& attrib_str, AttribMap& attrib_map) { - attrib_map.clear(); - - std::vector kv_pairs; - StringHelpFunctions::SplitStringByQuote(attrib_str, ";", "\"", kv_pairs); - for(std::string& str : kv_pairs) { - std::string key, val; - size_t index = str.find_first_of(":", 0); - if (index != std::string::npos) { - key = str.substr(0, index); - val = str.substr(index + 1); - } else { - key = str; - } - - attrib_map.insert(std::make_pair(key, val)); - } - - return SERVER_SUCCESS; -} - -} -} -} diff --git a/cpp/src/utils/AttributeSerializer.h b/cpp/src/utils/AttributeSerializer.h deleted file mode 100644 index 8d0341aa2e..0000000000 --- a/cpp/src/utils/AttributeSerializer.h +++ /dev/null @@ -1,27 +0,0 @@ -/******************************************************************************* - * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved - * Unauthorized copying of this file, via any medium is strictly prohibited. - * Proprietary and confidential. - ******************************************************************************/ -#pragma once - -#include - -#include "Error.h" - -namespace zilliz { -namespace milvus { -namespace server { - -using AttribMap = std::map; - -class AttributeSerializer { -public: - static ServerError Encode(const AttribMap& attrib_map, std::string& attrib_str); - static ServerError Decode(const std::string& attrib_str, AttribMap& attrib_map); -}; - - -} -} -} diff --git a/cpp/src/utils/StringHelpFunctions.cpp b/cpp/src/utils/StringHelpFunctions.cpp index aaefd7236a..36ba017beb 100644 --- a/cpp/src/utils/StringHelpFunctions.cpp +++ b/cpp/src/utils/StringHelpFunctions.cpp @@ -9,13 +9,6 @@ namespace zilliz { namespace milvus { namespace server { -void StringHelpFunctions::TrimStringLineBreak(std::string &string) { - if (!string.empty()) { - static std::string s_format("\n\r"); - string.erase(string.find_last_not_of(s_format) + 1); - } -} - void StringHelpFunctions::TrimStringBlank(std::string &string) { if (!string.empty()) { static std::string s_format(" \n\r\t"); diff --git a/cpp/src/utils/StringHelpFunctions.h b/cpp/src/utils/StringHelpFunctions.h index 2521c48e22..b5d8ce22b7 100644 --- a/cpp/src/utils/StringHelpFunctions.h +++ b/cpp/src/utils/StringHelpFunctions.h @@ -18,8 +18,6 @@ private: StringHelpFunctions() = default; public: - static void TrimStringLineBreak(std::string &string); - static void TrimStringBlank(std::string &string); static void TrimStringQuote(std::string &string, const std::string &qoute); diff --git a/cpp/unittest/server/CMakeLists.txt b/cpp/unittest/server/CMakeLists.txt index d844b7a9a3..4d34f0e2b7 100644 --- a/cpp/unittest/server/CMakeLists.txt +++ b/cpp/unittest/server/CMakeLists.txt @@ -14,9 +14,10 @@ aux_source_directory(../../src/cache cache_srcs) aux_source_directory(../../src/wrapper wrapper_src) aux_source_directory(./ test_srcs) -set(server_src_files +set(utils_srcs ${MILVUS_ENGINE_SRC}/utils/StringHelpFunctions.cpp - ${MILVUS_ENGINE_SRC}/utils/AttributeSerializer.cpp + ${MILVUS_ENGINE_SRC}/utils/TimeRecorder.cpp + ${MILVUS_ENGINE_SRC}/utils/CommonUtil.cpp ) cuda_add_executable(server_test @@ -25,7 +26,7 @@ cuda_add_executable(server_test ${cache_srcs} ${wrapper_src} ${test_srcs} - ${server_src_files} + ${utils_srcs} ${require_files} ) diff --git a/cpp/unittest/server/cache_test.cpp b/cpp/unittest/server/cache_test.cpp index 98c06f788c..05c292b11b 100644 --- a/cpp/unittest/server/cache_test.cpp +++ b/cpp/unittest/server/cache_test.cpp @@ -11,18 +11,18 @@ using namespace zilliz::milvus; -TEST(CacheTest, CACHE_TEST) { - cache::CacheMgr* cpu_mgr = cache::CpuCacheMgr::GetInstance(); +TEST(CacheTest, CPU_CACHE_TEST) { + cache::CacheMgr *cpu_mgr = cache::CpuCacheMgr::GetInstance(); const int64_t gbyte = 1 << 30; int64_t g_num = 16; - int64_t cap = g_num*gbyte; + int64_t cap = g_num * gbyte; cpu_mgr->SetCapacity(cap); ASSERT_EQ(cpu_mgr->CacheCapacity(), cap); const int dim = 256; - for(int i = 0; i < 20; i++) { + for (int i = 0; i < 20; i++) { std::shared_ptr raw_index(faiss::index_factory(dim, "IDMap,Flat")); engine::Index_ptr index = std::make_shared(raw_index); index->ntotal = 1000000;//less 1G per index @@ -31,6 +31,12 @@ TEST(CacheTest, CACHE_TEST) { } ASSERT_LT(cpu_mgr->ItemCount(), g_num); + auto obj = cpu_mgr->GetIndex("index_0"); + ASSERT_TRUE(obj == nullptr); + + obj = cpu_mgr->GetIndex("index_19"); + ASSERT_TRUE(obj != nullptr); + { std::string item = "index_15"; ASSERT_TRUE(cpu_mgr->ItemExists(item)); @@ -49,4 +55,27 @@ TEST(CacheTest, CACHE_TEST) { cpu_mgr->InsertItem("index_6g", index); ASSERT_EQ(cpu_mgr->ItemCount(), 0);//data greater than capacity can not be inserted sucessfully } + + cpu_mgr->PrintInfo(); +} + +TEST(CacheTest, GPU_CACHE_TEST) { + cache::CacheMgr* gpu_mgr = cache::GpuCacheMgr::GetInstance(); + + const int dim = 256; + + for(int i = 0; i < 20; i++) { + std::shared_ptr raw_index(faiss::index_factory(dim, "IDMap,Flat")); + engine::Index_ptr index = std::make_shared(raw_index); + index->ntotal = 1000; + + cache::DataObjPtr obj = std::make_shared(index); + + gpu_mgr->InsertItem("index_" + std::to_string(i), obj); + } + + auto obj = gpu_mgr->GetItem("index_0"); + + gpu_mgr->ClearCache(); + ASSERT_EQ(gpu_mgr->ItemCount(), 0); } \ No newline at end of file diff --git a/cpp/unittest/server/common_test.cpp b/cpp/unittest/server/common_test.cpp deleted file mode 100644 index 27d8c522e5..0000000000 --- a/cpp/unittest/server/common_test.cpp +++ /dev/null @@ -1,26 +0,0 @@ -//////////////////////////////////////////////////////////////////////////////// -// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved -// Unauthorized copying of this file, via any medium is strictly prohibited. -// Proprietary and confidential. -//////////////////////////////////////////////////////////////////////////////// -#include -#include "utils/CommonUtil.h" -#include "utils/Error.h" - -using namespace zilliz::milvus; - - -TEST(CommonTest, COMMON_TEST) { - std::string path1 = "/tmp/milvus_test/"; - std::string path2 = path1 + "common_test_12345/"; - std::string path3 = path2 + "abcdef"; - server::ServerError err = server::CommonUtil::CreateDirectory(path3); - ASSERT_EQ(err, server::SERVER_SUCCESS); - - ASSERT_TRUE(server::CommonUtil::IsDirectoryExit(path3)); - - err = server::CommonUtil::DeleteDirectory(path1); - ASSERT_EQ(err, server::SERVER_SUCCESS); - - ASSERT_FALSE(server::CommonUtil::IsDirectoryExit(path1)); -} diff --git a/cpp/unittest/server/config_test.cpp b/cpp/unittest/server/config_test.cpp new file mode 100644 index 0000000000..4f76707dd2 --- /dev/null +++ b/cpp/unittest/server/config_test.cpp @@ -0,0 +1,77 @@ +//////////////////////////////////////////////////////////////////////////////// +// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved +// Unauthorized copying of this file, via any medium is strictly prohibited. +// Proprietary and confidential. +//////////////////////////////////////////////////////////////////////////////// +#include + +#include "config/IConfigMgr.h" +#include "server/ServerConfig.h" + +using namespace zilliz::milvus; + +static const std::string CONFIG_FILE_PATH = "../../../conf/server_config.yaml"; + +TEST(ConfigTest, CONFIG_TEST) { + server::IConfigMgr* config_mgr = server::IConfigMgr::GetInstance(); + server::ServerError err = config_mgr->LoadConfigFile(CONFIG_FILE_PATH); + ASSERT_EQ(err, server::SERVER_SUCCESS); + + server::ConfigNode& root_config = config_mgr->GetRootNode(); + server::ConfigNode& server_config = root_config.GetChild("server_config"); + server::ConfigNode& db_config = root_config.GetChild("db_config"); + server::ConfigNode& metric_config = root_config.GetChild("metric_config"); + server::ConfigNode& cache_config = root_config.GetChild("cache_config"); + + std::string address = server_config.GetValue("address"); + ASSERT_TRUE(!address.empty()); + int64_t port = server_config.GetInt64Value("port"); + ASSERT_TRUE(port != 0); + + server_config.SetValue("test", "2.5"); + double test = server_config.GetDoubleValue("test"); + ASSERT_EQ(test, 2.5); + + server::ConfigNode fake; + server_config.AddChild("fake", fake); + fake = server_config.GetChild("fake"); + server::ConfigNodeArr arr; + server_config.GetChildren(arr); + ASSERT_EQ(arr.size(), 1UL); + + server_config.ClearChildren(); + auto children = server_config.GetChildren(); + ASSERT_TRUE(children.empty()); + + root_config.PrintAll(); + std::string all = root_config.DumpString(); + ASSERT_TRUE(!all.empty()); + + server_config.ClearConfig(); + auto configs = server_config.GetConfig(); + ASSERT_TRUE(configs.empty()); + + server_config.AddSequenceItem("seq", "aaa"); + server_config.AddSequenceItem("seq", "bbb"); + auto seq = server_config.GetSequence("seq"); + ASSERT_EQ(seq.size(), 2UL); + + server_config.ClearSequences(); + auto seqs = server_config.GetSequences(); + ASSERT_TRUE(seqs.empty()); +} + +TEST(ConfigTest, SERVER_CONFIG_TEST) { + server::ServerConfig& config = server::ServerConfig::GetInstance(); + server::ServerError err = config.LoadConfigFile(CONFIG_FILE_PATH); + ASSERT_EQ(err, server::SERVER_SUCCESS); + + server::ConfigNode node1 = config.GetConfig("server_config"); + server::ConfigNode& node2 = config.GetConfig("cache_config"); + node1.Combine(node2); + + int32_t cap = node1.GetInt32Value("cpu_cache_capacity"); + ASSERT_GT(cap, 0); + + config.PrintAll(); +} \ No newline at end of file diff --git a/cpp/unittest/server/util_test.cpp b/cpp/unittest/server/util_test.cpp index 0bc951d88e..9a38092799 100644 --- a/cpp/unittest/server/util_test.cpp +++ b/cpp/unittest/server/util_test.cpp @@ -4,31 +4,99 @@ // Proprietary and confidential. //////////////////////////////////////////////////////////////////////////////// #include +#include -#include "utils/AttributeSerializer.h" +#include "utils/CommonUtil.h" +#include "utils/Error.h" #include "utils/StringHelpFunctions.h" +#include "utils/TimeRecorder.h" +#include "utils/BlockingQueue.h" using namespace zilliz::milvus; -TEST(AttribSerializeTest, ATTRIBSERIAL_TEST) { - std::map attrib; - attrib["uid"] = "ABCDEF"; - attrib["color"] = "red"; - attrib["number"] = "9900"; - attrib["comment"] = "please note: it is a car, not a ship"; - attrib["address"] = " china;shanghai "; +namespace { - std::string attri_str; - server::AttributeSerializer::Encode(attrib, attri_str); - - std::map attrib_out; - server::ServerError err = server::AttributeSerializer::Decode(attri_str, attrib_out); - ASSERT_EQ(err, server::SERVER_SUCCESS); - - ASSERT_EQ(attrib_out.size(), attrib.size()); - for(auto iter : attrib) { - ASSERT_EQ(attrib_out[iter.first], attrib_out[iter.first]); - } +using TimeUnit = server::TimeRecorder::TimeDisplayUnit; +double TestTimeRecorder(TimeUnit unit, int64_t log_level) { + server::TimeRecorder rc("test rc", unit, log_level); + rc.Record("begin"); + std::this_thread::sleep_for(std::chrono::microseconds(10)); + rc.Elapse("end"); + return rc.Span(); +} } +TEST(CommonTest, COMMON_TEST) { + std::string path1 = "/tmp/milvus_test/"; + std::string path2 = path1 + "common_test_12345/"; + std::string path3 = path2 + "abcdef"; + server::ServerError err = server::CommonUtil::CreateDirectory(path3); + ASSERT_EQ(err, server::SERVER_SUCCESS); + + ASSERT_TRUE(server::CommonUtil::IsDirectoryExit(path3)); + + err = server::CommonUtil::DeleteDirectory(path1); + ASSERT_EQ(err, server::SERVER_SUCCESS); + + ASSERT_FALSE(server::CommonUtil::IsDirectoryExit(path1)); +} + +TEST(UtilTest, STRINGFUNCTIONS_TEST) { + std::string str = " test zilliz"; + server::StringHelpFunctions::TrimStringBlank(str); + ASSERT_EQ(str, "test zilliz"); + + str = "a,b,c"; + std::vector result; + server::StringHelpFunctions::SplitStringByDelimeter(str , ",", result); + ASSERT_EQ(result.size(), 3UL); + + str = "55,\"aa,gg,yy\",b"; + result.clear(); + server::StringHelpFunctions::SplitStringByQuote(str , ",", "\"", result); + ASSERT_EQ(result.size(), 3UL); +} + +TEST(UtilTest, TIMERECORDER_TEST) { + double span = TestTimeRecorder(TimeUnit::eTimeAutoUnit, 0); + ASSERT_GT(span, 0.0); + span = TestTimeRecorder(TimeUnit::eTimeHourUnit, 1); + ASSERT_GT(span, 0.0); + span = TestTimeRecorder(TimeUnit::eTimeMinuteUnit, 2); + ASSERT_GT(span, 0.0); + span = TestTimeRecorder(TimeUnit::eTimeSecondUnit, 3); + ASSERT_GT(span, 0.0); + span = TestTimeRecorder(TimeUnit::eTimeMilliSecUnit, 4); + ASSERT_GT(span, 0.0); + span = TestTimeRecorder(TimeUnit::eTimeMicroSecUnit, -1); + ASSERT_GT(span, 0.0); +} + +TEST(UtilTest, BLOCKINGQUEUE_TEST) { + server::BlockingQueue bq; + + static const size_t count = 10; + bq.SetCapacity(count); + + for(size_t i = 1; i <= count; i++) { + std::string id = "No." + std::to_string(i); + bq.Put(id); + } + + ASSERT_EQ(bq.Size(), count); + ASSERT_FALSE(bq.Empty()); + + std::string str = bq.Front(); + ASSERT_EQ(str, "No.1"); + + str = bq.Back(); + ASSERT_EQ(str, "No." + std::to_string(count)); + + for(size_t i = 1; i <= count; i++) { + std::string id = "No." + std::to_string(i); + str = bq.Take(); + ASSERT_EQ(id, str); + } +} + diff --git a/cpp/unittest/storage/S3ClientTest.cpp b/cpp/unittest/storage/S3ClientTest.cpp index c83209ecf6..5147f701ae 100644 --- a/cpp/unittest/storage/S3ClientTest.cpp +++ b/cpp/unittest/storage/S3ClientTest.cpp @@ -1,66 +1,66 @@ -//////////////////////////////////////////////////////////////////////////////// -// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved -// Unauthorized copying of this file, via any medium is strictly prohibited. -// Proprietary and confidential. -//////////////////////////////////////////////////////////////////////////////// - -#include "storage/IStorage.h" -#include "storage/s3/S3ClientWrapper.h" -#include -#include -#include - - -using namespace zilliz::milvus::engine; - -TEST(s3_client_wrapper, CLIENT_WRAPPER_TEST) { - - std::shared_ptr storage_ptr = std::make_shared(); - - std::string ip_address = "127.0.0.1"; - std::string port = "9000"; - std::string access_key = "AKIAIOSFODNN7EXAMPLE"; - std::string secret_key = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"; - - Status status = storage_ptr->Create(ip_address, port, access_key, secret_key); - ASSERT_TRUE(status.ok()); - - std::string filename = "/tmp/s3_test_file"; - std::string bucket_name = "bucktname"; - std::string object_name = "test_file"; - - status = storage_ptr->CreateBucket(bucket_name); - std::cout << status.IsAlreadyExist() << std::endl; - if (status.IsAlreadyExist()) { - status = storage_ptr->DeleteBucket(bucket_name); - status = storage_ptr->CreateBucket(bucket_name); - } - - ASSERT_TRUE(status.ok()); - - std::ofstream ofile(filename); - std::stringstream ss; - for (int i = 0; i < 1024; ++i) { - ss << i; - } - ofile << ss.str() << std::endl; - ofile.close(); - status = storage_ptr->UploadFile(bucket_name, object_name, filename); - ASSERT_TRUE(status.ok()); - - status = storage_ptr->DownloadFile(bucket_name, object_name, filename); - std::ifstream infile(filename); - std::string in_buffer; - infile >> in_buffer; - ASSERT_STREQ(in_buffer.c_str(), ss.str().c_str()); - - status = storage_ptr->DeleteFile(bucket_name, object_name); - ASSERT_TRUE(status.ok()); - - status = storage_ptr->DeleteBucket(bucket_name); - ASSERT_TRUE(status.ok()); - - status = storage_ptr->Close(); - ASSERT_TRUE(status.ok()); -} - +////////////////////////////////////////////////////////////////////////////////// +//// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved +//// Unauthorized copying of this file, via any medium is strictly prohibited. +//// Proprietary and confidential. +////////////////////////////////////////////////////////////////////////////////// +// +//#include "storage/IStorage.h" +//#include "storage/s3/S3ClientWrapper.h" +//#include +//#include +//#include +// +// +//using namespace zilliz::milvus::engine; +// +//TEST(s3_client_wrapper, CLIENT_WRAPPER_TEST) { +// +// std::shared_ptr storage_ptr = std::make_shared(); +// +// std::string ip_address = "127.0.0.1"; +// std::string port = "9000"; +// std::string access_key = "AKIAIOSFODNN7EXAMPLE"; +// std::string secret_key = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"; +// +// Status status = storage_ptr->Create(ip_address, port, access_key, secret_key); +// ASSERT_TRUE(status.ok()); +// +// std::string filename = "/tmp/s3_test_file"; +// std::string bucket_name = "bucktname"; +// std::string object_name = "test_file"; +// +// status = storage_ptr->CreateBucket(bucket_name); +// std::cout << status.IsAlreadyExist() << std::endl; +// if (status.IsAlreadyExist()) { +// status = storage_ptr->DeleteBucket(bucket_name); +// status = storage_ptr->CreateBucket(bucket_name); +// } +// +// ASSERT_TRUE(status.ok()); +// +// std::ofstream ofile(filename); +// std::stringstream ss; +// for (int i = 0; i < 1024; ++i) { +// ss << i; +// } +// ofile << ss.str() << std::endl; +// ofile.close(); +// status = storage_ptr->UploadFile(bucket_name, object_name, filename); +// ASSERT_TRUE(status.ok()); +// +// status = storage_ptr->DownloadFile(bucket_name, object_name, filename); +// std::ifstream infile(filename); +// std::string in_buffer; +// infile >> in_buffer; +// ASSERT_STREQ(in_buffer.c_str(), ss.str().c_str()); +// +// status = storage_ptr->DeleteFile(bucket_name, object_name); +// ASSERT_TRUE(status.ok()); +// +// status = storage_ptr->DeleteBucket(bucket_name); +// ASSERT_TRUE(status.ok()); +// +// status = storage_ptr->Close(); +// ASSERT_TRUE(status.ok()); +//} +//