admin/milvus
1
0
Fork 0
mirror of https://gitee.com/milvus-io/milvus.git synced 2026-01-03 01:12:25 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'main/branch-0.3.0' into branch-0.3.0

Browse Source 
Former-commit-id: 96cc5d821ffb37c0909960834085a71f96cdc3d3
This commit is contained in:
yu yunfeng 2019-06-28 15:37:06 +08:00
commit df08cdbe29
26 changed files with 568 additions and 491 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
cpp/.gitignore vendored
View File

@ -1,4 +1,5 @@
milvus/
conf/server_config.yaml
conf/log_config.conf
version.h
megasearch/

1
cpp/CHANGELOG.md
View File

@ -24,6 +24,7 @@ Please mark all change in change log and use the ticket from JIRA.
- MS-122 - Archive criteria config
- MS-124 - HasTable interface
- MS-126 - Add more error code
- MS-128 - Change default db path
## New Feature

11
cpp/CMakeLists.txt
View File

@ -135,12 +135,19 @@ if (BUILD_COVERAGE STREQUAL "ON")
endif()
if (BUILD_UNIT_TEST)
if ("${BUILD_UNIT_TEST}" STREQUAL "ON")
add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/unittest)
endif(BUILD_UNIT_TEST)
add_custom_target(Clean-All COMMAND ${CMAKE_BUILD_TOOL} clean)
if("${MILVUS_DB_PATH}" STREQUAL "")
set(MILVUS_DB_PATH "/tmp/milvus")
endif()
configure_file(${CMAKE_CURRENT_SOURCE_DIR}/conf/server_config.template ${CMAKE_CURRENT_SOURCE_DIR}/conf/server_config.yaml)
configure_file(${CMAKE_CURRENT_SOURCE_DIR}/conf/log_config.template ${CMAKE_CURRENT_SOURCE_DIR}/conf/log_config.conf)
#install
install(FILES
scripts/start_server.sh
@ -153,4 +160,6 @@ install(FILES
DESTINATION
conf)
config_summary()

24
cpp/build.sh
View File

@ -1,13 +1,14 @@
#!/bin/bash
BUILD_TYPE="Debug"
BUILD_UNITTEST="off"
BUILD_UNITTEST="OFF"
LICENSE_CHECK="OFF"
INSTALL_PREFIX=$(pwd)/milvus
MAKE_CLEAN="OFF"
BUILD_COVERAGE="OFF"
DB_PATH="/opt/milvus"
while getopts "p:t:uhlrc" arg
while getopts "p:d:t:uhlrc" arg
do
case $arg in
t)
@ -15,11 +16,14 @@ do
;;
u)
echo "Build and run unittest cases" ;
BUILD_UNITTEST="on";
BUILD_UNITTEST="ON";
;;
p)
INSTALL_PREFIX=$OPTARG
;;
d)
DB_PATH=$OPTARG
;;
l)
LICENSE_CHECK="ON"
;;
@ -36,12 +40,13 @@ do
echo "
parameter:
-t: build type
-u: building unit test options
-p: install prefix
-l: build license version
-r: remove previous build directory
-c: code coverage
-t: build type(default: Debug)
-u: building unit test options(default: OFF)
-p: install prefix(default: $(pwd)/milvus)
-d: db path(default: /opt/milvus)
-l: build license version(default: OFF)
-r: remove previous build directory(default: OFF)
-c: code coverage(default: OFF)
usage:
./build.sh -t \${BUILD_TYPE} [-u] [-h] [-g] [-r] [-c]
@ -71,6 +76,7 @@ if [[ ${MAKE_CLEAN} == "ON" ]]; then
-DCMAKE_CUDA_COMPILER=${CUDA_COMPILER} \
-DCMAKE_LICENSE_CHECK=${LICENSE_CHECK} \
-DBUILD_COVERAGE=${BUILD_COVERAGE} \
-DMILVUS_DB_PATH=${DB_PATH} \
$@ ../"
echo ${CMAKE_CMD}

6
cpp/conf/log_config.conf
View File

@ -20,8 +20,8 @@
TO_STANDARD_OUTPUT = false
## Error logs
* ERROR:
ENABLED = false
ENABLED = true
FILENAME = "/tmp/milvus/logs/milvus-%datetime{%H:%m}-error.log"
* FATAL:
ENABLED = false
FILENAME = "/tmp/milvus/logs/milvus-%datetime{%H:%m}-fatal.log"
ENABLED = true
FILENAME = "/tmp/milvus/logs/milvus-%datetime{%H:%m}-fatal.log"

27
cpp/conf/log_config.template Normal file
View File

@ -0,0 +1,27 @@
* GLOBAL:
FORMAT = "%datetime | %level | %logger | %msg"
FILENAME = "@MILVUS_DB_PATH@/logs/milvus-%datetime{%H:%m}-global.log"
ENABLED = true
TO_FILE = true
TO_STANDARD_OUTPUT = false
SUBSECOND_PRECISION = 3
PERFORMANCE_TRACKING = false
MAX_LOG_FILE_SIZE = 2097152 ## Throw log files away after 2MB
* DEBUG:
FILENAME = "@MILVUS_DB_PATH@/logs/milvus-%datetime{%H:%m}-debug.log"
ENABLED = true
* WARNING:
FILENAME = "@MILVUS_DB_PATH@/logs/milvus-%datetime{%H:%m}-warning.log"
* TRACE:
FILENAME = "@MILVUS_DB_PATH@/logs/milvus-%datetime{%H:%m}-trace.log"
* VERBOSE:
FORMAT = "%datetime{%d/%M/%y} | %level-%vlevel | %msg"
TO_FILE = false
TO_STANDARD_OUTPUT = false
## Error logs
* ERROR:
ENABLED = true
FILENAME = "@MILVUS_DB_PATH@/logs/milvus-%datetime{%H:%m}-error.log"
* FATAL:
ENABLED = true
FILENAME = "@MILVUS_DB_PATH@/logs/milvus-%datetime{%H:%m}-fatal.log"

28
cpp/conf/server_config.template Normal file
View File

@ -0,0 +1,28 @@
server_config:
address: 0.0.0.0
port: 19530 # the port milvus listen to, default: 19530, range: 1025 ~ 65534
gpu_index: 0 # the gpu milvus use, default: 0, range: 0 ~ gpu number - 1
mode: single # milvus deployment type: single, cluster
db_config:
db_path: @MILVUS_DB_PATH@ # milvus data storage path
db_backend_url: http://127.0.0.1 # meta database uri
index_building_threshold: 1024 # index building trigger threshold, default: 1024, unit: MB
archive_disk_threshold: 512 # triger archive action if storage size exceed this value, unit: GB
archive_days_threshold: 30 # files older than x days will be archived, unit: day
metric_config:
is_startup: off # if monitoring start: on, off
collector: prometheus # metrics collector: prometheus
prometheus_config: # following are prometheus configure
collect_type: pull # prometheus collect data method
port: 8080 # the port prometheus use to fetch metrics
push_gateway_ip_address: 127.0.0.1 # push method configure: push gateway ip address
push_gateway_port: 9091 # push method configure: push gateway port
license_config: # license configure
license_path: "@MILVUS_DB_PATH@/system.license" # license file path
cache_config: # cache configure
cpu_cache_capacity: 16 # how many memory are used as cache, unit: GB, range: 0 ~ less than total memory

4
cpp/conf/server_config.yaml
View File

@ -22,7 +22,7 @@ metric_config:
license_config: # license configure
license_path: "/tmp/system.license" # license file path
license_path: "/tmp/milvus/system.license" # license file path
cache_config: # cache configure
cpu_cache_capacity: 16 # how many memory are used as cache, unit: GB, range: 0 ~ less than total memory
cpu_cache_capacity: 16 # how many memory are used as cache, unit: GB, range: 0 ~ less than total memory

1
cpp/src/sdk/src/client/ClientProxy.cpp
View File

@ -4,7 +4,6 @@
* Proprietary and confidential.
******************************************************************************/
#include "ClientProxy.h"
#include "util/ConvertUtil.h"
namespace milvus {

44
cpp/src/sdk/src/util/ConvertUtil.cpp
View File

@ -1,44 +0,0 @@
/*******************************************************************************
* Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved
* Unauthorized copying of this file, via any medium is strictly prohibited.
* Proprietary and confidential.
******************************************************************************/
#include "ConvertUtil.h"
#include "Exception.h"
#include <map>
namespace milvus {
static const std::string INDEX_RAW = "raw";
static const std::string INDEX_IVFFLAT = "ivfflat";
std::string ConvertUtil::IndexType2Str(IndexType index) {
static const std::map<IndexType, std::string> s_index2str = {
{IndexType::cpu_idmap, INDEX_RAW},
{IndexType::gpu_ivfflat, INDEX_IVFFLAT}
};
const auto& iter = s_index2str.find(index);
if(iter == s_index2str.end()) {
throw Exception(StatusCode::InvalidAgument, "Invalid index type");
}
return iter->second;
}
IndexType ConvertUtil::Str2IndexType(const std::string& type) {
static const std::map<std::string, IndexType> s_str2index = {
{INDEX_RAW, IndexType::cpu_idmap},
{INDEX_IVFFLAT, IndexType::gpu_ivfflat}
};
const auto& iter = s_str2index.find(type);
if(iter == s_str2index.end()) {
throw Exception(StatusCode::InvalidAgument, "Invalid index type");
}
return iter->second;
}
}

18
cpp/src/sdk/src/util/ConvertUtil.h
View File

@ -1,18 +0,0 @@
/*******************************************************************************
* Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved
* Unauthorized copying of this file, via any medium is strictly prohibited.
* Proprietary and confidential.
******************************************************************************/
#pragma once
#include "MilvusApi.h"
namespace milvus {
class ConvertUtil {
public:
static std::string IndexType2Str(IndexType index);
static IndexType Str2IndexType(const std::string& type);
};
}

10
cpp/src/server/DBWrapper.cpp
View File

@ -37,12 +37,16 @@ DBWrapper::DBWrapper() {
opt.meta.archive_conf.SetCriterias(criterial);
//create db root folder
CommonUtil::CreateDirectory(opt.meta.path);
ServerError err = CommonUtil::CreateDirectory(opt.meta.path);
if(err != SERVER_SUCCESS) {
std::cout << "ERROR! Failed to create database root path: " << opt.meta.path << std::endl;
kill(0, SIGUSR1);
}
zilliz::milvus::engine::DB::Open(opt, &db_);
if(db_ == nullptr) {
SERVER_LOG_ERROR << "Failed to open db";
throw ServerException(SERVER_NULL_POINTER, "Failed to open db");
std::cout << "ERROR! Failed to open database" << std::endl;
kill(0, SIGUSR1);
}
}

4
cpp/src/server/MilvusServer.cpp
View File

@ -23,6 +23,7 @@
#include <thrift/concurrency/PosixThreadFactory.h>
#include <thread>
#include <iostream>
namespace zilliz {
namespace milvus {
@ -92,7 +93,8 @@ MilvusServer::StartService() {
return;
}
} catch (apache::thrift::TException& ex) {
//SERVER_LOG_ERROR << "Server encounter exception: " << ex.what();
std::cout << "ERROR! " << ex.what() << std::endl;
kill(0, SIGUSR1);
}
}

1
cpp/src/server/RequestTask.h
View File

@ -7,7 +7,6 @@
#include "RequestScheduler.h"
#include "utils/Error.h"
#include "utils/AttributeSerializer.h"
#include "db/Types.h"
#include "milvus_types.h"

318
cpp/src/storage/s3/S3ClientWrapper.cpp
View File

@ -1,159 +1,159 @@
#include "S3ClientWrapper.h"
#include <aws/s3/model/CreateBucketRequest.h>
#include <aws/s3/model/DeleteBucketRequest.h>
#include <aws/s3/model/PutObjectRequest.h>
#include <aws/s3/model/GetObjectRequest.h>
#include <aws/s3/model/DeleteObjectRequest.h>
#include <iostream>
#include <fstream>
namespace zilliz {
namespace milvus {
namespace engine {
namespace storage {
Status
S3ClientWrapper::Create(const std::string &ip_address,
const std::string &port,
const std::string &access_key,
const std::string &secret_key) {
Aws::InitAPI(options_);
Aws::Client::ClientConfiguration cfg;
// TODO: ip_address need to be validated.
cfg.endpointOverride = ip_address + ":" + port; // S3 server ip address and port
cfg.scheme = Aws::Http::Scheme::HTTP;
cfg.verifySSL =
false; //Aws::Auth::AWSCredentials cred("RPW421T9GSIO4A45Y9ZR", "2owKYy9emSS90Q0pXuyqpX1OxBCyEDYodsiBemcq"); // 认证的Key
client_ =
new S3Client(Aws::Auth::AWSCredentials(access_key, secret_key),
cfg,
Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Always,
false);
if (client_ == nullptr) {
std::string error = "Can't connect server.";
return Status::Error(error);
} else {
return Status::OK();
}
}
Status
S3ClientWrapper::Close() {
if (client_ != nullptr) {
delete client_;
client_ = nullptr;
}
Aws::ShutdownAPI(options_);
return Status::OK();
}
Status
S3ClientWrapper::CreateBucket(std::string& bucket_name) {
Aws::S3::Model::CreateBucketRequest request;
request.SetBucket(bucket_name);
auto outcome = client_->CreateBucket(request);
if (outcome.IsSuccess())
{
return Status::OK();
}
else
{
std::cout << "CreateBucket error: "
<< outcome.GetError().GetExceptionName() << std::endl
<< outcome.GetError().GetMessage() << std::endl;
switch(outcome.GetError().GetErrorType()) {
case Aws::S3::S3Errors::BUCKET_ALREADY_EXISTS:
case Aws::S3::S3Errors::BUCKET_ALREADY_OWNED_BY_YOU:
return Status::AlreadyExist(outcome.GetError().GetMessage());
default:
return Status::Error(outcome.GetError().GetMessage());
}
}
}
Status
S3ClientWrapper::DeleteBucket(std::string& bucket_name) {
Aws::S3::Model::DeleteBucketRequest bucket_request;
bucket_request.SetBucket(bucket_name);
auto outcome = client_->DeleteBucket(bucket_request);
if (outcome.IsSuccess())
{
return Status::OK();
}
else
{
std::cout << "DeleteBucket error: "
<< outcome.GetError().GetExceptionName() << " - "
<< outcome.GetError().GetMessage() << std::endl;
return Status::Error(outcome.GetError().GetMessage());
}
}
Status
S3ClientWrapper::UploadFile(std::string &BucketName, std::string &objectKey, std::string &pathkey) {
PutObjectRequest putObjectRequest;
putObjectRequest.WithBucket(BucketName.c_str()).WithKey(objectKey.c_str());
auto input_data = Aws::MakeShared<Aws::FStream>("PutObjectInputStream",
pathkey.c_str(),
std::ios_base::in | std::ios_base::binary);
putObjectRequest.SetBody(input_data);
auto put_object_result = client_->PutObject(putObjectRequest);
if (put_object_result.IsSuccess()) {
return Status::OK();
} else {
std::cout << "PutObject error: " << put_object_result.GetError().GetExceptionName() << " "
<< put_object_result.GetError().GetMessage() << std::endl;
return Status::Error(put_object_result.GetError().GetMessage());
}
}
Status
S3ClientWrapper::DownloadFile(std::string &BucketName, std::string &objectKey, std::string &pathkey) {
GetObjectRequest object_request;
object_request.WithBucket(BucketName.c_str()).WithKey(objectKey.c_str());
auto get_object_outcome = client_->GetObject(object_request);
if (get_object_outcome.IsSuccess()) {
Aws::OFStream local_file(pathkey.c_str(), std::ios::out | std::ios::binary);
local_file << get_object_outcome.GetResult().GetBody().rdbuf();
return Status::OK();
} else {
std::cout << "GetObject error: " << get_object_outcome.GetError().GetExceptionName() << " "
<< get_object_outcome.GetError().GetMessage() << std::endl;
return Status::Error(get_object_outcome.GetError().GetMessage());
}
}
Status
S3ClientWrapper::DeleteFile(std::string &bucket_name, std::string &object_key) {
Aws::S3::Model::DeleteObjectRequest object_request;
object_request.WithBucket(bucket_name).WithKey(object_key);
auto delete_object_outcome = client_->DeleteObject(object_request);
if (delete_object_outcome.IsSuccess()) {
return Status::OK();
} else {
std::cout << "DeleteObject error: " <<
delete_object_outcome.GetError().GetExceptionName() << " " <<
delete_object_outcome.GetError().GetMessage() << std::endl;
return Status::Error(delete_object_outcome.GetError().GetMessage());
}
}
}
}
}
}
//#include "S3ClientWrapper.h"
//
//#include <aws/s3/model/CreateBucketRequest.h>
//#include <aws/s3/model/DeleteBucketRequest.h>
//#include <aws/s3/model/PutObjectRequest.h>
//#include <aws/s3/model/GetObjectRequest.h>
//#include <aws/s3/model/DeleteObjectRequest.h>
//
//#include <iostream>
//#include <fstream>
//
//
//namespace zilliz {
//namespace milvus {
//namespace engine {
//namespace storage {
//
//Status
//S3ClientWrapper::Create(const std::string &ip_address,
// const std::string &port,
// const std::string &access_key,
// const std::string &secret_key) {
// Aws::InitAPI(options_);
// Aws::Client::ClientConfiguration cfg;
//
// // TODO: ip_address need to be validated.
//
// cfg.endpointOverride = ip_address + ":" + port; // S3 server ip address and port
// cfg.scheme = Aws::Http::Scheme::HTTP;
// cfg.verifySSL =
// false; //Aws::Auth::AWSCredentials cred("RPW421T9GSIO4A45Y9ZR", "2owKYy9emSS90Q0pXuyqpX1OxBCyEDYodsiBemcq"); // 认证的Key
// client_ =
// new S3Client(Aws::Auth::AWSCredentials(access_key, secret_key),
// cfg,
// Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Always,
// false);
// if (client_ == nullptr) {
// std::string error = "Can't connect server.";
// return Status::Error(error);
// } else {
// return Status::OK();
// }
//}
//
//
//Status
//S3ClientWrapper::Close() {
// if (client_ != nullptr) {
// delete client_;
// client_ = nullptr;
// }
// Aws::ShutdownAPI(options_);
// return Status::OK();
//}
//
//Status
//S3ClientWrapper::CreateBucket(std::string& bucket_name) {
// Aws::S3::Model::CreateBucketRequest request;
// request.SetBucket(bucket_name);
//
// auto outcome = client_->CreateBucket(request);
//
// if (outcome.IsSuccess())
// {
// return Status::OK();
// }
// else
// {
// std::cout << "CreateBucket error: "
// << outcome.GetError().GetExceptionName() << std::endl
// << outcome.GetError().GetMessage() << std::endl;
// switch(outcome.GetError().GetErrorType()) {
// case Aws::S3::S3Errors::BUCKET_ALREADY_EXISTS:
// case Aws::S3::S3Errors::BUCKET_ALREADY_OWNED_BY_YOU:
// return Status::AlreadyExist(outcome.GetError().GetMessage());
// default:
// return Status::Error(outcome.GetError().GetMessage());
// }
// }
//}
//
//Status
//S3ClientWrapper::DeleteBucket(std::string& bucket_name) {
// Aws::S3::Model::DeleteBucketRequest bucket_request;
// bucket_request.SetBucket(bucket_name);
//
// auto outcome = client_->DeleteBucket(bucket_request);
//
// if (outcome.IsSuccess())
// {
// return Status::OK();
// }
// else
// {
// std::cout << "DeleteBucket error: "
// << outcome.GetError().GetExceptionName() << " - "
// << outcome.GetError().GetMessage() << std::endl;
// return Status::Error(outcome.GetError().GetMessage());
// }
//}
//
//Status
//S3ClientWrapper::UploadFile(std::string &BucketName, std::string &objectKey, std::string &pathkey) {
//
// PutObjectRequest putObjectRequest;
// putObjectRequest.WithBucket(BucketName.c_str()).WithKey(objectKey.c_str());
//
// auto input_data = Aws::MakeShared<Aws::FStream>("PutObjectInputStream",
// pathkey.c_str(),
// std::ios_base::in | std::ios_base::binary);
// putObjectRequest.SetBody(input_data);
// auto put_object_result = client_->PutObject(putObjectRequest);
// if (put_object_result.IsSuccess()) {
// return Status::OK();
// } else {
// std::cout << "PutObject error: " << put_object_result.GetError().GetExceptionName() << " "
// << put_object_result.GetError().GetMessage() << std::endl;
// return Status::Error(put_object_result.GetError().GetMessage());
// }
//}
//
//Status
//S3ClientWrapper::DownloadFile(std::string &BucketName, std::string &objectKey, std::string &pathkey) {
// GetObjectRequest object_request;
// object_request.WithBucket(BucketName.c_str()).WithKey(objectKey.c_str());
// auto get_object_outcome = client_->GetObject(object_request);
// if (get_object_outcome.IsSuccess()) {
// Aws::OFStream local_file(pathkey.c_str(), std::ios::out | std::ios::binary);
// local_file << get_object_outcome.GetResult().GetBody().rdbuf();
// return Status::OK();
// } else {
// std::cout << "GetObject error: " << get_object_outcome.GetError().GetExceptionName() << " "
// << get_object_outcome.GetError().GetMessage() << std::endl;
// return Status::Error(get_object_outcome.GetError().GetMessage());
// }
//}
//
//Status
//S3ClientWrapper::DeleteFile(std::string &bucket_name, std::string &object_key) {
// Aws::S3::Model::DeleteObjectRequest object_request;
// object_request.WithBucket(bucket_name).WithKey(object_key);
//
// auto delete_object_outcome = client_->DeleteObject(object_request);
//
// if (delete_object_outcome.IsSuccess()) {
// return Status::OK();
// } else {
// std::cout << "DeleteObject error: " <<
// delete_object_outcome.GetError().GetExceptionName() << " " <<
// delete_object_outcome.GetError().GetMessage() << std::endl;
//
// return Status::Error(delete_object_outcome.GetError().GetMessage());
// }
//}
//
//}
//}
//}
//}

90
cpp/src/storage/s3/S3ClientWrapper.h
View File

@ -1,45 +1,45 @@
#pragma once
#include "storage/IStorage.h"
#include <aws/s3/S3Client.h>
#include <aws/core/Aws.h>
#include <aws/core/auth/AWSCredentialsProvider.h>
using namespace Aws::S3;
using namespace Aws::S3::Model;
namespace zilliz {
namespace milvus {
namespace engine {
namespace storage {
class S3ClientWrapper : public IStorage {
public:
S3ClientWrapper() = default;
~S3ClientWrapper() = default;
Status Create(const std::string &ip_address,
const std::string &port,
const std::string &access_key,
const std::string &secret_key) override;
Status Close() override;
Status CreateBucket(std::string& bucket_name) override;
Status DeleteBucket(std::string& bucket_name) override;
Status UploadFile(std::string &BucketName, std::string &objectKey, std::string &pathkey) override;
Status DownloadFile(std::string &BucketName, std::string &objectKey, std::string &pathkey) override;
Status DeleteFile(std::string &bucket_name, std::string &object_key) override;
private:
S3Client *client_ = nullptr;
Aws::SDKOptions options_;
};
}
}
}
}
//#pragma once
//
//#include "storage/IStorage.h"
//
//
//#include <aws/s3/S3Client.h>
//#include <aws/core/Aws.h>
//#include <aws/core/auth/AWSCredentialsProvider.h>
//
//
//using namespace Aws::S3;
//using namespace Aws::S3::Model;
//
//namespace zilliz {
//namespace milvus {
//namespace engine {
//namespace storage {
//
//class S3ClientWrapper : public IStorage {
// public:
//
// S3ClientWrapper() = default;
// ~S3ClientWrapper() = default;
//
// Status Create(const std::string &ip_address,
// const std::string &port,
// const std::string &access_key,
// const std::string &secret_key) override;
// Status Close() override;
//
// Status CreateBucket(std::string& bucket_name) override;
// Status DeleteBucket(std::string& bucket_name) override;
// Status UploadFile(std::string &BucketName, std::string &objectKey, std::string &pathkey) override;
// Status DownloadFile(std::string &BucketName, std::string &objectKey, std::string &pathkey) override;
// Status DeleteFile(std::string &bucket_name, std::string &object_key) override;
//
// private:
// S3Client *client_ = nullptr;
// Aws::SDKOptions options_;
//};
//
//}
//}
//}
//}

50
cpp/src/utils/AttributeSerializer.cpp
View File

@ -1,50 +0,0 @@
/*******************************************************************************
* Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved
* Unauthorized copying of this file, via any medium is strictly prohibited.
* Proprietary and confidential.
******************************************************************************/
#include "AttributeSerializer.h"
#include "StringHelpFunctions.h"
namespace zilliz {
namespace milvus {
namespace server {
ServerError AttributeSerializer::Encode(const AttribMap& attrib_map, std::string& attrib_str) {
attrib_str = "";
for(auto iter : attrib_map) {
attrib_str += iter.first;
attrib_str += ":\"";
attrib_str += iter.second;
attrib_str += "\";";
}
return SERVER_SUCCESS;
}
ServerError AttributeSerializer::Decode(const std::string& attrib_str, AttribMap& attrib_map) {
attrib_map.clear();
std::vector<std::string> kv_pairs;
StringHelpFunctions::SplitStringByQuote(attrib_str, ";", "\"", kv_pairs);
for(std::string& str : kv_pairs) {
std::string key, val;
size_t index = str.find_first_of(":", 0);
if (index != std::string::npos) {
key = str.substr(0, index);
val = str.substr(index + 1);
} else {
key = str;
}
attrib_map.insert(std::make_pair(key, val));
}
return SERVER_SUCCESS;
}
}
}
}

27
cpp/src/utils/AttributeSerializer.h
View File

@ -1,27 +0,0 @@
/*******************************************************************************
* Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved
* Unauthorized copying of this file, via any medium is strictly prohibited.
* Proprietary and confidential.
******************************************************************************/
#pragma once
#include <map>
#include "Error.h"
namespace zilliz {
namespace milvus {
namespace server {
using AttribMap = std::map<std::string, std::string>;
class AttributeSerializer {
public:
static ServerError Encode(const AttribMap& attrib_map, std::string& attrib_str);
static ServerError Decode(const std::string& attrib_str, AttribMap& attrib_map);
};
}
}
}

7
cpp/src/utils/StringHelpFunctions.cpp
View File

@ -9,13 +9,6 @@ namespace zilliz {
namespace milvus {
namespace server {
void StringHelpFunctions::TrimStringLineBreak(std::string &string) {
if (!string.empty()) {
static std::string s_format("\n\r");
string.erase(string.find_last_not_of(s_format) + 1);
}
}
void StringHelpFunctions::TrimStringBlank(std::string &string) {
if (!string.empty()) {
static std::string s_format(" \n\r\t");

2
cpp/src/utils/StringHelpFunctions.h
View File

@ -18,8 +18,6 @@ private:
StringHelpFunctions() = default;
public:
static void TrimStringLineBreak(std::string &string);
static void TrimStringBlank(std::string &string);
static void TrimStringQuote(std::string &string, const std::string &qoute);

7
cpp/unittest/server/CMakeLists.txt
View File

@ -14,9 +14,10 @@ aux_source_directory(../../src/cache cache_srcs)
aux_source_directory(../../src/wrapper wrapper_src)
aux_source_directory(./ test_srcs)
set(server_src_files
set(utils_srcs
${MILVUS_ENGINE_SRC}/utils/StringHelpFunctions.cpp
${MILVUS_ENGINE_SRC}/utils/AttributeSerializer.cpp
${MILVUS_ENGINE_SRC}/utils/TimeRecorder.cpp
${MILVUS_ENGINE_SRC}/utils/CommonUtil.cpp
)
cuda_add_executable(server_test
@ -25,7 +26,7 @@ cuda_add_executable(server_test
${cache_srcs}
${wrapper_src}
${test_srcs}
${server_src_files}
${utils_srcs}
${require_files}
)

37
cpp/unittest/server/cache_test.cpp
View File

@ -11,18 +11,18 @@
using namespace zilliz::milvus;
TEST(CacheTest, CACHE_TEST) {
cache::CacheMgr* cpu_mgr = cache::CpuCacheMgr::GetInstance();
TEST(CacheTest, CPU_CACHE_TEST) {
cache::CacheMgr *cpu_mgr = cache::CpuCacheMgr::GetInstance();
const int64_t gbyte = 1 << 30;
int64_t g_num = 16;
int64_t cap = g_num*gbyte;
int64_t cap = g_num * gbyte;
cpu_mgr->SetCapacity(cap);
ASSERT_EQ(cpu_mgr->CacheCapacity(), cap);
const int dim = 256;
for(int i = 0; i < 20; i++) {
for (int i = 0; i < 20; i++) {
std::shared_ptr<faiss::Index> raw_index(faiss::index_factory(dim, "IDMap,Flat"));
engine::Index_ptr index = std::make_shared<engine::Index>(raw_index);
index->ntotal = 1000000;//less 1G per index
@ -31,6 +31,12 @@ TEST(CacheTest, CACHE_TEST) {
}
ASSERT_LT(cpu_mgr->ItemCount(), g_num);
auto obj = cpu_mgr->GetIndex("index_0");
ASSERT_TRUE(obj == nullptr);
obj = cpu_mgr->GetIndex("index_19");
ASSERT_TRUE(obj != nullptr);
{
std::string item = "index_15";
ASSERT_TRUE(cpu_mgr->ItemExists(item));
@ -49,4 +55,27 @@ TEST(CacheTest, CACHE_TEST) {
cpu_mgr->InsertItem("index_6g", index);
ASSERT_EQ(cpu_mgr->ItemCount(), 0);//data greater than capacity can not be inserted sucessfully
}
cpu_mgr->PrintInfo();
}
TEST(CacheTest, GPU_CACHE_TEST) {
cache::CacheMgr* gpu_mgr = cache::GpuCacheMgr::GetInstance();
const int dim = 256;
for(int i = 0; i < 20; i++) {
std::shared_ptr<faiss::Index> raw_index(faiss::index_factory(dim, "IDMap,Flat"));
engine::Index_ptr index = std::make_shared<engine::Index>(raw_index);
index->ntotal = 1000;
cache::DataObjPtr obj = std::make_shared<cache::DataObj>(index);
gpu_mgr->InsertItem("index_" + std::to_string(i), obj);
}
auto obj = gpu_mgr->GetItem("index_0");
gpu_mgr->ClearCache();
ASSERT_EQ(gpu_mgr->ItemCount(), 0);
}

26
cpp/unittest/server/common_test.cpp
View File

@ -1,26 +0,0 @@
////////////////////////////////////////////////////////////////////////////////
// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved
// Unauthorized copying of this file, via any medium is strictly prohibited.
// Proprietary and confidential.
////////////////////////////////////////////////////////////////////////////////
#include <gtest/gtest.h>
#include "utils/CommonUtil.h"
#include "utils/Error.h"
using namespace zilliz::milvus;
TEST(CommonTest, COMMON_TEST) {
std::string path1 = "/tmp/milvus_test/";
std::string path2 = path1 + "common_test_12345/";
std::string path3 = path2 + "abcdef";
server::ServerError err = server::CommonUtil::CreateDirectory(path3);
ASSERT_EQ(err, server::SERVER_SUCCESS);
ASSERT_TRUE(server::CommonUtil::IsDirectoryExit(path3));
err = server::CommonUtil::DeleteDirectory(path1);
ASSERT_EQ(err, server::SERVER_SUCCESS);
ASSERT_FALSE(server::CommonUtil::IsDirectoryExit(path1));
}

77
cpp/unittest/server/config_test.cpp Normal file
View File

@ -0,0 +1,77 @@
////////////////////////////////////////////////////////////////////////////////
// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved
// Unauthorized copying of this file, via any medium is strictly prohibited.
// Proprietary and confidential.
////////////////////////////////////////////////////////////////////////////////
#include <gtest/gtest.h>
#include "config/IConfigMgr.h"
#include "server/ServerConfig.h"
using namespace zilliz::milvus;
static const std::string CONFIG_FILE_PATH = "../../../conf/server_config.yaml";
TEST(ConfigTest, CONFIG_TEST) {
server::IConfigMgr* config_mgr = server::IConfigMgr::GetInstance();
server::ServerError err = config_mgr->LoadConfigFile(CONFIG_FILE_PATH);
ASSERT_EQ(err, server::SERVER_SUCCESS);
server::ConfigNode& root_config = config_mgr->GetRootNode();
server::ConfigNode& server_config = root_config.GetChild("server_config");
server::ConfigNode& db_config = root_config.GetChild("db_config");
server::ConfigNode& metric_config = root_config.GetChild("metric_config");
server::ConfigNode& cache_config = root_config.GetChild("cache_config");
std::string address = server_config.GetValue("address");
ASSERT_TRUE(!address.empty());
int64_t port = server_config.GetInt64Value("port");
ASSERT_TRUE(port != 0);
server_config.SetValue("test", "2.5");
double test = server_config.GetDoubleValue("test");
ASSERT_EQ(test, 2.5);
server::ConfigNode fake;
server_config.AddChild("fake", fake);
fake = server_config.GetChild("fake");
server::ConfigNodeArr arr;
server_config.GetChildren(arr);
ASSERT_EQ(arr.size(), 1UL);
server_config.ClearChildren();
auto children = server_config.GetChildren();
ASSERT_TRUE(children.empty());
root_config.PrintAll();
std::string all = root_config.DumpString();
ASSERT_TRUE(!all.empty());
server_config.ClearConfig();
auto configs = server_config.GetConfig();
ASSERT_TRUE(configs.empty());
server_config.AddSequenceItem("seq", "aaa");
server_config.AddSequenceItem("seq", "bbb");
auto seq = server_config.GetSequence("seq");
ASSERT_EQ(seq.size(), 2UL);
server_config.ClearSequences();
auto seqs = server_config.GetSequences();
ASSERT_TRUE(seqs.empty());
}
TEST(ConfigTest, SERVER_CONFIG_TEST) {
server::ServerConfig& config = server::ServerConfig::GetInstance();
server::ServerError err = config.LoadConfigFile(CONFIG_FILE_PATH);
ASSERT_EQ(err, server::SERVER_SUCCESS);
server::ConfigNode node1 = config.GetConfig("server_config");
server::ConfigNode& node2 = config.GetConfig("cache_config");
node1.Combine(node2);
int32_t cap = node1.GetInt32Value("cpu_cache_capacity");
ASSERT_GT(cap, 0);
config.PrintAll();
}

106
cpp/unittest/server/util_test.cpp
View File

@ -4,31 +4,99 @@
// Proprietary and confidential.
////////////////////////////////////////////////////////////////////////////////
#include <gtest/gtest.h>
#include <thread>
#include "utils/AttributeSerializer.h"
#include "utils/CommonUtil.h"
#include "utils/Error.h"
#include "utils/StringHelpFunctions.h"
#include "utils/TimeRecorder.h"
#include "utils/BlockingQueue.h"
using namespace zilliz::milvus;
TEST(AttribSerializeTest, ATTRIBSERIAL_TEST) {
std::map<std::string, std::string> attrib;
attrib["uid"] = "ABCDEF";
attrib["color"] = "red";
attrib["number"] = "9900";
attrib["comment"] = "please note: it is a car, not a ship";
attrib["address"] = " china;shanghai ";
namespace {
std::string attri_str;
server::AttributeSerializer::Encode(attrib, attri_str);
std::map<std::string, std::string> attrib_out;
server::ServerError err = server::AttributeSerializer::Decode(attri_str, attrib_out);
ASSERT_EQ(err, server::SERVER_SUCCESS);
ASSERT_EQ(attrib_out.size(), attrib.size());
for(auto iter : attrib) {
ASSERT_EQ(attrib_out[iter.first], attrib_out[iter.first]);
}
using TimeUnit = server::TimeRecorder::TimeDisplayUnit;
double TestTimeRecorder(TimeUnit unit, int64_t log_level) {
server::TimeRecorder rc("test rc", unit, log_level);
rc.Record("begin");
std::this_thread::sleep_for(std::chrono::microseconds(10));
rc.Elapse("end");
return rc.Span();
}
}
TEST(CommonTest, COMMON_TEST) {
std::string path1 = "/tmp/milvus_test/";
std::string path2 = path1 + "common_test_12345/";
std::string path3 = path2 + "abcdef";
server::ServerError err = server::CommonUtil::CreateDirectory(path3);
ASSERT_EQ(err, server::SERVER_SUCCESS);
ASSERT_TRUE(server::CommonUtil::IsDirectoryExit(path3));
err = server::CommonUtil::DeleteDirectory(path1);
ASSERT_EQ(err, server::SERVER_SUCCESS);
ASSERT_FALSE(server::CommonUtil::IsDirectoryExit(path1));
}
TEST(UtilTest, STRINGFUNCTIONS_TEST) {
std::string str = " test zilliz";
server::StringHelpFunctions::TrimStringBlank(str);
ASSERT_EQ(str, "test zilliz");
str = "a,b,c";
std::vector<std::string> result;
server::StringHelpFunctions::SplitStringByDelimeter(str , ",", result);
ASSERT_EQ(result.size(), 3UL);
str = "55,\"aa,gg,yy\",b";
result.clear();
server::StringHelpFunctions::SplitStringByQuote(str , ",", "\"", result);
ASSERT_EQ(result.size(), 3UL);
}
TEST(UtilTest, TIMERECORDER_TEST) {
double span = TestTimeRecorder(TimeUnit::eTimeAutoUnit, 0);
ASSERT_GT(span, 0.0);
span = TestTimeRecorder(TimeUnit::eTimeHourUnit, 1);
ASSERT_GT(span, 0.0);
span = TestTimeRecorder(TimeUnit::eTimeMinuteUnit, 2);
ASSERT_GT(span, 0.0);
span = TestTimeRecorder(TimeUnit::eTimeSecondUnit, 3);
ASSERT_GT(span, 0.0);
span = TestTimeRecorder(TimeUnit::eTimeMilliSecUnit, 4);
ASSERT_GT(span, 0.0);
span = TestTimeRecorder(TimeUnit::eTimeMicroSecUnit, -1);
ASSERT_GT(span, 0.0);
}
TEST(UtilTest, BLOCKINGQUEUE_TEST) {
server::BlockingQueue<std::string> bq;
static const size_t count = 10;
bq.SetCapacity(count);
for(size_t i = 1; i <= count; i++) {
std::string id = "No." + std::to_string(i);
bq.Put(id);
}
ASSERT_EQ(bq.Size(), count);
ASSERT_FALSE(bq.Empty());
std::string str = bq.Front();
ASSERT_EQ(str, "No.1");
str = bq.Back();
ASSERT_EQ(str, "No." + std::to_string(count));
for(size_t i = 1; i <= count; i++) {
std::string id = "No." + std::to_string(i);
str = bq.Take();
ASSERT_EQ(id, str);
}
}

132
cpp/unittest/storage/S3ClientTest.cpp
View File

@ -1,66 +1,66 @@
////////////////////////////////////////////////////////////////////////////////
// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved
// Unauthorized copying of this file, via any medium is strictly prohibited.
// Proprietary and confidential.
////////////////////////////////////////////////////////////////////////////////
#include "storage/IStorage.h"
#include "storage/s3/S3ClientWrapper.h"
#include <gtest/gtest.h>
#include <memory.h>
#include <fstream>
using namespace zilliz::milvus::engine;
TEST(s3_client_wrapper, CLIENT_WRAPPER_TEST) {
std::shared_ptr<storage::IStorage> storage_ptr = std::make_shared<storage::S3ClientWrapper>();
std::string ip_address = "127.0.0.1";
std::string port = "9000";
std::string access_key = "AKIAIOSFODNN7EXAMPLE";
std::string secret_key = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY";
Status status = storage_ptr->Create(ip_address, port, access_key, secret_key);
ASSERT_TRUE(status.ok());
std::string filename = "/tmp/s3_test_file";
std::string bucket_name = "bucktname";
std::string object_name = "test_file";
status = storage_ptr->CreateBucket(bucket_name);
std::cout << status.IsAlreadyExist() << std::endl;
if (status.IsAlreadyExist()) {
status = storage_ptr->DeleteBucket(bucket_name);
status = storage_ptr->CreateBucket(bucket_name);
}
ASSERT_TRUE(status.ok());
std::ofstream ofile(filename);
std::stringstream ss;
for (int i = 0; i < 1024; ++i) {
ss << i;
}
ofile << ss.str() << std::endl;
ofile.close();
status = storage_ptr->UploadFile(bucket_name, object_name, filename);
ASSERT_TRUE(status.ok());
status = storage_ptr->DownloadFile(bucket_name, object_name, filename);
std::ifstream infile(filename);
std::string in_buffer;
infile >> in_buffer;
ASSERT_STREQ(in_buffer.c_str(), ss.str().c_str());
status = storage_ptr->DeleteFile(bucket_name, object_name);
ASSERT_TRUE(status.ok());
status = storage_ptr->DeleteBucket(bucket_name);
ASSERT_TRUE(status.ok());
status = storage_ptr->Close();
ASSERT_TRUE(status.ok());
}
//////////////////////////////////////////////////////////////////////////////////
//// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved
//// Unauthorized copying of this file, via any medium is strictly prohibited.
//// Proprietary and confidential.
//////////////////////////////////////////////////////////////////////////////////
//
//#include "storage/IStorage.h"
//#include "storage/s3/S3ClientWrapper.h"
//#include <gtest/gtest.h>
//#include <memory.h>
//#include <fstream>
//
//
//using namespace zilliz::milvus::engine;
//
//TEST(s3_client_wrapper, CLIENT_WRAPPER_TEST) {
//
// std::shared_ptr<storage::IStorage> storage_ptr = std::make_shared<storage::S3ClientWrapper>();
//
// std::string ip_address = "127.0.0.1";
// std::string port = "9000";
// std::string access_key = "AKIAIOSFODNN7EXAMPLE";
// std::string secret_key = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY";
//
// Status status = storage_ptr->Create(ip_address, port, access_key, secret_key);
// ASSERT_TRUE(status.ok());
//
// std::string filename = "/tmp/s3_test_file";
// std::string bucket_name = "bucktname";
// std::string object_name = "test_file";
//
// status = storage_ptr->CreateBucket(bucket_name);
// std::cout << status.IsAlreadyExist() << std::endl;
// if (status.IsAlreadyExist()) {
// status = storage_ptr->DeleteBucket(bucket_name);
// status = storage_ptr->CreateBucket(bucket_name);
// }
//
// ASSERT_TRUE(status.ok());
//
// std::ofstream ofile(filename);
// std::stringstream ss;
// for (int i = 0; i < 1024; ++i) {
// ss << i;
// }
// ofile << ss.str() << std::endl;
// ofile.close();
// status = storage_ptr->UploadFile(bucket_name, object_name, filename);
// ASSERT_TRUE(status.ok());
//
// status = storage_ptr->DownloadFile(bucket_name, object_name, filename);
// std::ifstream infile(filename);
// std::string in_buffer;
// infile >> in_buffer;
// ASSERT_STREQ(in_buffer.c_str(), ss.str().c_str());
//
// status = storage_ptr->DeleteFile(bucket_name, object_name);
// ASSERT_TRUE(status.ok());
//
// status = storage_ptr->DeleteBucket(bucket_name);
// ASSERT_TRUE(status.ok());
//
// status = storage_ptr->Close();
// ASSERT_TRUE(status.ok());
//}
//