admin/milvus
1
0
Fork 0
mirror of https://gitee.com/milvus-io/milvus.git synced 2026-01-04 18:02:08 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'yyw' into jinhai

Browse Source 
Former-commit-id: ee701128eda3c82e754436c2c13d148e43386b7d
This commit is contained in:
xj.lin 2019-05-15 15:14:28 +08:00
commit 4c16b19bb5
18 changed files with 1440 additions and 53 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
cpp/CMakeLists.txt
View File

@ -45,6 +45,10 @@ endif ()
if(CMAKE_BUILD_TYPE STREQUAL "Release")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O3 -fPIC -DELPP_THREAD_SAFE")
if (GPU_VERSION STREQUAL "ON")
set(ENABLE_LICENSE "ON")
add_definitions("-DENABLE_LICENSE")
endif ()
else()
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O0 -g -fPIC -DELPP_THREAD_SAFE")
endif()
@ -69,7 +73,7 @@ link_directories(${VECWISE_THIRD_PARTY_BUILD}/lib64)
# WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/third_party)
add_subdirectory(src)
add_subdirectory(test_client)
#add_subdirectory(test_client)
if (BUILD_UNIT_TEST)
add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/unittest)

3
cpp/conf/server_config.yaml
View File

@ -10,6 +10,9 @@ db_config:
db_flush_interval: 5 #unit: second
idmapper_max_open_file: 128
license_config:
license_path: "/tmp/megasearch/abc.license"
log_config:
global:
format: "%datetime | %level | %logger | %msg"

53
cpp/src/CMakeLists.txt
View File

@ -12,7 +12,13 @@ aux_source_directory(db db_files)
aux_source_directory(wrapper wrapper_files)
set(license_check_files
${CMAKE_CURRENT_SOURCE_DIR}/license/License.cpp
license/LicenseLibrary.cpp
license/LicenseCheck.cpp
)
set(license_generator_src
license/LicenseGenerator.cpp
license/LicenseLibrary.cpp
)
set(service_files
@ -28,6 +34,9 @@ set(vecwise_engine_src
${wrapper_files}
)
set(get_sys_info_src
license/GetSysInfo.cpp)
include_directories(/usr/include)
include_directories(/usr/local/cuda/include)
@ -53,19 +62,37 @@ else()
libgfortran.a
libquadmath.a
libsqlite3.a
)
)
endif ()
cuda_add_library(vecwise_engine STATIC ${vecwise_engine_src})
if (ENABLE_LICENSE STREQUAL "ON")
link_directories(/usr/local/cuda/lib64/stubs)
link_directories(/usr/local/cuda/lib64)
set(license_libs
nvidia-ml
libboost_system.a
libboost_filesystem.a
libboost_serialization.a
crypto
cudart
cublas
)
endif ()
cuda_add_library(vecwise_engine STATIC ${vecwise_engine_src})
target_link_libraries(vecwise_engine ${engine_libs})
if (ENABLE_LICENSE STREQUAL "ON")
add_library(vecwise_license STATIC ${license_check_files})
target_link_libraries(vecwise_license ${license_libs})
endif ()
add_executable(vecwise_server
${config_files}
${server_files}
${utils_files}
${service_files}
${license_check_files}
${VECWISE_THIRD_PARTY_BUILD}/include/easylogging++.cc
)
@ -85,11 +112,17 @@ set(server_libs
dl
)
target_link_libraries(vecwise_server ${server_libs})
if (ENABLE_LICENSE STREQUAL "ON")
target_link_libraries(vecwise_server ${server_libs} vecwise_license)
else ()
target_link_libraries(vecwise_server ${server_libs})
endif()
set(license_generator_src
${CMAKE_CURRENT_SOURCE_DIR}/license/LicenseGenerator.cpp)
if (ENABLE_LICENSE STREQUAL "ON")
add_executable(license_generator ${license_generator_src})
add_executable(get_sys_info ${get_sys_info_src})
target_link_libraries(get_sys_info ${license_libs} vecwise_license)
target_link_libraries(license_generator ${license_libs})
endif ()
add_executable(license_generator ${license_generator_src})
install(TARGETS vecwise_server DESTINATION bin)
install(TARGETS vecwise_server DESTINATION bin)

69
cpp/src/license/GPUInfoFile.h Normal file
View File

@ -0,0 +1,69 @@
/*******************************************************************************
* Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved
* Unauthorized copying of this file, via any medium is strictly prohibited.
* Proprietary and confidential.
******************************************************************************/
#pragma once
#include <boost/serialization/access.hpp>
#include <string>
#include <map>
class GPUInfoFile {
public:
GPUInfoFile() = default;
GPUInfoFile(const int &device_count, const std::map<int, std::string> &uuid_encryption_map)
: device_count_(device_count), uuid_encryption_map_(uuid_encryption_map) {}
int get_device_count() {
return device_count_;
}
std::map<int, std::string> &get_uuid_encryption_map() {
return uuid_encryption_map_;
}
public:
friend class boost::serialization::access;
template<typename Archive>
void serialize(Archive &ar, const unsigned int version) {
ar & device_count_;
ar & uuid_encryption_map_;
}
public:
int device_count_ = 0;
std::map<int, std::string> uuid_encryption_map_;
};
class SerializedGPUInfoFile {
public:
~SerializedGPUInfoFile() {
if (gpu_info_file_ != nullptr) {
delete (gpu_info_file_);
gpu_info_file_ = nullptr;
}
}
void
set_gpu_info_file(GPUInfoFile *gpu_info_file) {
gpu_info_file_ = gpu_info_file;
}
GPUInfoFile *get_gpu_info_file() {
return gpu_info_file_;
}
private:
friend class boost::serialization::access;
template<typename Archive>
void serialize(Archive &ar, const unsigned int version) {
ar & gpu_info_file_;
}
private:
GPUInfoFile *gpu_info_file_ = nullptr;
};

83
cpp/src/license/GetSysInfo.cpp Normal file
View File

@ -0,0 +1,83 @@
#include "utils/Log.h"
#include "LicenseLibrary.h"
#include "utils/Error.h"
#include <iostream>
#include <getopt.h>
#include <memory.h>
// Not provide path: current work path will be used and system.info.
using namespace zilliz::vecwise;
void
print_usage(const std::string &app_name) {
printf("\n Usage: %s [OPTIONS]\n\n", app_name.c_str());
printf(" Options:\n");
printf(" -h --help Print this help\n");
printf(" -s --sysinfo filename Generate system info file as given name\n");
printf("\n");
}
int main(int argc, char *argv[]) {
std::string app_name = argv[0];
if (argc != 1 && argc != 3) {
print_usage(app_name);
return EXIT_FAILURE;
}
static struct option long_options[] = {{"system_info", required_argument, 0, 's'},
{"help", no_argument, 0, 'h'},
{NULL, 0, 0, 0}};
int value = 0;
int option_index = 0;
std::string system_info_filename = "./system.info";
while ((value = getopt_long(argc, argv, "s:h", long_options, &option_index)) != -1) {
switch (value) {
case 's': {
char *system_info_filename_ptr = strdup(optarg);
system_info_filename = system_info_filename_ptr;
free(system_info_filename_ptr);
// printf("Generate system info file: %s\n", system_info_filename.c_str());
break;
}
case 'h':print_usage(app_name);
return EXIT_SUCCESS;
case '?':print_usage(app_name);
return EXIT_FAILURE;
default:print_usage(app_name);
break;
}
}
int device_count = 0;
server::ServerError err = server::LicenseLibrary::GetDeviceCount(device_count);
if (err != server::SERVER_SUCCESS) return -1;
// 1. Get All GPU UUID
std::vector<std::string> uuid_array;
err = server::LicenseLibrary::GetUUID(device_count, uuid_array);
if (err != server::SERVER_SUCCESS) return -1;
// 2. Get UUID SHA256
std::vector<std::string> uuid_sha256_array;
err = server::LicenseLibrary::GetUUIDSHA256(device_count, uuid_array, uuid_sha256_array);
if (err != server::SERVER_SUCCESS) return -1;
// 3. Generate GPU ID map with GPU UUID
std::map<int, std::string> uuid_encrption_map;
for (int i = 0; i < device_count; ++i) {
uuid_encrption_map[i] = uuid_sha256_array[i];
}
// 4. Generate GPU_info File
err = server::LicenseLibrary::GPUinfoFileSerialization(system_info_filename,
device_count,
uuid_encrption_map);
if (err != server::SERVER_SUCCESS) return -1;
printf("Generate GPU_info File Success\n");
return 0;
}

97
cpp/src/license/LicenseCheck.cpp Normal file
View File

@ -0,0 +1,97 @@
#include "LicenseCheck.h"
#include <iostream>
#include <boost/archive/binary_oarchive.hpp>
#include <boost/archive/binary_iarchive.hpp>
//#include <boost/foreach.hpp>
//#include <boost/serialization/vector.hpp>
#include <boost/filesystem/path.hpp>
#include <boost/serialization/map.hpp>
#include <boost/filesystem/operations.hpp>
namespace zilliz {
namespace vecwise {
namespace server {
// Part 1: Legality check
ServerError
LicenseCheck::LegalityCheck(const std::string &license_file_path) {
int device_count;
LicenseLibrary::GetDeviceCount(device_count);
std::vector<std::string> uuid_array;
LicenseLibrary::GetUUID(device_count, uuid_array);
std::vector<std::string> sha_array;
LicenseLibrary::GetUUIDSHA256(device_count, uuid_array, sha_array);
int output_device_count;
std::map<int, std::string> uuid_encryption_map;
time_t starting_time;
time_t end_time;
ServerError err = LicenseLibrary::LicenseFileDeserialization(license_file_path,
output_device_count,
uuid_encryption_map,
starting_time,
end_time);
if(err !=SERVER_SUCCESS)
{
printf("License check error: 01\n");
return SERVER_UNEXPECTED_ERROR;
}
time_t system_time;
LicenseLibrary::GetSystemTime(system_time);
if (device_count != output_device_count) {
printf("License check error: 02\n");
return SERVER_UNEXPECTED_ERROR;
}
for (int i = 0; i < device_count; ++i) {
if (sha_array[i] != uuid_encryption_map[i]) {
printf("License check error: 03\n");
return SERVER_UNEXPECTED_ERROR;
}
}
if (system_time < starting_time || system_time > end_time) {
printf("License check error: 04\n");
return SERVER_UNEXPECTED_ERROR;
}
printf("Legality Check Success\n");
return SERVER_SUCCESS;
}
// Part 2: Timing check license
ServerError
LicenseCheck::AlterFile(const std::string &license_file_path,
const boost::system::error_code &ec,
boost::asio::deadline_timer *pt) {
ServerError err = LegalityCheck(license_file_path);
if(err!=SERVER_SUCCESS)
{
exit(1);
}
printf("---runing---\n");
pt->expires_at(pt->expires_at() + boost::posix_time::hours(1));
pt->async_wait(boost::bind(AlterFile, license_file_path, boost::asio::placeholders::error, pt));
return SERVER_SUCCESS;
}
ServerError
LicenseCheck::StartCountingDown(const std::string &license_file_path) {
if (!LicenseLibrary::IsFileExistent(license_file_path)) return SERVER_LICENSE_FILE_NOT_EXIST;
boost::asio::io_service io;
boost::asio::deadline_timer t(io, boost::posix_time::hours(1));
t.async_wait(boost::bind(AlterFile, license_file_path, boost::asio::placeholders::error, &t));
io.run();
return SERVER_SUCCESS;
}
}
}
}

48
cpp/src/license/LicenseCheck.h Normal file
View File

@ -0,0 +1,48 @@
#pragma once
#include "utils/Error.h"
#include "LicenseLibrary.h"
#include <boost/asio.hpp>
#include <boost/thread.hpp>
#include <boost/date_time/posix_time/posix_time.hpp>
namespace zilliz {
namespace vecwise {
namespace server {
class LicenseCheck {
public:
static LicenseCheck &
GetInstance() {
static LicenseCheck instance;
return instance;
};
// Part 1: Legality check
static ServerError
LegalityCheck(const std::string &license_file_path);
// Part 2: Timing check license
static ServerError
AlterFile(const std::string &license_file_path,
const boost::system::error_code &ec,
boost::asio::deadline_timer *pt);
static ServerError
StartCountingDown(const std::string &license_file_path);
private:
};
}
}
}

86
cpp/src/license/LicenseFile.h Normal file
View File

@ -0,0 +1,86 @@
/*******************************************************************************
* Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved
* Unauthorized copying of this file, via any medium is strictly prohibited.
* Proprietary and confidential.
******************************************************************************/
#pragma once
#include <boost/serialization/access.hpp>
#include <string>
#include <map>
class LicenseFile {
public:
LicenseFile() = default;
LicenseFile(const int &device_count,
const std::map<int, std::string> &uuid_encryption_map,
const time_t &starting_time,
const time_t &end_time)
: device_count_(device_count),
uuid_encryption_map_(uuid_encryption_map),
starting_time_(starting_time),
end_time_(end_time) {}
int get_device_count() {
return device_count_;
}
std::map<int, std::string> &get_uuid_encryption_map() {
return uuid_encryption_map_;
}
time_t get_starting_time() {
return starting_time_;
}
time_t get_end_time() {
return end_time_;
}
public:
friend class boost::serialization::access;
template<typename Archive>
void serialize(Archive &ar, const unsigned int version) {
ar & device_count_;
ar & uuid_encryption_map_;
ar & starting_time_;
ar & end_time_;
}
public:
int device_count_ = 0;
std::map<int, std::string> uuid_encryption_map_;
time_t starting_time_ = 0;
time_t end_time_ = 0;
};
class SerializedLicenseFile {
public:
~SerializedLicenseFile() {
if (license_file_ != nullptr) {
delete (license_file_);
license_file_ = nullptr;
}
}
void
set_license_file(LicenseFile *license_file) {
license_file_ = license_file;
}
LicenseFile *get_license_file() {
return license_file_;
}
private:
friend class boost::serialization::access;
template<typename Archive>
void serialize(Archive &ar, const unsigned int version) {
ar & license_file_;
}
private:
LicenseFile *license_file_ = nullptr;
};

123
cpp/src/license/LicenseGenerator.cpp
View File

@ -1,6 +1,121 @@
#include <iostream>
int main() {
std::cout << "This is license generator" << std::endl;
#include <iostream>
#include <getopt.h>
#include <memory.h>
#include "utils/Log.h"
#include "license/LicenseLibrary.h"
#include "utils/Error.h"
using namespace zilliz::vecwise;
// Not provide path: current work path will be used and system.info.
void
print_usage(const std::string &app_name) {
printf("\n Usage: %s [OPTIONS]\n\n", app_name.c_str());
printf(" Options:\n");
printf(" -h --help Print this help\n");
printf(" -s --sysinfo filename sysinfo file location\n");
printf(" -l --license filename Generate license file as given name\n");
printf(" -b --starting time Set start time (format: YYYY-MM-DD)\n");
printf(" -e --end time Set end time (format: YYYY-MM-DD)\n");
printf("\n");
}
int main(int argc, char *argv[]) {
std::string app_name = argv[0];
// if (argc != 1 && argc != 3) {
// print_usage(app_name);
// return EXIT_FAILURE;
// }
static struct option long_options[] = {{"system_info", required_argument, 0, 's'},
{"license", optional_argument, 0, 'l'},
{"help", no_argument, 0, 'h'},
{"starting_time", required_argument, 0, 'b'},
{"end_time", required_argument, 0, 'e'},
{NULL, 0, 0, 0}};
server::ServerError err;
int value = 0;
int option_index = 0;
std::string system_info_filename = "./system.info";
std::string license_filename = "./system.license";
char *string_starting_time = NULL;
char *string_end_time = NULL;
time_t starting_time = 0;
time_t end_time = 0;
int flag_s = 1;
int flag_b = 1;
int flag_e = 1;
while ((value = getopt_long(argc, argv, "hl:s:b:e:", long_options, NULL)) != -1) {
switch (value) {
case 's': {
flag_s = 0;
system_info_filename = (std::string) (optarg);
break;
}
case 'b': {
flag_b = 0;
string_starting_time = optarg;
break;
}
case 'e': {
flag_e = 0;
string_end_time = optarg;
break;
}
case 'l': {
license_filename = (std::string) (optarg);
break;
}
case 'h':print_usage(app_name);
return EXIT_SUCCESS;
case '?':print_usage(app_name);
return EXIT_FAILURE;
default:print_usage(app_name);
break;
}
}
if (flag_s) {
printf("Error: sysinfo file location must be entered\n");
return 1;
}
if (flag_b) {
printf("Error: start time must be entered\n");
return 1;
}
if (flag_e) {
printf("Error: end time must be entered\n");
return 1;
}
err = server::LicenseLibrary::GetDateTime(string_starting_time, starting_time);
if (err != server::SERVER_SUCCESS) return -1;
err = server::LicenseLibrary::GetDateTime(string_end_time, end_time);
if (err != server::SERVER_SUCCESS) return -1;
int output_info_device_count = 0;
std::map<int, std::string> output_info_uuid_encrption_map;
err = server::LicenseLibrary::GPUinfoFileDeserialization(system_info_filename,
output_info_device_count,
output_info_uuid_encrption_map);
if (err != server::SERVER_SUCCESS) return -1;
err = server::LicenseLibrary::LicenseFileSerialization(license_filename,
output_info_device_count,
output_info_uuid_encrption_map,
starting_time,
end_time);
if (err != server::SERVER_SUCCESS) return -1;
printf("Generate License File Success\n");
return 0;
}
}

345
cpp/src/license/LicenseLibrary.cpp Normal file
View File

@ -0,0 +1,345 @@
////////////////////////////////////////////////////////////////////////////////
// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved
// Unauthorized copying of this file, via any medium is strictly prohibited.
// Proprietary and confidential.
////////////////////////////////////////////////////////////////////////////////
#include "LicenseLibrary.h"
#include "utils/Log.h"
#include <cuda_runtime.h>
#include <nvml.h>
#include <openssl/md5.h>
#include <openssl/sha.h>
#include <boost/archive/binary_oarchive.hpp>
#include <boost/archive/binary_iarchive.hpp>
//#include <boost/foreach.hpp>
//#include <boost/serialization/vector.hpp>
#include <boost/filesystem/path.hpp>
#include <boost/serialization/map.hpp>
#include <boost/filesystem/operations.hpp>
namespace zilliz {
namespace vecwise {
namespace server {
constexpr int LicenseLibrary::sha256_length_;
// Part 0: File check
bool
LicenseLibrary::IsFileExistent(const std::string &path) {
boost::system::error_code error;
auto file_status = boost::filesystem::status(path, error);
if (error) {
return false;
}
if (!boost::filesystem::exists(file_status)) {
return false;
}
return !boost::filesystem::is_directory(file_status);
}
// Part 1: Get GPU Info
ServerError
LicenseLibrary::GetDeviceCount(int &device_count) {
nvmlReturn_t result = nvmlInit();
if (NVML_SUCCESS != result) {
printf("Failed to initialize NVML: %s\n", nvmlErrorString(result));
return SERVER_UNEXPECTED_ERROR;
}
cudaError_t error_id = cudaGetDeviceCount(&device_count);
if (error_id != cudaSuccess) {
printf("cudaGetDeviceCount returned %d\n-> %s\n", (int) error_id, cudaGetErrorString(error_id));
printf("Result = FAIL\n");
return SERVER_UNEXPECTED_ERROR;
}
return SERVER_SUCCESS;
}
ServerError
LicenseLibrary::GetUUID(int device_count, std::vector<std::string> &uuid_array) {
if (device_count == 0) {
printf("There are no available device(s) that support CUDA\n");
return SERVER_UNEXPECTED_ERROR;
}
for (int dev = 0; dev < device_count; ++dev) {
nvmlDevice_t device;
nvmlReturn_t result = nvmlDeviceGetHandleByIndex(dev, &device);
if (NVML_SUCCESS != result) {
printf("Failed to get handle for device %i: %s\n", dev, nvmlErrorString(result));
return SERVER_UNEXPECTED_ERROR;
}
char uuid[80];
unsigned int length = 80;
nvmlReturn_t err = nvmlDeviceGetUUID(device, uuid, length);
if (err != NVML_SUCCESS) {
printf("nvmlDeviceGetUUID error: %d\n", err);
return SERVER_UNEXPECTED_ERROR;
}
uuid_array.emplace_back(uuid);
}
return SERVER_SUCCESS;
}
ServerError
LicenseLibrary::GetUUIDMD5(int device_count,
std::vector<std::string> &uuid_array,
std::vector<std::string> &md5_array) {
MD5_CTX ctx;
unsigned char outmd[16];
char temp[2];
std::string md5;
for (int dev = 0; dev < device_count; ++dev) {
md5.clear();
memset(outmd, 0, sizeof(outmd));
MD5_Init(&ctx);
MD5_Update(&ctx, uuid_array[dev].c_str(), uuid_array[dev].size());
MD5_Final(outmd, &ctx);
for (int i = 0; i < 16; ++i) {
std::snprintf(temp, 2, "%02X", outmd[i]);
md5 += temp;
}
md5_array.push_back(md5);
}
return SERVER_SUCCESS;
}
ServerError
LicenseLibrary::GetUUIDSHA256(const int &device_count,
std::vector<std::string> &uuid_array,
std::vector<std::string> &sha_array) {
SHA256_CTX ctx;
unsigned char outmd[sha256_length_];
char temp[2];
std::string sha;
for (int dev = 0; dev < device_count; ++dev) {
sha.clear();
memset(outmd, 0, sizeof(outmd));
SHA256_Init(&ctx);
SHA256_Update(&ctx, uuid_array[dev].c_str(), uuid_array[dev].size());
SHA256_Final(outmd, &ctx);
for (int i = 0; i < sha256_length_; ++i) {
std::snprintf(temp, 2, "%02X", outmd[i]);
sha += temp;
}
sha_array.push_back(sha);
}
return SERVER_SUCCESS;
}
ServerError
LicenseLibrary::GetSystemTime(time_t &system_time) {
system_time = time(NULL);
return SERVER_SUCCESS;
}
// Part 2: Handle License File
ServerError
LicenseLibrary::LicenseFileSerialization(const std::string &path,
int device_count,
const std::map<int, std::string> &uuid_encrption_map,
time_t starting_time,
time_t end_time) {
std::ofstream file(path);
boost::archive::binary_oarchive oa(file);
oa.register_type<LicenseFile>();
SerializedLicenseFile serialized_license_file;
serialized_license_file.set_license_file(new LicenseFile(device_count,
uuid_encrption_map,
starting_time,
end_time));
oa << serialized_license_file;
file.close();
return SERVER_SUCCESS;
}
ServerError
LicenseLibrary::LicenseFileDeserialization(const std::string &path,
int &device_count,
std::map<int, std::string> &uuid_encrption_map,
time_t &starting_time,
time_t &end_time) {
if (!IsFileExistent(path)) return SERVER_LICENSE_FILE_NOT_EXIST;
std::ifstream file(path);
boost::archive::binary_iarchive ia(file);
ia.register_type<LicenseFile>();
SerializedLicenseFile serialized_license_file;
ia >> serialized_license_file;
device_count = serialized_license_file.get_license_file()->get_device_count();
uuid_encrption_map = serialized_license_file.get_license_file()->get_uuid_encryption_map();
starting_time = serialized_license_file.get_license_file()->get_starting_time();
end_time = serialized_license_file.get_license_file()->get_end_time();
file.close();
return SERVER_SUCCESS;
}
//ServerError
//LicenseLibrary::SecretFileSerialization(const std::string &path,
// const time_t &update_time,
// const off_t &file_size,
// const time_t &starting_time,
// const time_t &end_time,
// const std::string &file_md5) {
// std::ofstream file(path);
// boost::archive::binary_oarchive oa(file);
// oa.register_type<SecretFile>();
//
// SerializedSecretFile serialized_secret_file;
//
// serialized_secret_file.set_secret_file(new SecretFile(update_time, file_size, starting_time, end_time, file_md5));
// oa << serialized_secret_file;
//
// file.close();
// return SERVER_SUCCESS;
//}
//
//ServerError
//LicenseLibrary::SecretFileDeserialization(const std::string &path,
// time_t &update_time,
// off_t &file_size,
// time_t &starting_time,
// time_t &end_time,
// std::string &file_md5) {
// if (!IsFileExistent(path)) return SERVER_LICENSE_FILE_NOT_EXIST;
//
// std::ifstream file(path);
// boost::archive::binary_iarchive ia(file);
// ia.register_type<SecretFile>();
// SerializedSecretFile serialized_secret_file;
//
// ia >> serialized_secret_file;
// update_time = serialized_secret_file.get_secret_file()->get_update_time();
// file_size = serialized_secret_file.get_secret_file()->get_file_size();
// starting_time = serialized_secret_file.get_secret_file()->get_starting_time();
// end_time = serialized_secret_file.get_secret_file()->get_end_time();
// file_md5 = serialized_secret_file.get_secret_file()->get_file_md5();
// file.close();
// return SERVER_SUCCESS;
//}
// Part 3: File attribute: UpdateTime Time/ Size/ MD5
ServerError
LicenseLibrary::GetFileUpdateTimeAndSize(const std::string &path, time_t &update_time, off_t &file_size) {
if (!IsFileExistent(path)) return SERVER_LICENSE_FILE_NOT_EXIST;
struct stat buf;
int err_no = stat(path.c_str(), &buf);
if (err_no != 0) {
std::cout << strerror(err_no) << std::endl;
return SERVER_UNEXPECTED_ERROR;
}
update_time = buf.st_mtime;
file_size = buf.st_size;
return SERVER_SUCCESS;
}
ServerError
LicenseLibrary::GetFileMD5(const std::string &path, std::string &filemd5) {
if (!IsFileExistent(path)) return SERVER_LICENSE_FILE_NOT_EXIST;
filemd5.clear();
std::ifstream file(path.c_str(), std::ifstream::binary);
if (!file) {
return -1;
}
MD5_CTX md5Context;
MD5_Init(&md5Context);
char buf[1024 * 16];
while (file.good()) {
file.read(buf, sizeof(buf));
MD5_Update(&md5Context, buf, file.gcount());
}
unsigned char result[MD5_DIGEST_LENGTH];
MD5_Final(result, &md5Context);
char hex[35];
memset(hex, 0, sizeof(hex));
for (int i = 0; i < MD5_DIGEST_LENGTH; ++i) {
sprintf(hex + i * 2, "%02X", result[i]);
}
hex[32] = '\0';
filemd5 = std::string(hex);
return SERVER_SUCCESS;
}
// Part 4: GPU Info File Serialization/Deserialization
ServerError
LicenseLibrary::GPUinfoFileSerialization(const std::string &path,
int device_count,
const std::map<int, std::string> &uuid_encrption_map) {
std::ofstream file(path);
boost::archive::binary_oarchive oa(file);
oa.register_type<GPUInfoFile>();
SerializedGPUInfoFile serialized_gpu_info_file;
serialized_gpu_info_file.set_gpu_info_file(new GPUInfoFile(device_count, uuid_encrption_map));
oa << serialized_gpu_info_file;
file.close();
return SERVER_SUCCESS;
}
ServerError
LicenseLibrary::GPUinfoFileDeserialization(const std::string &path,
int &device_count,
std::map<int, std::string> &uuid_encrption_map) {
if (!IsFileExistent(path)) return SERVER_LICENSE_FILE_NOT_EXIST;
std::ifstream file(path);
boost::archive::binary_iarchive ia(file);
ia.register_type<GPUInfoFile>();
SerializedGPUInfoFile serialized_gpu_info_file;
ia >> serialized_gpu_info_file;
device_count = serialized_gpu_info_file.get_gpu_info_file()->get_device_count();
uuid_encrption_map = serialized_gpu_info_file.get_gpu_info_file()->get_uuid_encryption_map();
file.close();
return SERVER_SUCCESS;
}
ServerError
LicenseLibrary::GetDateTime(char *cha, time_t &data_time) {
tm tm_;
int year, month, day;
sscanf(cha, "%d-%d-%d", &year, &month, &day);
tm_.tm_year = year - 1900;
tm_.tm_mon = month - 1;
tm_.tm_mday = day;
tm_.tm_hour = 0;
tm_.tm_min = 0;
tm_.tm_sec = 0;
tm_.tm_isdst = 0;
data_time = mktime(&tm_);
return SERVER_SUCCESS;
}
}
}
}

105
cpp/src/license/LicenseLibrary.h Normal file
View File

@ -0,0 +1,105 @@
#pragma once
#include "LicenseFile.h"
#include "GPUInfoFile.h"
#include "utils/Error.h"
#include <boost/asio.hpp>
#include <boost/thread.hpp>
#include <boost/date_time/posix_time/posix_time.hpp>
#include <vector>
#include <map>
#include <time.h>
namespace zilliz {
namespace vecwise {
namespace server {
class LicenseLibrary {
public:
// Part 0: File check
static bool
IsFileExistent(const std::string &path);
// Part 1: Get GPU Info
static ServerError
GetDeviceCount(int &device_count);
static ServerError
GetUUID(int device_count, std::vector<std::string> &uuid_array);
static ServerError
GetUUIDMD5(int device_count, std::vector<std::string> &uuid_array, std::vector<std::string> &md5_array);
static ServerError
GetUUIDSHA256(const int &device_count,
std::vector<std::string> &uuid_array,
std::vector<std::string> &sha_array);
static ServerError
GetSystemTime(time_t &system_time);
// Part 2: Handle License File
static ServerError
LicenseFileSerialization(const std::string &path,
int device_count,
const std::map<int, std::string> &uuid_encrption_map,
time_t starting_time,
time_t end_time);
static ServerError
LicenseFileDeserialization(const std::string &path,
int &device_count,
std::map<int, std::string> &uuid_encrption_map,
time_t &starting_time,
time_t &end_time);
// static ServerError
// SecretFileSerialization(const std::string &path,
// const time_t &update_time,
// const off_t &file_size,
// const time_t &starting_time,
// const time_t &end_time,
// const std::string &file_md5);
//
// static ServerError
// SecretFileDeserialization(const std::string &path,
// time_t &update_time,
// off_t &file_size,
// time_t &starting_time,
// time_t &end_time,
// std::string &file_md5);
// Part 3: File attribute: UpdateTime Time/ Size/ MD5
static ServerError
GetFileUpdateTimeAndSize(const std::string &path, time_t &update_time, off_t &file_size);
static ServerError
GetFileMD5(const std::string &path, std::string &filemd5);
// Part 4: GPU Info File Serialization/Deserialization
static ServerError
GPUinfoFileSerialization(const std::string &path,
int device_count,
const std::map<int, std::string> &uuid_encrption_map);
static ServerError
GPUinfoFileDeserialization(const std::string &path,
int &device_count,
std::map<int, std::string> &uuid_encrption_map);
static ServerError
GetDateTime(char *cha, time_t &data_time);
private:
static constexpr int sha256_length_ = 32;
};
}
}
}

23
cpp/src/server/Server.cpp
View File

@ -9,7 +9,7 @@
#include "utils/Log.h"
#include "utils/SignalUtil.h"
#include "utils/TimeRecorder.h"
#include "license/License.h"
#include "license/LicenseCheck.h"
#include <fcntl.h>
#include <sys/stat.h>
@ -133,12 +133,6 @@ Server::Daemonize() {
int
Server::Start() {
std::string license_file_path = "/tmp/vecwise.license";
if(LicenseValidate(license_file_path) != SERVER_SUCCESS) {
SERVER_LOG_ERROR << "License check failed";
return 1;
}
if (daemonized_) {
Daemonize();
}
@ -157,6 +151,21 @@ Server::Start() {
//print config into console and log
config.PrintAll();
#ifdef ENABLE_LICENSE
ConfigNode license_config = config.GetConfig(CONFIG_LICENSE);
std::string license_file_path = license_config.GetValue(CONFIG_LICENSE_PATH);
SERVER_LOG_INFO << "License path: " << license_file_path;
if(server::LicenseCheck::LegalityCheck(license_file_path) != SERVER_SUCCESS) {
SERVER_LOG_ERROR << "License check failed";
exit(1);
}
if(server::LicenseCheck::StartCountingDown(license_file_path) != SERVER_SUCCESS) {
SERVER_LOG_ERROR << "License counter start error";
exit(1);
}
#endif
// Handle Signal
signal(SIGINT, SignalUtil::HandleSignal);
signal(SIGHUP, SignalUtil::HandleSignal);

3
cpp/src/server/ServerConfig.h
View File

@ -32,6 +32,9 @@ static const std::string CONFIG_CACHE = "cache_config";
static const std::string CONFIG_CPU_CACHE_CAPACITY = "cpu_cache_capacity";
static const std::string CONFIG_GPU_CACHE_CAPACITY = "gpu_cache_capacity";
static const std::string CONFIG_LICENSE = "license_config";
static const std::string CONFIG_LICENSE_PATH = "license_path";
class ServerConfig {
public:
static ServerConfig &GetInstance();

2
cpp/src/utils/Error.h
View File

@ -31,6 +31,8 @@ constexpr ServerError SERVER_INVALID_ARGUMENT = ToGlobalServerErrorCode(0x004);
constexpr ServerError SERVER_FILE_NOT_FOUND = ToGlobalServerErrorCode(0x005);
constexpr ServerError SERVER_NOT_IMPLEMENT = ToGlobalServerErrorCode(0x006);
constexpr ServerError SERVER_BLOCKING_QUEUE_EMPTY = ToGlobalServerErrorCode(0x007);
constexpr ServerError SERVER_LICENSE_VALIDATION_FAIL = ToGlobalServerErrorCode(0x008);
constexpr ServerError SERVER_LICENSE_FILE_NOT_EXIST = ToGlobalServerErrorCode(0x009);
class ServerException : public std::exception {
public:

33
cpp/unittest/license/CMakeLists.txt
View File

@ -11,34 +11,37 @@ aux_source_directory(../../src/cache cache_srcs)
aux_source_directory(../../src/wrapper wrapper_src)
include_directories(/usr/local/cuda/include)
link_directories("/usr/local/cuda/lib64")
link_directories(/usr/local/cuda)
link_directories(/usr/local/cuda/lib64)
link_directories(/usr/lib/x86_64-linux-gnu)
link_directories(/usr/lib/nvidia-415)
link_directories(/usr/local/cuda/targets/x86_64-linux/lib/stubs/)
set(require_files
../../src/server/ServerConfig.cpp
../../src/utils/CommonUtil.cpp
../../src/utils/TimeRecorder.cpp
../../src/license/License.cpp
)
../../src/license/LicenseLibrary.cpp
../../src/license/LicenseCheck.cpp
)
set(db_test_src
${unittest_srcs}
${config_files}
${cache_srcs}
${db_srcs}
${wrapper_src}
${require_files}
license_tests.cpp)
license_library_tests.cpp
license_check_test.cpp
${require_files})
cuda_add_executable(license_test ${db_test_src})
set(db_libs
faiss
nvidia-ml
cudart
cublas
sqlite3
boost_system
boost_filesystem
lz4
crypto
boost_serialization
)
target_link_libraries(license_test ${unittest_libs} ${db_libs})

184
cpp/unittest/license/license_check_test.cpp Normal file
View File

@ -0,0 +1,184 @@
//
// Created by zilliz on 19-5-13.
//
#include "utils/Log.h"
#include "license/LicenseCheck.h"
#include "utils/Error.h"
#include <gtest/gtest.h>
#include <iostream>
using namespace zilliz::vecwise;
TEST(LicenseLibraryTest, CHECK_TEST) {
server::ServerError err;
// 1. Set license file name
std::string license_file_path("/tmp/megasearch/abc.license");
// 2. Legality check
err = server::LicenseCheck::LegalityCheck(license_file_path);
ASSERT_EQ(err, server::SERVER_SUCCESS);
}
TEST(LicenseLibraryTest, CHECK_ERROR1_TEST){
server::ServerError err;
// 1. Set license file name
std::string license_file_path("/tmp/megasearch/abc");
// 2. Legality check
err = server::LicenseCheck::LegalityCheck(license_file_path);
ASSERT_EQ(err, server::SERVER_SUCCESS);
}
TEST(LicenseLibraryTest, CHECK_ERROR2_TEST){
server::ServerError err;
// 1. Set license file name
std::string license_file_path("/tmp/megasearch/abc.license");
// 2. Define output var
int device_count;
std::map<int, std::string> uuid_encryption_map;
time_t starting_time;
time_t end_time;
// 3. Read License File
err = server::LicenseLibrary::LicenseFileDeserialization(license_file_path,
device_count,
uuid_encryption_map,
starting_time,
end_time);
ASSERT_EQ(err, server::SERVER_SUCCESS);
// 4. Change device count
++device_count;
err = server::LicenseLibrary::LicenseFileSerialization(license_file_path,
device_count,
uuid_encryption_map,
starting_time,
end_time);
ASSERT_EQ(err, server::SERVER_SUCCESS);
// 5. Legality check
err = server::LicenseCheck::LegalityCheck(license_file_path);
ASSERT_EQ(err, server::SERVER_SUCCESS);
}
TEST(LicenseLibraryTest, CHECK_ERROR3_TEST){
server::ServerError err;
// 1. Set license file name
std::string license_file_path("/tmp/megasearch/abc.license");
// 2. Define output var
int device_count;
std::map<int, std::string> uuid_encryption_map;
time_t starting_time;
time_t end_time;
// 3. Read License File
err = server::LicenseLibrary::LicenseFileDeserialization(license_file_path,
device_count,
uuid_encryption_map,
starting_time,
end_time);
ASSERT_EQ(err, server::SERVER_SUCCESS);
// 4. Change device count
if(device_count) uuid_encryption_map[0]+="u";
err = server::LicenseLibrary::LicenseFileSerialization(license_file_path,
device_count,
uuid_encryption_map,
starting_time,
end_time);
ASSERT_EQ(err, server::SERVER_SUCCESS);
// 5. Legality check
err = server::LicenseCheck::LegalityCheck(license_file_path);
ASSERT_EQ(err, server::SERVER_SUCCESS);
}
TEST(LicenseLibraryTest, CHECK_ERROR4_1_TEST){
server::ServerError err;
// 1. Set license file name
std::string license_file_path("/tmp/megasearch/abc.license");
// 2. Define output var
int device_count;
std::map<int, std::string> uuid_encryption_map;
time_t starting_time;
time_t end_time;
// 3. Read License File
err = server::LicenseLibrary::LicenseFileDeserialization(license_file_path,
device_count,
uuid_encryption_map,
starting_time,
end_time);
ASSERT_EQ(err, server::SERVER_SUCCESS);
// 4. Change starting time
time_t system_time;
err = server::LicenseLibrary::GetSystemTime(system_time);
ASSERT_EQ(err, server::SERVER_SUCCESS);
system_time+=60*60*24;
err = server::LicenseLibrary::LicenseFileSerialization(license_file_path,
device_count,
uuid_encryption_map,
system_time,
end_time);
ASSERT_EQ(err, server::SERVER_SUCCESS);
// 5. Legality check
err = server::LicenseCheck::LegalityCheck(license_file_path);
ASSERT_EQ(err, server::SERVER_SUCCESS);
}
TEST(LicenseLibraryTest, CHECK_ERROR4_2_TEST){
server::ServerError err;
// 1. Set license file name
std::string license_file_path("/tmp/megasearch/abc.license");
// 2. Define output var
int device_count;
std::map<int, std::string> uuid_encryption_map;
time_t starting_time;
time_t end_time;
// 3. Read License File
err = server::LicenseLibrary::LicenseFileDeserialization(license_file_path,
device_count,
uuid_encryption_map,
starting_time,
end_time);
ASSERT_EQ(err, server::SERVER_SUCCESS);
// 4. Change end time
time_t system_time;
err = server::LicenseLibrary::GetSystemTime(system_time);
ASSERT_EQ(err, server::SERVER_SUCCESS);
system_time-=100;
err = server::LicenseLibrary::LicenseFileSerialization(license_file_path,
device_count,
uuid_encryption_map,
starting_time,
system_time);
ASSERT_EQ(err, server::SERVER_SUCCESS);
// 5. Legality check
err = server::LicenseCheck::LegalityCheck(license_file_path);
ASSERT_EQ(err, server::SERVER_SUCCESS);
}

214
cpp/unittest/license/license_library_tests.cpp Normal file
View File

@ -0,0 +1,214 @@
/*******************************************************************************
* Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved
* Unauthorized copying of this file, via any medium is strictly prohibited.
* Proprietary and confidential.
******************************************************************************/
#include "utils/Log.h"
#include "license/LicenseLibrary.h"
#include "utils/Error.h"
#include <gtest/gtest.h>
#include <iostream>
using namespace zilliz::vecwise;
TEST(LicenseLibraryTest, FILE_EXISTENT_TEST) {
std::string hosts_file = "/etc/hosts";
ASSERT_EQ(server::LicenseLibrary::IsFileExistent(hosts_file), true);
std::string no_exist_file = "/temp/asdaasd";
ASSERT_EQ(server::LicenseLibrary::IsFileExistent(no_exist_file), false);
std::string directory = "/tmp";
ASSERT_EQ(server::LicenseLibrary::IsFileExistent(directory), false);
}
TEST(LicenseLibraryTest, GPU_INFO_TEST) {
int device_count = 0;
server::ServerError err = server::LicenseLibrary::GetDeviceCount(device_count);
ASSERT_EQ(err, server::SERVER_SUCCESS);
std::cout << "Device Count: " << device_count << std::endl;
std::vector<std::string> uuid_array;
err = server::LicenseLibrary::GetUUID(device_count, uuid_array);
ASSERT_EQ(err, server::SERVER_SUCCESS);
for (long i = 0; i < device_count; ++i) {
std::cout << "Device Id: " << i << ", UUID: " << uuid_array[i] << std::endl;
}
std::vector<std::string> uuid_md5_array;
err = server::LicenseLibrary::GetUUIDMD5(device_count, uuid_array, uuid_md5_array);
ASSERT_EQ(err, server::SERVER_SUCCESS);
for (long i = 0; i < device_count; ++i) {
std::cout << "Device Id: " << i << ", UUID: " << uuid_array[i] << ", UUID_MD5: " << uuid_md5_array[i]
<< std::endl;
}
std::vector<std::string> uuid_sha256_array;
err = server::LicenseLibrary::GetUUIDSHA256(device_count, uuid_array, uuid_sha256_array);
ASSERT_EQ(err, server::SERVER_SUCCESS);
for (long i = 0; i < device_count; ++i) {
std::cout << "Device Id: " << i << ", UUID: " << uuid_array[i] << ", UUID_SHA256: "
<< uuid_sha256_array[i] << std::endl;
}
time_t systemtime;
err = server::LicenseLibrary::GetSystemTime(systemtime);
ASSERT_EQ(err, server::SERVER_SUCCESS);
std::cout << "System Time: " << systemtime << std::endl;
}
TEST(LicenseLibraryTest, LICENSE_FILE_TEST) {
// 0. File check
std::string test("/tmp/a.test");
bool is = server::LicenseLibrary::IsFileExistent(test);
ASSERT_EQ(is, false);
// 1. Get Device Count
int device_count = 0;
server::ServerError err = server::LicenseLibrary::GetDeviceCount(device_count);
ASSERT_EQ(err, server::SERVER_SUCCESS);
// 2. Get All GPU UUID
std::vector<std::string> uuid_array;
err = server::LicenseLibrary::GetUUID(device_count, uuid_array);
ASSERT_EQ(err, server::SERVER_SUCCESS);
// 3. Get UUID SHA256
std::vector<std::string> uuid_sha256_array;
err = server::LicenseLibrary::GetUUIDSHA256(device_count, uuid_array, uuid_sha256_array);
ASSERT_EQ(err, server::SERVER_SUCCESS);
// 4. Generate GPU ID map with GPU UUID
std::map<int, std::string> uuid_encrption_map;
for (int i = 0; i < device_count; ++i) {
uuid_encrption_map[i] = uuid_sha256_array[i];
}
// 5.GPU_info File
std::string GPU_info_file_path("/tmp/megasearch.info");
// 6. Generate GPU_info File
err = server::LicenseLibrary::GPUinfoFileSerialization(GPU_info_file_path,
device_count,
uuid_encrption_map);
ASSERT_EQ(err, server::SERVER_SUCCESS);
// 7. Define output var
int output_info_device_count = 0;
std::map<int, std::string> output_info_uuid_encrption_map;
// 8. Read GPU_info File
err = server::LicenseLibrary::GPUinfoFileDeserialization(GPU_info_file_path,
output_info_device_count,
output_info_uuid_encrption_map);
ASSERT_EQ(err, server::SERVER_SUCCESS);
ASSERT_EQ(device_count, output_info_device_count);
for (int i = 0; i < device_count; ++i) {
ASSERT_EQ(uuid_encrption_map[i], output_info_uuid_encrption_map[i]);
}
// 9. Set license file name
std::string license_file_path("/tmp/megasearch.license");
// 10. Get System Time/starting_time ans End Time
time_t sysyem_time;
err = server::LicenseLibrary::GetSystemTime(sysyem_time);
ASSERT_EQ(err, server::SERVER_SUCCESS);
// 11.GetDateTime
time_t starting_time;
time_t end_time;
char *string_starting_time = "2019-05-10";
char *string_end_time = "2022-05-10";
err = server::LicenseLibrary::GetDateTime(string_starting_time, starting_time);
ASSERT_EQ(err, server::SERVER_SUCCESS);
err = server::LicenseLibrary::GetDateTime(string_end_time, end_time);
ASSERT_EQ(err, server::SERVER_SUCCESS);
// 12. Generate License File
err = server::LicenseLibrary::LicenseFileSerialization(license_file_path,
device_count,
uuid_encrption_map,
starting_time,
end_time);
ASSERT_EQ(err, server::SERVER_SUCCESS);
// 13. Define output var
int output_device_count = 0;
std::map<int, std::string> output_uuid_encrption_map;
time_t output_starting_time;
time_t output_end_time;
// 14. Read License File
err = server::LicenseLibrary::LicenseFileDeserialization(license_file_path,
output_device_count,
output_uuid_encrption_map,
output_starting_time,
output_end_time);
ASSERT_EQ(err, server::SERVER_SUCCESS);
ASSERT_EQ(device_count, output_device_count);
ASSERT_EQ(starting_time, output_starting_time);
ASSERT_EQ(end_time, output_end_time);
for (int i = 0; i < device_count; ++i) {
ASSERT_EQ(uuid_encrption_map[i], output_uuid_encrption_map[i]);
}
// // 15. Get License File Attribute
// time_t update_time;
// off_t file_size;
// err = server::LicenseLibrary::GetFileUpdateTimeAndSize(license_file_path, update_time, file_size);
// ASSERT_EQ(err, server::SERVER_SUCCESS);
//
// // 16. Get License File MD5
// std::string file_md5;
// err = server::LicenseLibrary::GetFileMD5(license_file_path, file_md5);
// ASSERT_EQ(err, server::SERVER_SUCCESS);
// // 17. Generate Secret File
// std::string secret_file_path("/tmp/megasearch.secret");
// err = server::LicenseLibrary::SecretFileSerialization(secret_file_path,
// update_time,
// file_size,
// starting_time,
// end_time,
// file_md5);
// ASSERT_EQ(err, server::SERVER_SUCCESS);
// // 18. Define output var
// time_t output_update_time;
// off_t output_file_size;
// time_t output_starting_time;
// time_t output_end_time;
// std::string output_file_md5;
// // 19. Read License File
// err = server::LicenseLibrary::SecretFileDeserialization(secret_file_path,
// output_update_time,
// output_file_size,
// output_starting_time,
// output_end_time,
// output_file_md5);
// ASSERT_EQ(err, server::SERVER_SUCCESS);
//
// ASSERT_EQ(update_time, output_update_time);
// ASSERT_EQ(file_size, output_file_size);
// ASSERT_EQ(starting_time, output_starting_time);
// ASSERT_EQ(end_time, output_end_time);
// ASSERT_EQ(file_md5, output_file_md5);
}

16
cpp/unittest/license/license_tests.cpp
View File

@ -1,16 +0,0 @@
////////////////////////////////////////////////////////////////////////////////
// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved
// Unauthorized copying of this file, via any medium is strictly prohibited.
// Proprietary and confidential.
////////////////////////////////////////////////////////////////////////////////
#include <gtest/gtest.h>
#include "license/License.h"
#include "utils/Error.h"
using namespace zilliz::vecwise;
TEST(LicenseTest, LICENSE_TEST) {
std::string path1 = "/tmp/vecwise_engine.license";
server::ServerError err = server::LicenseValidate(path1);
ASSERT_EQ(err, server::SERVER_SUCCESS);
}