Merge branch 'branch-0.3.1' of http://192.168.1.105:6060/megasearch/vecwise_engine into branch-0.3.1

Former-commit-id: d456d64d94d7eba0f6d9784ff6b92c24f510f0d6

cpp/.gitignore

@@ -1,4 +1,10 @@
 milvus/
 conf/server_config.yaml
+conf/log_config.conf
 version.h
 megasearch/
+lcov_out/
+base.info
+output.info
+output_new.info
+server.info

cpp/CHANGELOG.md

@@ -36,6 +36,8 @@ Please mark all change in change log and use the ticket from JIRA.
 - MS-115 - Change is_startup of metric_config switch from true to on
 - MS-122 - Archive criteria config 
 - MS-124 - HasTable interface
+- MS-126 - Add more error code
+- MS-128 - Change default db path
 
 ## New Feature
 
@@ -57,7 +59,7 @@ Please mark all change in change log and use the ticket from JIRA.
 - MS-85 - add NetIO metric
 - MS-96 - add new query interface for specified files
 - MS-97 - Add S3 SDK for MinIO Storage
-
+- MS-130 - Add prometheus_test
 ## Task
 - MS-74 - Change README.md in cpp
 - MS-88 - Add support for arm architecture

cpp/CMakeLists.txt

@@ -135,12 +135,19 @@ if (BUILD_COVERAGE STREQUAL "ON")
 endif()
 
 
-if (BUILD_UNIT_TEST)
+if (BUILD_UNIT_TEST STREQUAL "ON")
     add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/unittest)
-endif(BUILD_UNIT_TEST)
+endif()
 
 add_custom_target(Clean-All COMMAND ${CMAKE_BUILD_TOOL} clean)
 
+
+if("${MILVUS_DB_PATH}" STREQUAL "")
+    set(MILVUS_DB_PATH "/tmp/milvus")
+endif()
+configure_file(${CMAKE_CURRENT_SOURCE_DIR}/conf/server_config.template ${CMAKE_CURRENT_SOURCE_DIR}/conf/server_config.yaml)
+configure_file(${CMAKE_CURRENT_SOURCE_DIR}/conf/log_config.template ${CMAKE_CURRENT_SOURCE_DIR}/conf/log_config.conf)
+
 #install
 install(FILES
         scripts/start_server.sh
@@ -153,4 +160,6 @@ install(FILES
         DESTINATION
         conf)
 
+
+
 config_summary()

cpp/README.md

@@ -36,6 +36,11 @@ If you encounter the following error when building:
     or
     ./build.sh --unittest
 
+#### To run code coverage
+
+    apt-get install lcov
+    ./build.sh -u -c
+
 ### Launch server
 Set config in cpp/conf/server_config.yaml
 

cpp/build.sh

@@ -1,13 +1,14 @@
 #!/bin/bash
 
 BUILD_TYPE="Debug"
-BUILD_UNITTEST="off"
+BUILD_UNITTEST="OFF"
 LICENSE_CHECK="OFF"
 INSTALL_PREFIX=$(pwd)/milvus
 MAKE_CLEAN="OFF"
 BUILD_COVERAGE="OFF"
+DB_PATH="/opt/milvus"
 
-while getopts "p:t:uhlrc" arg
+while getopts "p:d:t:uhlrc" arg
 do
         case $arg in
              t)
@@ -15,11 +16,14 @@ do
                 ;;
              u)
                 echo "Build and run unittest cases" ;
-                BUILD_UNITTEST="on";
+                BUILD_UNITTEST="ON";
                 ;;
              p)
                 INSTALL_PREFIX=$OPTARG
                 ;;
+             d)
+                DB_PATH=$OPTARG
+                ;;
              l)
                 LICENSE_CHECK="ON"
                 ;;
@@ -36,12 +40,13 @@ do
                 echo "
 
 parameter:
--t: build type
--u: building unit test options
--p: install prefix
--l: build license version
--r: remove previous build directory
--c: code coverage
+-t: build type(default: Debug)
+-u: building unit test options(default: OFF)
+-p: install prefix(default: $(pwd)/milvus)
+-d: db path(default: /opt/milvus)
+-l: build license version(default: OFF)
+-r: remove previous build directory(default: OFF)
+-c: code coverage(default: OFF)
 
 usage:
 ./build.sh -t \${BUILD_TYPE} [-u] [-h] [-g] [-r] [-c]
@@ -71,6 +76,7 @@ if [[ ${MAKE_CLEAN} == "ON" ]]; then
     -DCMAKE_CUDA_COMPILER=${CUDA_COMPILER} \
     -DCMAKE_LICENSE_CHECK=${LICENSE_CHECK} \
     -DBUILD_COVERAGE=${BUILD_COVERAGE} \
+    -DMILVUS_DB_PATH=${DB_PATH} \
     $@ ../"
     echo ${CMAKE_CMD}
 

cpp/conf/log_config.template

@@ -0,0 +1,27 @@
+* GLOBAL:
+    FORMAT                  =   "%datetime | %level | %logger | %msg"
+    FILENAME                =   "@MILVUS_DB_PATH@/logs/milvus-%datetime{%H:%m}-global.log"
+    ENABLED                 =   true
+    TO_FILE                 =   true
+    TO_STANDARD_OUTPUT      =   false
+    SUBSECOND_PRECISION     =   3
+    PERFORMANCE_TRACKING    =   false
+    MAX_LOG_FILE_SIZE       =   2097152 ## Throw log files away after 2MB
+* DEBUG:
+    FILENAME                =   "@MILVUS_DB_PATH@/logs/milvus-%datetime{%H:%m}-debug.log"
+    ENABLED                 =   true
+* WARNING:
+    FILENAME                =   "@MILVUS_DB_PATH@/logs/milvus-%datetime{%H:%m}-warning.log"
+* TRACE:
+    FILENAME                =   "@MILVUS_DB_PATH@/logs/milvus-%datetime{%H:%m}-trace.log"
+* VERBOSE:
+    FORMAT                  =   "%datetime{%d/%M/%y} | %level-%vlevel | %msg"
+    TO_FILE                 =   false
+    TO_STANDARD_OUTPUT      =   false
+## Error logs
+* ERROR:
+    ENABLED                 =   true
+    FILENAME                =   "@MILVUS_DB_PATH@/logs/milvus-%datetime{%H:%m}-error.log"
+* FATAL:
+    ENABLED                 =   true
+    FILENAME                =   "@MILVUS_DB_PATH@/logs/milvus-%datetime{%H:%m}-fatal.log"

cpp/conf/server_config.template

@@ -0,0 +1,28 @@
+server_config:
+  address: 0.0.0.0
+  port: 19530                 # the port milvus listen to, default: 19530, range: 1025 ~ 65534
+  gpu_index: 0                # the gpu milvus use, default: 0, range: 0 ~ gpu number - 1
+  mode: single                # milvus deployment type: single, cluster
+
+db_config:
+  db_path: @MILVUS_DB_PATH@             # milvus data storage path
+  db_backend_url: http://127.0.0.1 # meta database uri
+  index_building_threshold: 1024   # index building trigger threshold, default: 1024, unit: MB
+  archive_disk_threshold: 512      # triger archive action if storage size exceed this value, unit: GB
+  archive_days_threshold: 30       # files older than x days will be archived, unit: day
+
+metric_config:
+  is_startup: off      # if monitoring start: on, off
+  collector: prometheus # metrics collector: prometheus
+  prometheus_config:    # following are prometheus configure
+    collect_type: pull  # prometheus collect data method
+    port: 8080          # the port prometheus use to fetch metrics
+    push_gateway_ip_address: 127.0.0.1 # push method configure: push gateway ip address
+    push_gateway_port: 9091            # push method configure: push gateway port
+
+
+license_config:         # license configure
+  license_path: "@MILVUS_DB_PATH@/system.license" # license file path
+
+cache_config:           # cache configure
+  cpu_cache_capacity: 16   # how many memory are used as cache, unit: GB, range: 0 ~ less than total memory

cpp/src/db/DBMetaImpl.cpp

@@ -192,9 +192,11 @@ Status DBMetaImpl::CreateTable(TableSchema &table_schema) {
             auto table = ConnectorPtr->select(columns(&TableSchema::state_),
                                                where(c(&TableSchema::table_id_) == table_schema.table_id_));
             if (table.size() == 1) {
-                std::string msg = (TableSchema::TO_DELETE == std::get<0>(table[0])) ?
-                          "Table already exists and it is in delete state, please wait a second" : "Table already exists";
-                return Status::Error(msg);
+                if(TableSchema::TO_DELETE == std::get<0>(table[0])) {
+                    return Status::Error("Table already exists and it is in delete state, please wait a second");
+                } else {
+                    return Status::OK();//table already exists, no error
+                }
             }
         }
 
@@ -328,7 +330,7 @@ Status DBMetaImpl::HasTable(const std::string &table_id, bool &has_or_not) {
         }
 
     } catch (std::exception &e) {
-        HandleException("Encounter exception when lookup table", e);
+        return HandleException("Encounter exception when lookup table", e);
     }
 
     return Status::OK();
@@ -358,7 +360,7 @@ Status DBMetaImpl::AllTables(std::vector<TableSchema>& table_schema_array) {
         }
 
     } catch (std::exception &e) {
-        HandleException("Encounter exception when lookup all tables", e);
+        return HandleException("Encounter exception when lookup all tables", e);
     }
 
     return Status::OK();

cpp/src/license/GPUInfoFile.h

@@ -1,69 +1,69 @@
-/*******************************************************************************
- * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved
- * Unauthorized copying of this file, via any medium is strictly prohibited.
- * Proprietary and confidential.
- ******************************************************************************/
-#pragma once
-
-#include <boost/serialization/access.hpp>
-#include <string>
-#include <map>
-
-
-class GPUInfoFile {
- public:
-    GPUInfoFile() = default;
-
-    GPUInfoFile(const int &device_count, const std::map<int, std::string> &uuid_encryption_map)
-        : device_count_(device_count), uuid_encryption_map_(uuid_encryption_map) {}
-
-    int get_device_count() {
-        return device_count_;
-    }
-    std::map<int, std::string> &get_uuid_encryption_map() {
-        return uuid_encryption_map_;
-    }
-
-
- public:
-    friend class boost::serialization::access;
-
-    template<typename Archive>
-    void serialize(Archive &ar, const unsigned int version) {
-        ar & device_count_;
-        ar & uuid_encryption_map_;
-    }
-
- public:
-    int device_count_ = 0;
-    std::map<int, std::string> uuid_encryption_map_;
-};
-
-class SerializedGPUInfoFile {
- public:
-    ~SerializedGPUInfoFile() {
-        if (gpu_info_file_ != nullptr) {
-            delete (gpu_info_file_);
-            gpu_info_file_ = nullptr;
-        }
-    }
-
-    void
-    set_gpu_info_file(GPUInfoFile *gpu_info_file) {
-        gpu_info_file_ = gpu_info_file;
-    }
-
-    GPUInfoFile *get_gpu_info_file() {
-        return gpu_info_file_;
-    }
- private:
-    friend class boost::serialization::access;
-
-    template<typename Archive>
-    void serialize(Archive &ar, const unsigned int version) {
-        ar & gpu_info_file_;
-    }
-
- private:
-    GPUInfoFile *gpu_info_file_ = nullptr;
-};
+///*******************************************************************************
+// * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved
+// * Unauthorized copying of this file, via any medium is strictly prohibited.
+// * Proprietary and confidential.
+// ******************************************************************************/
+//#pragma once
+//
+//#include <boost/serialization/access.hpp>
+//#include <string>
+//#include <map>
+//
+//
+//class GPUInfoFile {
+// public:
+//    GPUInfoFile() = default;
+//
+//    GPUInfoFile(const int &device_count, const std::map<int, std::string> &uuid_encryption_map)
+//        : device_count_(device_count), uuid_encryption_map_(uuid_encryption_map) {}
+//
+//    int get_device_count() {
+//        return device_count_;
+//    }
+//    std::map<int, std::string> &get_uuid_encryption_map() {
+//        return uuid_encryption_map_;
+//    }
+//
+//
+// public:
+//    friend class boost::serialization::access;
+//
+//    template<typename Archive>
+//    void serialize(Archive &ar, const unsigned int version) {
+//        ar & device_count_;
+//        ar & uuid_encryption_map_;
+//    }
+//
+// public:
+//    int device_count_ = 0;
+//    std::map<int, std::string> uuid_encryption_map_;
+//};
+//
+//class SerializedGPUInfoFile {
+// public:
+//    ~SerializedGPUInfoFile() {
+//        if (gpu_info_file_ != nullptr) {
+//            delete (gpu_info_file_);
+//            gpu_info_file_ = nullptr;
+//        }
+//    }
+//
+//    void
+//    set_gpu_info_file(GPUInfoFile *gpu_info_file) {
+//        gpu_info_file_ = gpu_info_file;
+//    }
+//
+//    GPUInfoFile *get_gpu_info_file() {
+//        return gpu_info_file_;
+//    }
+// private:
+//    friend class boost::serialization::access;
+//
+//    template<typename Archive>
+//    void serialize(Archive &ar, const unsigned int version) {
+//        ar & gpu_info_file_;
+//    }
+//
+// private:
+//    GPUInfoFile *gpu_info_file_ = nullptr;
+//};

cpp/src/license/GetSysInfo.cpp

@@ -1,83 +1,83 @@
-
-#include "utils/Log.h"
-#include "LicenseLibrary.h"
-#include "utils/Error.h"
-
-#include <iostream>
-#include <getopt.h>
-#include <memory.h>
-// Not provide path: current work path will be used and system.info.
-using namespace zilliz::milvus;
-
-void
-print_usage(const std::string &app_name) {
-    printf("\n Usage: %s [OPTIONS]\n\n", app_name.c_str());
-    printf("  Options:\n");
-    printf("   -h --help               Print this help\n");
-    printf("   -s --sysinfo filename   Generate system info file as given name\n");
-    printf("\n");
-}
-
-int main(int argc, char *argv[]) {
-    std::string app_name = argv[0];
-    if (argc != 1 && argc != 3) {
-        print_usage(app_name);
-        return EXIT_FAILURE;
-    }
-
-    static struct option long_options[] = {{"system_info", required_argument, 0, 's'},
-                                           {"help", no_argument, 0, 'h'},
-                                           {NULL, 0, 0, 0}};
-    int value = 0;
-    int option_index = 0;
-    std::string system_info_filename = "./system.info";
-    while ((value = getopt_long(argc, argv, "s:h", long_options, &option_index)) != -1) {
-        switch (value) {
-            case 's': {
-                char *system_info_filename_ptr = strdup(optarg);
-                system_info_filename = system_info_filename_ptr;
-                free(system_info_filename_ptr);
-//                printf("Generate system info file: %s\n", system_info_filename.c_str());
-                break;
-            }
-            case 'h':print_usage(app_name);
-                return EXIT_SUCCESS;
-            case '?':print_usage(app_name);
-                return EXIT_FAILURE;
-            default:print_usage(app_name);
-                break;
-        }
-    }
-
-    int device_count = 0;
-    server::ServerError err = server::LicenseLibrary::GetDeviceCount(device_count);
-    if (err != server::SERVER_SUCCESS) return -1;
-
-    // 1. Get All GPU UUID
-    std::vector<std::string> uuid_array;
-    err = server::LicenseLibrary::GetUUID(device_count, uuid_array);
-    if (err != server::SERVER_SUCCESS) return -1;
-
-    // 2. Get UUID SHA256
-    std::vector<std::string> uuid_sha256_array;
-    err = server::LicenseLibrary::GetUUIDSHA256(device_count, uuid_array, uuid_sha256_array);
-    if (err != server::SERVER_SUCCESS) return -1;
-
-    // 3. Generate GPU ID map with GPU UUID
-    std::map<int, std::string> uuid_encrption_map;
-    for (int i = 0; i < device_count; ++i) {
-        uuid_encrption_map[i] = uuid_sha256_array[i];
-    }
-
-
-    // 4. Generate GPU_info File
-    err = server::LicenseLibrary::GPUinfoFileSerialization(system_info_filename,
-                                                           device_count,
-                                                           uuid_encrption_map);
-    if (err != server::SERVER_SUCCESS) return -1;
-
-    printf("Generate GPU_info File Success\n");
-
-
-    return 0;
-}
+//
+//#include "utils/Log.h"
+//#include "LicenseLibrary.h"
+//#include "utils/Error.h"
+//
+//#include <iostream>
+//#include <getopt.h>
+//#include <memory.h>
+//// Not provide path: current work path will be used and system.info.
+//using namespace zilliz::milvus;
+//
+//void
+//print_usage(const std::string &app_name) {
+//    printf("\n Usage: %s [OPTIONS]\n\n", app_name.c_str());
+//    printf("  Options:\n");
+//    printf("   -h --help               Print this help\n");
+//    printf("   -s --sysinfo filename   Generate system info file as given name\n");
+//    printf("\n");
+//}
+//
+//int main(int argc, char *argv[]) {
+//    std::string app_name = argv[0];
+//    if (argc != 1 && argc != 3) {
+//        print_usage(app_name);
+//        return EXIT_FAILURE;
+//    }
+//
+//    static struct option long_options[] = {{"system_info", required_argument, 0, 's'},
+//                                           {"help", no_argument, 0, 'h'},
+//                                           {NULL, 0, 0, 0}};
+//    int value = 0;
+//    int option_index = 0;
+//    std::string system_info_filename = "./system.info";
+//    while ((value = getopt_long(argc, argv, "s:h", long_options, &option_index)) != -1) {
+//        switch (value) {
+//            case 's': {
+//                char *system_info_filename_ptr = strdup(optarg);
+//                system_info_filename = system_info_filename_ptr;
+//                free(system_info_filename_ptr);
+////                printf("Generate system info file: %s\n", system_info_filename.c_str());
+//                break;
+//            }
+//            case 'h':print_usage(app_name);
+//                return EXIT_SUCCESS;
+//            case '?':print_usage(app_name);
+//                return EXIT_FAILURE;
+//            default:print_usage(app_name);
+//                break;
+//        }
+//    }
+//
+//    int device_count = 0;
+//    server::ServerError err = server::LicenseLibrary::GetDeviceCount(device_count);
+//    if (err != server::SERVER_SUCCESS) return -1;
+//
+//    // 1. Get All GPU UUID
+//    std::vector<std::string> uuid_array;
+//    err = server::LicenseLibrary::GetUUID(device_count, uuid_array);
+//    if (err != server::SERVER_SUCCESS) return -1;
+//
+//    // 2. Get UUID SHA256
+//    std::vector<std::string> uuid_sha256_array;
+//    err = server::LicenseLibrary::GetUUIDSHA256(device_count, uuid_array, uuid_sha256_array);
+//    if (err != server::SERVER_SUCCESS) return -1;
+//
+//    // 3. Generate GPU ID map with GPU UUID
+//    std::map<int, std::string> uuid_encrption_map;
+//    for (int i = 0; i < device_count; ++i) {
+//        uuid_encrption_map[i] = uuid_sha256_array[i];
+//    }
+//
+//
+//    // 4. Generate GPU_info File
+//    err = server::LicenseLibrary::GPUinfoFileSerialization(system_info_filename,
+//                                                           device_count,
+//                                                           uuid_encrption_map);
+//    if (err != server::SERVER_SUCCESS) return -1;
+//
+//    printf("Generate GPU_info File Success\n");
+//
+//
+//    return 0;
+//}

cpp/src/license/LicenseCheck.cpp

@@ -1,131 +1,131 @@
-#include "LicenseCheck.h"
-#include <iostream>
-#include <thread>
-
-#include <boost/archive/binary_oarchive.hpp>
-#include <boost/archive/binary_iarchive.hpp>
-//#include <boost/foreach.hpp>
-//#include <boost/serialization/vector.hpp>
-#include <boost/filesystem/path.hpp>
-#include <boost/serialization/map.hpp>
-#include <boost/filesystem/operations.hpp>
-#include <boost/thread.hpp>
-#include <boost/date_time/posix_time/posix_time.hpp>
-
-
-namespace zilliz {
-namespace milvus {
-namespace server {
-
-LicenseCheck::LicenseCheck() {
-
-}
-
-LicenseCheck::~LicenseCheck() {
-    StopCountingDown();
-}
-
-ServerError
-LicenseCheck::LegalityCheck(const std::string &license_file_path) {
-
-    int device_count;
-    LicenseLibrary::GetDeviceCount(device_count);
-    std::vector<std::string> uuid_array;
-    LicenseLibrary::GetUUID(device_count, uuid_array);
-
-    std::vector<std::string> sha_array;
-    LicenseLibrary::GetUUIDSHA256(device_count, uuid_array, sha_array);
-
-    int output_device_count;
-    std::map<int, std::string> uuid_encryption_map;
-    time_t starting_time;
-    time_t end_time;
-    ServerError err = LicenseLibrary::LicenseFileDeserialization(license_file_path,
-                                               output_device_count,
-                                               uuid_encryption_map,
-                                               starting_time,
-                                               end_time);
-    if(err !=SERVER_SUCCESS)
-    {
-        std::cout << "License check error: 01" << std::endl;
-        return SERVER_UNEXPECTED_ERROR;
-    }
-    time_t system_time;
-    LicenseLibrary::GetSystemTime(system_time);
-
-    if (device_count != output_device_count) {
-        std::cout << "License check error: 02" << std::endl;
-        return SERVER_UNEXPECTED_ERROR;
-    }
-    for (int i = 0; i < device_count; ++i) {
-        if (sha_array[i] != uuid_encryption_map[i]) {
-            std::cout << "License check error: 03" << std::endl;
-            return SERVER_UNEXPECTED_ERROR;
-        }
-    }
-    if (system_time < starting_time || system_time > end_time) {
-        std::cout << "License check error: 04" << std::endl;
-        return SERVER_UNEXPECTED_ERROR;
-    }
-    std::cout << "Legality Check Success" << std::endl;
-    return SERVER_SUCCESS;
-}
-
-// Part 2: Timing check license
-
-ServerError
-LicenseCheck::AlterFile(const std::string &license_file_path,
-                        const boost::system::error_code &ec,
-                        boost::asio::deadline_timer *pt) {
-
-    ServerError err = LicenseCheck::LegalityCheck(license_file_path);
-    if(err!=SERVER_SUCCESS) {
-        std::cout << "license file check error" << std::endl;
-        exit(1);
-    }
-
-    std::cout << "---runing---" << std::endl;
-    pt->expires_at(pt->expires_at() + boost::posix_time::hours(1));
-    pt->async_wait(boost::bind(LicenseCheck::AlterFile, license_file_path, boost::asio::placeholders::error, pt));
-
-    return SERVER_SUCCESS;
-
-}
-
-ServerError
-LicenseCheck::StartCountingDown(const std::string &license_file_path) {
-
-    if (!LicenseLibrary::IsFileExistent(license_file_path)) {
-        std::cout << "license file not exist" << std::endl;
-        exit(1);
-    }
-
-    //create a thread to run AlterFile
-    if(counting_thread_ == nullptr) {
-        counting_thread_ = std::make_shared<std::thread>([&]() {
-            boost::asio::deadline_timer t(io_service_, boost::posix_time::hours(1));
-            t.async_wait(boost::bind(LicenseCheck::AlterFile, license_file_path, boost::asio::placeholders::error, &t));
-            io_service_.run();//this thread will block here
-        });
-    }
-
-    return SERVER_SUCCESS;
-}
-
-ServerError
-LicenseCheck::StopCountingDown() {
-    if(!io_service_.stopped()) {
-        io_service_.stop();
-    }
-
-    if(counting_thread_ != nullptr) {
-        counting_thread_->join();
-        counting_thread_ = nullptr;
-    }
-
-    return SERVER_SUCCESS;
-}
-
-}
-}
-}
+//#include "LicenseCheck.h"
+//#include <iostream>
+//#include <thread>
+//
+//#include <boost/archive/binary_oarchive.hpp>
+//#include <boost/archive/binary_iarchive.hpp>
+////#include <boost/foreach.hpp>
+////#include <boost/serialization/vector.hpp>
+//#include <boost/filesystem/path.hpp>
+//#include <boost/serialization/map.hpp>
+//#include <boost/filesystem/operations.hpp>
+//#include <boost/thread.hpp>
+//#include <boost/date_time/posix_time/posix_time.hpp>
+//
+//
+//namespace zilliz {
+//namespace milvus {
+//namespace server {
+//
+//LicenseCheck::LicenseCheck() {
+//
+//}
+//
+//LicenseCheck::~LicenseCheck() {
+//    StopCountingDown();
+//}
+//
+//ServerError
+//LicenseCheck::LegalityCheck(const std::string &license_file_path) {
+//
+//    int device_count;
+//    LicenseLibrary::GetDeviceCount(device_count);
+//    std::vector<std::string> uuid_array;
+//    LicenseLibrary::GetUUID(device_count, uuid_array);
+//
+//    std::vector<std::string> sha_array;
+//    LicenseLibrary::GetUUIDSHA256(device_count, uuid_array, sha_array);
+//
+//    int output_device_count;
+//    std::map<int, std::string> uuid_encryption_map;
+//    time_t starting_time;
+//    time_t end_time;
+//    ServerError err = LicenseLibrary::LicenseFileDeserialization(license_file_path,
+//                                               output_device_count,
+//                                               uuid_encryption_map,
+//                                               starting_time,
+//                                               end_time);
+//    if(err !=SERVER_SUCCESS)
+//    {
+//        std::cout << "License check error: 01" << std::endl;
+//        return SERVER_UNEXPECTED_ERROR;
+//    }
+//    time_t system_time;
+//    LicenseLibrary::GetSystemTime(system_time);
+//
+//    if (device_count != output_device_count) {
+//        std::cout << "License check error: 02" << std::endl;
+//        return SERVER_UNEXPECTED_ERROR;
+//    }
+//    for (int i = 0; i < device_count; ++i) {
+//        if (sha_array[i] != uuid_encryption_map[i]) {
+//            std::cout << "License check error: 03" << std::endl;
+//            return SERVER_UNEXPECTED_ERROR;
+//        }
+//    }
+//    if (system_time < starting_time || system_time > end_time) {
+//        std::cout << "License check error: 04" << std::endl;
+//        return SERVER_UNEXPECTED_ERROR;
+//    }
+//    std::cout << "Legality Check Success" << std::endl;
+//    return SERVER_SUCCESS;
+//}
+//
+//// Part 2: Timing check license
+//
+//ServerError
+//LicenseCheck::AlterFile(const std::string &license_file_path,
+//                        const boost::system::error_code &ec,
+//                        boost::asio::deadline_timer *pt) {
+//
+//    ServerError err = LicenseCheck::LegalityCheck(license_file_path);
+//    if(err!=SERVER_SUCCESS) {
+//        std::cout << "license file check error" << std::endl;
+//        exit(1);
+//    }
+//
+//    std::cout << "---runing---" << std::endl;
+//    pt->expires_at(pt->expires_at() + boost::posix_time::hours(1));
+//    pt->async_wait(boost::bind(LicenseCheck::AlterFile, license_file_path, boost::asio::placeholders::error, pt));
+//
+//    return SERVER_SUCCESS;
+//
+//}
+//
+//ServerError
+//LicenseCheck::StartCountingDown(const std::string &license_file_path) {
+//
+//    if (!LicenseLibrary::IsFileExistent(license_file_path)) {
+//        std::cout << "license file not exist" << std::endl;
+//        exit(1);
+//    }
+//
+//    //create a thread to run AlterFile
+//    if(counting_thread_ == nullptr) {
+//        counting_thread_ = std::make_shared<std::thread>([&]() {
+//            boost::asio::deadline_timer t(io_service_, boost::posix_time::hours(1));
+//            t.async_wait(boost::bind(LicenseCheck::AlterFile, license_file_path, boost::asio::placeholders::error, &t));
+//            io_service_.run();//this thread will block here
+//        });
+//    }
+//
+//    return SERVER_SUCCESS;
+//}
+//
+//ServerError
+//LicenseCheck::StopCountingDown() {
+//    if(!io_service_.stopped()) {
+//        io_service_.stop();
+//    }
+//
+//    if(counting_thread_ != nullptr) {
+//        counting_thread_->join();
+//        counting_thread_ = nullptr;
+//    }
+//
+//    return SERVER_SUCCESS;
+//}
+//
+//}
+//}
+//}

cpp/src/license/LicenseCheck.h

@@ -1,52 +1,52 @@
-#pragma once
-
-#include "utils/Error.h"
-#include "LicenseLibrary.h"
-
-#include <boost/asio.hpp>
-
-#include <thread>
-#include <memory>
-
-namespace zilliz {
-namespace milvus {
-namespace server {
-
-class LicenseCheck {
-private:
-    LicenseCheck();
-    ~LicenseCheck();
-
-public:
-    static LicenseCheck &
-    GetInstance() {
-        static LicenseCheck instance;
-        return instance;
-    };
-
-    static ServerError
-    LegalityCheck(const std::string &license_file_path);
-
-    ServerError
-    StartCountingDown(const std::string &license_file_path);
-
-    ServerError
-    StopCountingDown();
-
-private:
-    static ServerError
-    AlterFile(const std::string &license_file_path,
-              const boost::system::error_code &ec,
-              boost::asio::deadline_timer *pt);
-
-private:
-    boost::asio::io_service io_service_;
-    std::shared_ptr<std::thread> counting_thread_;
-
-};
-
-}
-}
-}
-
-
+//#pragma once
+//
+//#include "utils/Error.h"
+//#include "LicenseLibrary.h"
+//
+//#include <boost/asio.hpp>
+//
+//#include <thread>
+//#include <memory>
+//
+//namespace zilliz {
+//namespace milvus {
+//namespace server {
+//
+//class LicenseCheck {
+//private:
+//    LicenseCheck();
+//    ~LicenseCheck();
+//
+//public:
+//    static LicenseCheck &
+//    GetInstance() {
+//        static LicenseCheck instance;
+//        return instance;
+//    };
+//
+//    static ServerError
+//    LegalityCheck(const std::string &license_file_path);
+//
+//    ServerError
+//    StartCountingDown(const std::string &license_file_path);
+//
+//    ServerError
+//    StopCountingDown();
+//
+//private:
+//    static ServerError
+//    AlterFile(const std::string &license_file_path,
+//              const boost::system::error_code &ec,
+//              boost::asio::deadline_timer *pt);
+//
+//private:
+//    boost::asio::io_service io_service_;
+//    std::shared_ptr<std::thread> counting_thread_;
+//
+//};
+//
+//}
+//}
+//}
+//
+//

cpp/src/license/LicenseFile.h

@@ -1,86 +1,86 @@
-/*******************************************************************************
- * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved
- * Unauthorized copying of this file, via any medium is strictly prohibited.
- * Proprietary and confidential.
- ******************************************************************************/
-#pragma once
-
-
-#include <boost/serialization/access.hpp>
-#include <string>
-#include <map>
-
-
-class LicenseFile {
- public:
-    LicenseFile() = default;
-
-    LicenseFile(const int &device_count,
-                const std::map<int, std::string> &uuid_encryption_map,
-                const time_t &starting_time,
-                const time_t &end_time)
-        : device_count_(device_count),
-          uuid_encryption_map_(uuid_encryption_map),
-          starting_time_(starting_time),
-          end_time_(end_time) {}
-
-    int get_device_count() {
-        return device_count_;
-    }
-    std::map<int, std::string> &get_uuid_encryption_map() {
-        return uuid_encryption_map_;
-    }
-    time_t get_starting_time() {
-        return starting_time_;
-    }
-    time_t get_end_time() {
-        return end_time_;
-    }
-
- public:
-    friend class boost::serialization::access;
-
-    template<typename Archive>
-    void serialize(Archive &ar, const unsigned int version) {
-        ar & device_count_;
-        ar & uuid_encryption_map_;
-        ar & starting_time_;
-        ar & end_time_;
-    }
-
- public:
-    int device_count_ = 0;
-    std::map<int, std::string> uuid_encryption_map_;
-    time_t starting_time_ = 0;
-    time_t end_time_ = 0;
-};
-
-class SerializedLicenseFile {
- public:
-    ~SerializedLicenseFile() {
-        if (license_file_ != nullptr) {
-            delete (license_file_);
-            license_file_ = nullptr;
-        }
-    }
-
-    void
-    set_license_file(LicenseFile *license_file) {
-        license_file_ = license_file;
-    }
-
-    LicenseFile *get_license_file() {
-        return license_file_;
-    }
- private:
-    friend class boost::serialization::access;
-
-    template<typename Archive>
-    void serialize(Archive &ar, const unsigned int version) {
-        ar & license_file_;
-    }
-
- private:
-    LicenseFile *license_file_ = nullptr;
-};
-
+///*******************************************************************************
+// * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved
+// * Unauthorized copying of this file, via any medium is strictly prohibited.
+// * Proprietary and confidential.
+// ******************************************************************************/
+//#pragma once
+//
+//
+//#include <boost/serialization/access.hpp>
+//#include <string>
+//#include <map>
+//
+//
+//class LicenseFile {
+// public:
+//    LicenseFile() = default;
+//
+//    LicenseFile(const int &device_count,
+//                const std::map<int, std::string> &uuid_encryption_map,
+//                const time_t &starting_time,
+//                const time_t &end_time)
+//        : device_count_(device_count),
+//          uuid_encryption_map_(uuid_encryption_map),
+//          starting_time_(starting_time),
+//          end_time_(end_time) {}
+//
+//    int get_device_count() {
+//        return device_count_;
+//    }
+//    std::map<int, std::string> &get_uuid_encryption_map() {
+//        return uuid_encryption_map_;
+//    }
+//    time_t get_starting_time() {
+//        return starting_time_;
+//    }
+//    time_t get_end_time() {
+//        return end_time_;
+//    }
+//
+// public:
+//    friend class boost::serialization::access;
+//
+//    template<typename Archive>
+//    void serialize(Archive &ar, const unsigned int version) {
+//        ar & device_count_;
+//        ar & uuid_encryption_map_;
+//        ar & starting_time_;
+//        ar & end_time_;
+//    }
+//
+// public:
+//    int device_count_ = 0;
+//    std::map<int, std::string> uuid_encryption_map_;
+//    time_t starting_time_ = 0;
+//    time_t end_time_ = 0;
+//};
+//
+//class SerializedLicenseFile {
+// public:
+//    ~SerializedLicenseFile() {
+//        if (license_file_ != nullptr) {
+//            delete (license_file_);
+//            license_file_ = nullptr;
+//        }
+//    }
+//
+//    void
+//    set_license_file(LicenseFile *license_file) {
+//        license_file_ = license_file;
+//    }
+//
+//    LicenseFile *get_license_file() {
+//        return license_file_;
+//    }
+// private:
+//    friend class boost::serialization::access;
+//
+//    template<typename Archive>
+//    void serialize(Archive &ar, const unsigned int version) {
+//        ar & license_file_;
+//    }
+//
+// private:
+//    LicenseFile *license_file_ = nullptr;
+//};
+//

cpp/src/license/LicenseGenerator.cpp

@@ -1,121 +1,121 @@
-
-#include <iostream>
-#include <getopt.h>
-#include <memory.h>
-
-#include "utils/Log.h"
-#include "license/LicenseLibrary.h"
-#include "utils/Error.h"
-
-
-using namespace zilliz::milvus;
-// Not provide path: current work path will be used and system.info.
-
-void
-print_usage(const std::string &app_name) {
-    printf("\n Usage: %s [OPTIONS]\n\n", app_name.c_str());
-    printf("  Options:\n");
-    printf("   -h --help               Print this help\n");
-    printf("   -s --sysinfo filename   sysinfo file location\n");
-    printf("   -l --license filename   Generate license file as given name\n");
-    printf("   -b --starting time      Set start time (format: YYYY-MM-DD)\n");
-    printf("   -e --end time           Set end time (format: YYYY-MM-DD)\n");
-    printf("\n");
-}
-
-int main(int argc, char *argv[]) {
-    std::string app_name = argv[0];
-//    if (argc != 1 && argc != 3) {
-//        print_usage(app_name);
-//        return EXIT_FAILURE;
+//
+//#include <iostream>
+//#include <getopt.h>
+//#include <memory.h>
+//
+//#include "utils/Log.h"
+//#include "license/LicenseLibrary.h"
+//#include "utils/Error.h"
+//
+//
+//using namespace zilliz::milvus;
+//// Not provide path: current work path will be used and system.info.
+//
+//void
+//print_usage(const std::string &app_name) {
+//    printf("\n Usage: %s [OPTIONS]\n\n", app_name.c_str());
+//    printf("  Options:\n");
+//    printf("   -h --help               Print this help\n");
+//    printf("   -s --sysinfo filename   sysinfo file location\n");
+//    printf("   -l --license filename   Generate license file as given name\n");
+//    printf("   -b --starting time      Set start time (format: YYYY-MM-DD)\n");
+//    printf("   -e --end time           Set end time (format: YYYY-MM-DD)\n");
+//    printf("\n");
+//}
+//
+//int main(int argc, char *argv[]) {
+//    std::string app_name = argv[0];
+////    if (argc != 1 && argc != 3) {
+////        print_usage(app_name);
+////        return EXIT_FAILURE;
+////    }
+//    static struct option long_options[] = {{"system_info", required_argument, 0, 's'},
+//                                           {"license", optional_argument, 0, 'l'},
+//                                           {"help", no_argument, 0, 'h'},
+//                                           {"starting_time", required_argument, 0, 'b'},
+//                                           {"end_time", required_argument, 0, 'e'},
+//                                           {NULL, 0, 0, 0}};
+//    server::ServerError err;
+//    int value = 0;
+//    int option_index = 0;
+//    std::string system_info_filename = "./system.info";
+//    std::string license_filename = "./system.license";
+//    char *string_starting_time = NULL;
+//    char *string_end_time = NULL;
+//    time_t starting_time = 0;
+//    time_t end_time = 0;
+//    int flag_s = 1;
+//    int flag_b = 1;
+//    int flag_e = 1;
+//    while ((value = getopt_long(argc, argv, "hl:s:b:e:", long_options, NULL)) != -1) {
+//        switch (value) {
+//            case 's': {
+//                flag_s = 0;
+//                system_info_filename = (std::string) (optarg);
+//                break;
+//            }
+//            case 'b': {
+//                flag_b = 0;
+//                string_starting_time = optarg;
+//                break;
+//            }
+//            case 'e': {
+//                flag_e = 0;
+//                string_end_time = optarg;
+//                break;
+//            }
+//            case 'l': {
+//                license_filename = (std::string) (optarg);
+//                break;
+//            }
+//            case 'h':print_usage(app_name);
+//                return EXIT_SUCCESS;
+//            case '?':print_usage(app_name);
+//                return EXIT_FAILURE;
+//            default:print_usage(app_name);
+//                break;
+//        }
+//
 //    }
-    static struct option long_options[] = {{"system_info", required_argument, 0, 's'},
-                                           {"license", optional_argument, 0, 'l'},
-                                           {"help", no_argument, 0, 'h'},
-                                           {"starting_time", required_argument, 0, 'b'},
-                                           {"end_time", required_argument, 0, 'e'},
-                                           {NULL, 0, 0, 0}};
-    server::ServerError err;
-    int value = 0;
-    int option_index = 0;
-    std::string system_info_filename = "./system.info";
-    std::string license_filename = "./system.license";
-    char *string_starting_time = NULL;
-    char *string_end_time = NULL;
-    time_t starting_time = 0;
-    time_t end_time = 0;
-    int flag_s = 1;
-    int flag_b = 1;
-    int flag_e = 1;
-    while ((value = getopt_long(argc, argv, "hl:s:b:e:", long_options, NULL)) != -1) {
-        switch (value) {
-            case 's': {
-                flag_s = 0;
-                system_info_filename = (std::string) (optarg);
-                break;
-            }
-            case 'b': {
-                flag_b = 0;
-                string_starting_time = optarg;
-                break;
-            }
-            case 'e': {
-                flag_e = 0;
-                string_end_time = optarg;
-                break;
-            }
-            case 'l': {
-                license_filename = (std::string) (optarg);
-                break;
-            }
-            case 'h':print_usage(app_name);
-                return EXIT_SUCCESS;
-            case '?':print_usage(app_name);
-                return EXIT_FAILURE;
-            default:print_usage(app_name);
-                break;
-        }
-
-    }
-    if (flag_s) {
-        printf("Error: sysinfo file location must be entered\n");
-        return 1;
-    }
-    if (flag_b) {
-        printf("Error: start time must be entered\n");
-        return 1;
-    }
-    if (flag_e) {
-        printf("Error: end time must be entered\n");
-        return 1;
-    }
-
-    err = server::LicenseLibrary::GetDateTime(string_starting_time, starting_time);
-    if (err != server::SERVER_SUCCESS) return -1;
-
-    err = server::LicenseLibrary::GetDateTime(string_end_time, end_time);
-    if (err != server::SERVER_SUCCESS) return -1;
-
-
-    int output_info_device_count = 0;
-    std::map<int, std::string> output_info_uuid_encrption_map;
-
-
-    err = server::LicenseLibrary::GPUinfoFileDeserialization(system_info_filename,
-                                                             output_info_device_count,
-                                                             output_info_uuid_encrption_map);
-    if (err != server::SERVER_SUCCESS) return -1;
-
-
-    err = server::LicenseLibrary::LicenseFileSerialization(license_filename,
-                                                           output_info_device_count,
-                                                           output_info_uuid_encrption_map,
-                                                           starting_time,
-                                                           end_time);
-    if (err != server::SERVER_SUCCESS) return -1;
-
-
-    printf("Generate License File Success\n");
-
-    return 0;
-}
+//    if (flag_s) {
+//        printf("Error: sysinfo file location must be entered\n");
+//        return 1;
+//    }
+//    if (flag_b) {
+//        printf("Error: start time must be entered\n");
+//        return 1;
+//    }
+//    if (flag_e) {
+//        printf("Error: end time must be entered\n");
+//        return 1;
+//    }
+//
+//    err = server::LicenseLibrary::GetDateTime(string_starting_time, starting_time);
+//    if (err != server::SERVER_SUCCESS) return -1;
+//
+//    err = server::LicenseLibrary::GetDateTime(string_end_time, end_time);
+//    if (err != server::SERVER_SUCCESS) return -1;
+//
+//
+//    int output_info_device_count = 0;
+//    std::map<int, std::string> output_info_uuid_encrption_map;
+//
+//
+//    err = server::LicenseLibrary::GPUinfoFileDeserialization(system_info_filename,
+//                                                             output_info_device_count,
+//                                                             output_info_uuid_encrption_map);
+//    if (err != server::SERVER_SUCCESS) return -1;
+//
+//
+//    err = server::LicenseLibrary::LicenseFileSerialization(license_filename,
+//                                                           output_info_device_count,
+//                                                           output_info_uuid_encrption_map,
+//                                                           starting_time,
+//                                                           end_time);
+//    if (err != server::SERVER_SUCCESS) return -1;
+//
+//
+//    printf("Generate License File Success\n");
+//
+//    return 0;
+//}

cpp/src/license/LicenseLibrary.cpp

@@ -1,345 +1,345 @@
-////////////////////////////////////////////////////////////////////////////////
-// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved
-// Unauthorized copying of this file, via any medium is strictly prohibited.
-// Proprietary and confidential.
-////////////////////////////////////////////////////////////////////////////////
-
-#include "LicenseLibrary.h"
-#include "utils/Log.h"
-#include <cuda_runtime.h>
-#include <nvml.h>
-#include <openssl/md5.h>
-#include <openssl/sha.h>
-
-#include <boost/archive/binary_oarchive.hpp>
-#include <boost/archive/binary_iarchive.hpp>
-//#include <boost/foreach.hpp>
-//#include <boost/serialization/vector.hpp>
-#include <boost/filesystem/path.hpp>
-#include <boost/serialization/map.hpp>
-#include <boost/filesystem/operations.hpp>
-
-
-namespace zilliz {
-namespace milvus {
-namespace server {
-
-constexpr int LicenseLibrary::sha256_length_;
-
-// Part 0: File check
-bool
-LicenseLibrary::IsFileExistent(const std::string &path) {
-
-    boost::system::error_code error;
-    auto file_status = boost::filesystem::status(path, error);
-    if (error) {
-        return false;
-    }
-
-    if (!boost::filesystem::exists(file_status)) {
-        return false;
-    }
-
-    return !boost::filesystem::is_directory(file_status);
-}
-
-// Part 1: Get GPU Info
-ServerError
-LicenseLibrary::GetDeviceCount(int &device_count) {
-    nvmlReturn_t result = nvmlInit();
-    if (NVML_SUCCESS != result) {
-        printf("Failed to initialize NVML: %s\n", nvmlErrorString(result));
-        return SERVER_UNEXPECTED_ERROR;
-    }
-    cudaError_t error_id = cudaGetDeviceCount(&device_count);
-    if (error_id != cudaSuccess) {
-        printf("cudaGetDeviceCount returned %d\n-> %s\n", (int) error_id, cudaGetErrorString(error_id));
-        printf("Result = FAIL\n");
-        return SERVER_UNEXPECTED_ERROR;
-    }
-    return SERVER_SUCCESS;
-}
-
-ServerError
-LicenseLibrary::GetUUID(int device_count, std::vector<std::string> &uuid_array) {
-    if (device_count == 0) {
-        printf("There are no available device(s) that support CUDA\n");
-        return SERVER_UNEXPECTED_ERROR;
-    }
-
-    for (int dev = 0; dev < device_count; ++dev) {
-        nvmlDevice_t device;
-        nvmlReturn_t result = nvmlDeviceGetHandleByIndex(dev, &device);
-        if (NVML_SUCCESS != result) {
-            printf("Failed to get handle for device %i: %s\n", dev, nvmlErrorString(result));
-            return SERVER_UNEXPECTED_ERROR;
-        }
-
-        char uuid[80];
-        unsigned int length = 80;
-        nvmlReturn_t err = nvmlDeviceGetUUID(device, uuid, length);
-        if (err != NVML_SUCCESS) {
-            printf("nvmlDeviceGetUUID error: %d\n", err);
-            return SERVER_UNEXPECTED_ERROR;
-        }
-
-        uuid_array.emplace_back(uuid);
-    }
-    return SERVER_SUCCESS;
-}
-
-ServerError
-LicenseLibrary::GetUUIDMD5(int device_count,
-                           std::vector<std::string> &uuid_array,
-                           std::vector<std::string> &md5_array) {
-    MD5_CTX ctx;
-    unsigned char outmd[16];
-    char temp[2];
-    std::string md5;
-    for (int dev = 0; dev < device_count; ++dev) {
-        md5.clear();
-        memset(outmd, 0, sizeof(outmd));
-        MD5_Init(&ctx);
-        MD5_Update(&ctx, uuid_array[dev].c_str(), uuid_array[dev].size());
-        MD5_Final(outmd, &ctx);
-        for (int i = 0; i < 16; ++i) {
-            std::snprintf(temp, 2, "%02X", outmd[i]);
-            md5 += temp;
-        }
-        md5_array.push_back(md5);
-    }
-    return SERVER_SUCCESS;
-}
-
-ServerError
-LicenseLibrary::GetUUIDSHA256(const int &device_count,
-                              std::vector<std::string> &uuid_array,
-                              std::vector<std::string> &sha_array) {
-    SHA256_CTX ctx;
-    unsigned char outmd[sha256_length_];
-    char temp[2];
-    std::string sha;
-    for (int dev = 0; dev < device_count; ++dev) {
-        sha.clear();
-        memset(outmd, 0, sizeof(outmd));
-        SHA256_Init(&ctx);
-        SHA256_Update(&ctx, uuid_array[dev].c_str(), uuid_array[dev].size());
-        SHA256_Final(outmd, &ctx);
-        for (int i = 0; i < sha256_length_; ++i) {
-            std::snprintf(temp, 2, "%02X", outmd[i]);
-            sha += temp;
-        }
-        sha_array.push_back(sha);
-    }
-    return SERVER_SUCCESS;
-}
-
-ServerError
-LicenseLibrary::GetSystemTime(time_t &system_time) {
-    system_time = time(NULL);
-    return SERVER_SUCCESS;
-}
-
-// Part 2: Handle License File
-ServerError
-LicenseLibrary::LicenseFileSerialization(const std::string &path,
-                                         int device_count,
-                                         const std::map<int, std::string> &uuid_encrption_map,
-                                         time_t starting_time,
-                                         time_t end_time) {
-
-    std::ofstream file(path);
-    boost::archive::binary_oarchive oa(file);
-    oa.register_type<LicenseFile>();
-
-    SerializedLicenseFile serialized_license_file;
-
-    serialized_license_file.set_license_file(new LicenseFile(device_count,
-                                                             uuid_encrption_map,
-                                                             starting_time,
-                                                             end_time));
-    oa << serialized_license_file;
-
-    file.close();
-    return SERVER_SUCCESS;
-}
-
-ServerError
-LicenseLibrary::LicenseFileDeserialization(const std::string &path,
-                                           int &device_count,
-                                           std::map<int, std::string> &uuid_encrption_map,
-                                           time_t &starting_time,
-                                           time_t &end_time) {
-    if (!IsFileExistent(path)) return SERVER_LICENSE_FILE_NOT_EXIST;
-    std::ifstream file(path);
-    boost::archive::binary_iarchive ia(file);
-    ia.register_type<LicenseFile>();
-
-    SerializedLicenseFile serialized_license_file;
-    ia >> serialized_license_file;
-
-    device_count = serialized_license_file.get_license_file()->get_device_count();
-    uuid_encrption_map = serialized_license_file.get_license_file()->get_uuid_encryption_map();
-    starting_time = serialized_license_file.get_license_file()->get_starting_time();
-    end_time = serialized_license_file.get_license_file()->get_end_time();
-
-    file.close();
-    return SERVER_SUCCESS;
-}
-
+//////////////////////////////////////////////////////////////////////////////////
+//// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved
+//// Unauthorized copying of this file, via any medium is strictly prohibited.
+//// Proprietary and confidential.
+//////////////////////////////////////////////////////////////////////////////////
+//
+//#include "LicenseLibrary.h"
+//#include "utils/Log.h"
+//#include <cuda_runtime.h>
+//#include <nvml.h>
+//#include <openssl/md5.h>
+//#include <openssl/sha.h>
+//
+//#include <boost/archive/binary_oarchive.hpp>
+//#include <boost/archive/binary_iarchive.hpp>
+////#include <boost/foreach.hpp>
+////#include <boost/serialization/vector.hpp>
+//#include <boost/filesystem/path.hpp>
+//#include <boost/serialization/map.hpp>
+//#include <boost/filesystem/operations.hpp>
+//
+//
+//namespace zilliz {
+//namespace milvus {
+//namespace server {
+//
+//constexpr int LicenseLibrary::sha256_length_;
+//
+//// Part 0: File check
+//bool
+//LicenseLibrary::IsFileExistent(const std::string &path) {
+//
+//    boost::system::error_code error;
+//    auto file_status = boost::filesystem::status(path, error);
+//    if (error) {
+//        return false;
+//    }
+//
+//    if (!boost::filesystem::exists(file_status)) {
+//        return false;
+//    }
+//
+//    return !boost::filesystem::is_directory(file_status);
+//}
+//
+//// Part 1: Get GPU Info
 //ServerError
-//LicenseLibrary::SecretFileSerialization(const std::string &path,
-//                                        const time_t &update_time,
-//                                        const off_t &file_size,
-//                                        const time_t &starting_time,
-//                                        const time_t &end_time,
-//                                        const std::string &file_md5) {
+//LicenseLibrary::GetDeviceCount(int &device_count) {
+//    nvmlReturn_t result = nvmlInit();
+//    if (NVML_SUCCESS != result) {
+//        printf("Failed to initialize NVML: %s\n", nvmlErrorString(result));
+//        return SERVER_UNEXPECTED_ERROR;
+//    }
+//    cudaError_t error_id = cudaGetDeviceCount(&device_count);
+//    if (error_id != cudaSuccess) {
+//        printf("cudaGetDeviceCount returned %d\n-> %s\n", (int) error_id, cudaGetErrorString(error_id));
+//        printf("Result = FAIL\n");
+//        return SERVER_UNEXPECTED_ERROR;
+//    }
+//    return SERVER_SUCCESS;
+//}
+//
+//ServerError
+//LicenseLibrary::GetUUID(int device_count, std::vector<std::string> &uuid_array) {
+//    if (device_count == 0) {
+//        printf("There are no available device(s) that support CUDA\n");
+//        return SERVER_UNEXPECTED_ERROR;
+//    }
+//
+//    for (int dev = 0; dev < device_count; ++dev) {
+//        nvmlDevice_t device;
+//        nvmlReturn_t result = nvmlDeviceGetHandleByIndex(dev, &device);
+//        if (NVML_SUCCESS != result) {
+//            printf("Failed to get handle for device %i: %s\n", dev, nvmlErrorString(result));
+//            return SERVER_UNEXPECTED_ERROR;
+//        }
+//
+//        char uuid[80];
+//        unsigned int length = 80;
+//        nvmlReturn_t err = nvmlDeviceGetUUID(device, uuid, length);
+//        if (err != NVML_SUCCESS) {
+//            printf("nvmlDeviceGetUUID error: %d\n", err);
+//            return SERVER_UNEXPECTED_ERROR;
+//        }
+//
+//        uuid_array.emplace_back(uuid);
+//    }
+//    return SERVER_SUCCESS;
+//}
+//
+//ServerError
+//LicenseLibrary::GetUUIDMD5(int device_count,
+//                           std::vector<std::string> &uuid_array,
+//                           std::vector<std::string> &md5_array) {
+//    MD5_CTX ctx;
+//    unsigned char outmd[16];
+//    char temp[2];
+//    std::string md5;
+//    for (int dev = 0; dev < device_count; ++dev) {
+//        md5.clear();
+//        memset(outmd, 0, sizeof(outmd));
+//        MD5_Init(&ctx);
+//        MD5_Update(&ctx, uuid_array[dev].c_str(), uuid_array[dev].size());
+//        MD5_Final(outmd, &ctx);
+//        for (int i = 0; i < 16; ++i) {
+//            std::snprintf(temp, 2, "%02X", outmd[i]);
+//            md5 += temp;
+//        }
+//        md5_array.push_back(md5);
+//    }
+//    return SERVER_SUCCESS;
+//}
+//
+//ServerError
+//LicenseLibrary::GetUUIDSHA256(const int &device_count,
+//                              std::vector<std::string> &uuid_array,
+//                              std::vector<std::string> &sha_array) {
+//    SHA256_CTX ctx;
+//    unsigned char outmd[sha256_length_];
+//    char temp[2];
+//    std::string sha;
+//    for (int dev = 0; dev < device_count; ++dev) {
+//        sha.clear();
+//        memset(outmd, 0, sizeof(outmd));
+//        SHA256_Init(&ctx);
+//        SHA256_Update(&ctx, uuid_array[dev].c_str(), uuid_array[dev].size());
+//        SHA256_Final(outmd, &ctx);
+//        for (int i = 0; i < sha256_length_; ++i) {
+//            std::snprintf(temp, 2, "%02X", outmd[i]);
+//            sha += temp;
+//        }
+//        sha_array.push_back(sha);
+//    }
+//    return SERVER_SUCCESS;
+//}
+//
+//ServerError
+//LicenseLibrary::GetSystemTime(time_t &system_time) {
+//    system_time = time(NULL);
+//    return SERVER_SUCCESS;
+//}
+//
+//// Part 2: Handle License File
+//ServerError
+//LicenseLibrary::LicenseFileSerialization(const std::string &path,
+//                                         int device_count,
+//                                         const std::map<int, std::string> &uuid_encrption_map,
+//                                         time_t starting_time,
+//                                         time_t end_time) {
+//
 //    std::ofstream file(path);
 //    boost::archive::binary_oarchive oa(file);
-//    oa.register_type<SecretFile>();
+//    oa.register_type<LicenseFile>();
 //
-//    SerializedSecretFile serialized_secret_file;
+//    SerializedLicenseFile serialized_license_file;
 //
-//    serialized_secret_file.set_secret_file(new SecretFile(update_time, file_size, starting_time, end_time, file_md5));
-//    oa << serialized_secret_file;
+//    serialized_license_file.set_license_file(new LicenseFile(device_count,
+//                                                             uuid_encrption_map,
+//                                                             starting_time,
+//                                                             end_time));
+//    oa << serialized_license_file;
 //
 //    file.close();
 //    return SERVER_SUCCESS;
 //}
 //
 //ServerError
-//LicenseLibrary::SecretFileDeserialization(const std::string &path,
-//                                          time_t &update_time,
-//                                          off_t &file_size,
-//                                          time_t &starting_time,
-//                                          time_t &end_time,
-//                                          std::string &file_md5) {
+//LicenseLibrary::LicenseFileDeserialization(const std::string &path,
+//                                           int &device_count,
+//                                           std::map<int, std::string> &uuid_encrption_map,
+//                                           time_t &starting_time,
+//                                           time_t &end_time) {
+//    if (!IsFileExistent(path)) return SERVER_LICENSE_FILE_NOT_EXIST;
+//    std::ifstream file(path);
+//    boost::archive::binary_iarchive ia(file);
+//    ia.register_type<LicenseFile>();
+//
+//    SerializedLicenseFile serialized_license_file;
+//    ia >> serialized_license_file;
+//
+//    device_count = serialized_license_file.get_license_file()->get_device_count();
+//    uuid_encrption_map = serialized_license_file.get_license_file()->get_uuid_encryption_map();
+//    starting_time = serialized_license_file.get_license_file()->get_starting_time();
+//    end_time = serialized_license_file.get_license_file()->get_end_time();
+//
+//    file.close();
+//    return SERVER_SUCCESS;
+//}
+//
+////ServerError
+////LicenseLibrary::SecretFileSerialization(const std::string &path,
+////                                        const time_t &update_time,
+////                                        const off_t &file_size,
+////                                        const time_t &starting_time,
+////                                        const time_t &end_time,
+////                                        const std::string &file_md5) {
+////    std::ofstream file(path);
+////    boost::archive::binary_oarchive oa(file);
+////    oa.register_type<SecretFile>();
+////
+////    SerializedSecretFile serialized_secret_file;
+////
+////    serialized_secret_file.set_secret_file(new SecretFile(update_time, file_size, starting_time, end_time, file_md5));
+////    oa << serialized_secret_file;
+////
+////    file.close();
+////    return SERVER_SUCCESS;
+////}
+////
+////ServerError
+////LicenseLibrary::SecretFileDeserialization(const std::string &path,
+////                                          time_t &update_time,
+////                                          off_t &file_size,
+////                                          time_t &starting_time,
+////                                          time_t &end_time,
+////                                          std::string &file_md5) {
+////    if (!IsFileExistent(path)) return SERVER_LICENSE_FILE_NOT_EXIST;
+////
+////    std::ifstream file(path);
+////    boost::archive::binary_iarchive ia(file);
+////    ia.register_type<SecretFile>();
+////    SerializedSecretFile serialized_secret_file;
+////
+////    ia >> serialized_secret_file;
+////    update_time = serialized_secret_file.get_secret_file()->get_update_time();
+////    file_size = serialized_secret_file.get_secret_file()->get_file_size();
+////    starting_time = serialized_secret_file.get_secret_file()->get_starting_time();
+////    end_time = serialized_secret_file.get_secret_file()->get_end_time();
+////    file_md5 = serialized_secret_file.get_secret_file()->get_file_md5();
+////    file.close();
+////    return SERVER_SUCCESS;
+////}
+//
+//
+//
+//// Part 3: File attribute: UpdateTime Time/ Size/ MD5
+//ServerError
+//LicenseLibrary::GetFileUpdateTimeAndSize(const std::string &path, time_t &update_time, off_t &file_size) {
+//
+//    if (!IsFileExistent(path)) return SERVER_LICENSE_FILE_NOT_EXIST;
+//
+//    struct stat buf;
+//    int err_no = stat(path.c_str(), &buf);
+//    if (err_no != 0) {
+//        std::cout << strerror(err_no) << std::endl;
+//        return SERVER_UNEXPECTED_ERROR;
+//    }
+//
+//    update_time = buf.st_mtime;
+//    file_size = buf.st_size;
+//
+//    return SERVER_SUCCESS;
+//}
+//
+//ServerError
+//LicenseLibrary::GetFileMD5(const std::string &path, std::string &filemd5) {
+//
+//    if (!IsFileExistent(path)) return SERVER_LICENSE_FILE_NOT_EXIST;
+//
+//    filemd5.clear();
+//
+//    std::ifstream file(path.c_str(), std::ifstream::binary);
+//    if (!file) {
+//        return -1;
+//    }
+//
+//    MD5_CTX md5Context;
+//    MD5_Init(&md5Context);
+//
+//    char buf[1024 * 16];
+//    while (file.good()) {
+//        file.read(buf, sizeof(buf));
+//        MD5_Update(&md5Context, buf, file.gcount());
+//    }
+//
+//    unsigned char result[MD5_DIGEST_LENGTH];
+//    MD5_Final(result, &md5Context);
+//
+//    char hex[35];
+//    memset(hex, 0, sizeof(hex));
+//    for (int i = 0; i < MD5_DIGEST_LENGTH; ++i) {
+//        sprintf(hex + i * 2, "%02X", result[i]);
+//    }
+//    hex[32] = '\0';
+//    filemd5 = std::string(hex);
+//
+//    return SERVER_SUCCESS;
+//}
+//// Part 4: GPU Info File Serialization/Deserialization
+//ServerError
+//LicenseLibrary::GPUinfoFileSerialization(const std::string &path,
+//                                         int device_count,
+//                                         const std::map<int, std::string> &uuid_encrption_map) {
+//    std::ofstream file(path);
+//    boost::archive::binary_oarchive oa(file);
+//    oa.register_type<GPUInfoFile>();
+//
+//    SerializedGPUInfoFile serialized_gpu_info_file;
+//
+//    serialized_gpu_info_file.set_gpu_info_file(new GPUInfoFile(device_count, uuid_encrption_map));
+//    oa << serialized_gpu_info_file;
+//
+//    file.close();
+//    return SERVER_SUCCESS;
+//}
+//ServerError
+//LicenseLibrary::GPUinfoFileDeserialization(const std::string &path,
+//                                           int &device_count,
+//                                           std::map<int, std::string> &uuid_encrption_map) {
 //    if (!IsFileExistent(path)) return SERVER_LICENSE_FILE_NOT_EXIST;
 //
 //    std::ifstream file(path);
 //    boost::archive::binary_iarchive ia(file);
-//    ia.register_type<SecretFile>();
-//    SerializedSecretFile serialized_secret_file;
+//    ia.register_type<GPUInfoFile>();
+//
+//    SerializedGPUInfoFile serialized_gpu_info_file;
+//    ia >> serialized_gpu_info_file;
+//
+//    device_count = serialized_gpu_info_file.get_gpu_info_file()->get_device_count();
+//    uuid_encrption_map = serialized_gpu_info_file.get_gpu_info_file()->get_uuid_encryption_map();
 //
-//    ia >> serialized_secret_file;
-//    update_time = serialized_secret_file.get_secret_file()->get_update_time();
-//    file_size = serialized_secret_file.get_secret_file()->get_file_size();
-//    starting_time = serialized_secret_file.get_secret_file()->get_starting_time();
-//    end_time = serialized_secret_file.get_secret_file()->get_end_time();
-//    file_md5 = serialized_secret_file.get_secret_file()->get_file_md5();
 //    file.close();
 //    return SERVER_SUCCESS;
 //}
-
-
-
-// Part 3: File attribute: UpdateTime Time/ Size/ MD5
-ServerError
-LicenseLibrary::GetFileUpdateTimeAndSize(const std::string &path, time_t &update_time, off_t &file_size) {
-
-    if (!IsFileExistent(path)) return SERVER_LICENSE_FILE_NOT_EXIST;
-
-    struct stat buf;
-    int err_no = stat(path.c_str(), &buf);
-    if (err_no != 0) {
-        std::cout << strerror(err_no) << std::endl;
-        return SERVER_UNEXPECTED_ERROR;
-    }
-
-    update_time = buf.st_mtime;
-    file_size = buf.st_size;
-
-    return SERVER_SUCCESS;
-}
-
-ServerError
-LicenseLibrary::GetFileMD5(const std::string &path, std::string &filemd5) {
-
-    if (!IsFileExistent(path)) return SERVER_LICENSE_FILE_NOT_EXIST;
-
-    filemd5.clear();
-
-    std::ifstream file(path.c_str(), std::ifstream::binary);
-    if (!file) {
-        return -1;
-    }
-
-    MD5_CTX md5Context;
-    MD5_Init(&md5Context);
-
-    char buf[1024 * 16];
-    while (file.good()) {
-        file.read(buf, sizeof(buf));
-        MD5_Update(&md5Context, buf, file.gcount());
-    }
-
-    unsigned char result[MD5_DIGEST_LENGTH];
-    MD5_Final(result, &md5Context);
-
-    char hex[35];
-    memset(hex, 0, sizeof(hex));
-    for (int i = 0; i < MD5_DIGEST_LENGTH; ++i) {
-        sprintf(hex + i * 2, "%02X", result[i]);
-    }
-    hex[32] = '\0';
-    filemd5 = std::string(hex);
-
-    return SERVER_SUCCESS;
-}
-// Part 4: GPU Info File Serialization/Deserialization
-ServerError
-LicenseLibrary::GPUinfoFileSerialization(const std::string &path,
-                                         int device_count,
-                                         const std::map<int, std::string> &uuid_encrption_map) {
-    std::ofstream file(path);
-    boost::archive::binary_oarchive oa(file);
-    oa.register_type<GPUInfoFile>();
-
-    SerializedGPUInfoFile serialized_gpu_info_file;
-
-    serialized_gpu_info_file.set_gpu_info_file(new GPUInfoFile(device_count, uuid_encrption_map));
-    oa << serialized_gpu_info_file;
-
-    file.close();
-    return SERVER_SUCCESS;
-}
-ServerError
-LicenseLibrary::GPUinfoFileDeserialization(const std::string &path,
-                                           int &device_count,
-                                           std::map<int, std::string> &uuid_encrption_map) {
-    if (!IsFileExistent(path)) return SERVER_LICENSE_FILE_NOT_EXIST;
-
-    std::ifstream file(path);
-    boost::archive::binary_iarchive ia(file);
-    ia.register_type<GPUInfoFile>();
-
-    SerializedGPUInfoFile serialized_gpu_info_file;
-    ia >> serialized_gpu_info_file;
-
-    device_count = serialized_gpu_info_file.get_gpu_info_file()->get_device_count();
-    uuid_encrption_map = serialized_gpu_info_file.get_gpu_info_file()->get_uuid_encryption_map();
-
-    file.close();
-    return SERVER_SUCCESS;
-}
-
-ServerError
-LicenseLibrary::GetDateTime(const char *cha, time_t &data_time) {
-    tm tm_;
-    int year, month, day;
-    sscanf(cha, "%d-%d-%d", &year, &month, &day);
-    tm_.tm_year = year - 1900;
-    tm_.tm_mon = month - 1;
-    tm_.tm_mday = day;
-    tm_.tm_hour = 0;
-    tm_.tm_min = 0;
-    tm_.tm_sec = 0;
-    tm_.tm_isdst = 0;
-    data_time = mktime(&tm_);
-    return SERVER_SUCCESS;
-
-}
-
-}
-}
-}
+//
+//ServerError
+//LicenseLibrary::GetDateTime(const char *cha, time_t &data_time) {
+//    tm tm_;
+//    int year, month, day;
+//    sscanf(cha, "%d-%d-%d", &year, &month, &day);
+//    tm_.tm_year = year - 1900;
+//    tm_.tm_mon = month - 1;
+//    tm_.tm_mday = day;
+//    tm_.tm_hour = 0;
+//    tm_.tm_min = 0;
+//    tm_.tm_sec = 0;
+//    tm_.tm_isdst = 0;
+//    data_time = mktime(&tm_);
+//    return SERVER_SUCCESS;
+//
+//}
+//
+//}
+//}
+//}

cpp/src/license/LicenseLibrary.h

@@ -1,105 +1,105 @@
-#pragma once
-
-#include "LicenseFile.h"
-#include "GPUInfoFile.h"
-
-#include "utils/Error.h"
-
-#include <boost/asio.hpp>
-#include <boost/thread.hpp>
-#include <boost/date_time/posix_time/posix_time.hpp>
-
-#include <vector>
-#include <map>
-#include <time.h>
-
-
-namespace zilliz {
-namespace milvus {
-namespace server {
-
-class LicenseLibrary {
- public:
-    // Part 0: File check
-    static bool
-    IsFileExistent(const std::string &path);
-
-    // Part 1: Get GPU Info
-    static ServerError
-    GetDeviceCount(int &device_count);
-
-    static ServerError
-    GetUUID(int device_count, std::vector<std::string> &uuid_array);
-
-    static ServerError
-    GetUUIDMD5(int device_count, std::vector<std::string> &uuid_array, std::vector<std::string> &md5_array);
-
-
-    static ServerError
-    GetUUIDSHA256(const int &device_count,
-                  std::vector<std::string> &uuid_array,
-                  std::vector<std::string> &sha_array);
-
-    static ServerError
-    GetSystemTime(time_t &system_time);
-
-    // Part 2: Handle License File
-    static ServerError
-    LicenseFileSerialization(const std::string &path,
-                             int device_count,
-                             const std::map<int, std::string> &uuid_encrption_map,
-                             time_t starting_time,
-                             time_t end_time);
-
-    static ServerError
-    LicenseFileDeserialization(const std::string &path,
-                               int &device_count,
-                               std::map<int, std::string> &uuid_encrption_map,
-                               time_t &starting_time,
-                               time_t &end_time);
-
+//#pragma once
+//
+//#include "LicenseFile.h"
+//#include "GPUInfoFile.h"
+//
+//#include "utils/Error.h"
+//
+//#include <boost/asio.hpp>
+//#include <boost/thread.hpp>
+//#include <boost/date_time/posix_time/posix_time.hpp>
+//
+//#include <vector>
+//#include <map>
+//#include <time.h>
+//
+//
+//namespace zilliz {
+//namespace milvus {
+//namespace server {
+//
+//class LicenseLibrary {
+// public:
+//    // Part 0: File check
+//    static bool
+//    IsFileExistent(const std::string &path);
+//
+//    // Part 1: Get GPU Info
 //    static ServerError
-//    SecretFileSerialization(const std::string &path,
-//                            const time_t &update_time,
-//                            const off_t &file_size,
-//                            const time_t &starting_time,
-//                            const time_t &end_time,
-//                            const std::string &file_md5);
+//    GetDeviceCount(int &device_count);
 //
 //    static ServerError
-//    SecretFileDeserialization(const std::string &path,
-//                              time_t &update_time,
-//                              off_t &file_size,
-//                              time_t &starting_time,
-//                              time_t &end_time,
-//                              std::string &file_md5);
-
-    // Part 3: File attribute: UpdateTime Time/ Size/ MD5
-    static ServerError
-    GetFileUpdateTimeAndSize(const std::string &path, time_t &update_time, off_t &file_size);
-
-    static ServerError
-    GetFileMD5(const std::string &path, std::string &filemd5);
-
-    // Part 4: GPU Info File Serialization/Deserialization
-    static ServerError
-    GPUinfoFileSerialization(const std::string &path,
-                             int device_count,
-                             const std::map<int, std::string> &uuid_encrption_map);
-    static ServerError
-    GPUinfoFileDeserialization(const std::string &path,
-                               int &device_count,
-                               std::map<int, std::string> &uuid_encrption_map);
-
-    static ServerError
-    GetDateTime(const char *cha, time_t &data_time);
-
-
- private:
-    static constexpr int sha256_length_ = 32;
-};
-
-
-}
-}
-}
+//    GetUUID(int device_count, std::vector<std::string> &uuid_array);
+//
+//    static ServerError
+//    GetUUIDMD5(int device_count, std::vector<std::string> &uuid_array, std::vector<std::string> &md5_array);
+//
+//
+//    static ServerError
+//    GetUUIDSHA256(const int &device_count,
+//                  std::vector<std::string> &uuid_array,
+//                  std::vector<std::string> &sha_array);
+//
+//    static ServerError
+//    GetSystemTime(time_t &system_time);
+//
+//    // Part 2: Handle License File
+//    static ServerError
+//    LicenseFileSerialization(const std::string &path,
+//                             int device_count,
+//                             const std::map<int, std::string> &uuid_encrption_map,
+//                             time_t starting_time,
+//                             time_t end_time);
+//
+//    static ServerError
+//    LicenseFileDeserialization(const std::string &path,
+//                               int &device_count,
+//                               std::map<int, std::string> &uuid_encrption_map,
+//                               time_t &starting_time,
+//                               time_t &end_time);
+//
+////    static ServerError
+////    SecretFileSerialization(const std::string &path,
+////                            const time_t &update_time,
+////                            const off_t &file_size,
+////                            const time_t &starting_time,
+////                            const time_t &end_time,
+////                            const std::string &file_md5);
+////
+////    static ServerError
+////    SecretFileDeserialization(const std::string &path,
+////                              time_t &update_time,
+////                              off_t &file_size,
+////                              time_t &starting_time,
+////                              time_t &end_time,
+////                              std::string &file_md5);
+//
+//    // Part 3: File attribute: UpdateTime Time/ Size/ MD5
+//    static ServerError
+//    GetFileUpdateTimeAndSize(const std::string &path, time_t &update_time, off_t &file_size);
+//
+//    static ServerError
+//    GetFileMD5(const std::string &path, std::string &filemd5);
+//
+//    // Part 4: GPU Info File Serialization/Deserialization
+//    static ServerError
+//    GPUinfoFileSerialization(const std::string &path,
+//                             int device_count,
+//                             const std::map<int, std::string> &uuid_encrption_map);
+//    static ServerError
+//    GPUinfoFileDeserialization(const std::string &path,
+//                               int &device_count,
+//                               std::map<int, std::string> &uuid_encrption_map);
+//
+//    static ServerError
+//    GetDateTime(const char *cha, time_t &data_time);
+//
+//
+// private:
+//    static constexpr int sha256_length_ = 32;
+//};
+//
+//
+//}
+//}
+//}

cpp/src/metrics/MetricBase.h

@@ -22,40 +22,21 @@ class MetricsBase{
     }
 
     virtual ServerError Init() {};
-    virtual void AddGroupSuccessTotalIncrement(double value = 1) {};
-    virtual void AddGroupFailTotalIncrement(double value = 1) {};
-    virtual void HasGroupSuccessTotalIncrement(double value = 1) {};
-    virtual void HasGroupFailTotalIncrement(double value = 1) {};
-    virtual void GetGroupSuccessTotalIncrement(double value = 1) {};
-    virtual void GetGroupFailTotalIncrement(double value = 1) {};
-    virtual void GetGroupFilesSuccessTotalIncrement(double value = 1) {};
-    virtual void GetGroupFilesFailTotalIncrement(double value = 1) {};
+
     virtual void AddVectorsSuccessTotalIncrement(double value = 1) {};
     virtual void AddVectorsFailTotalIncrement(double value = 1) {};
     virtual void AddVectorsDurationHistogramOberve(double value) {};
-    virtual void SearchSuccessTotalIncrement(double value = 1) {};
-    virtual void SearchFailTotalIncrement(double value = 1) {};
-    virtual void SearchDurationHistogramObserve(double value) {};
+
     virtual void RawFileSizeHistogramObserve(double value) {};
     virtual void IndexFileSizeHistogramObserve(double value) {};
     virtual void BuildIndexDurationSecondsHistogramObserve(double value) {};
-    virtual void AllBuildIndexDurationSecondsHistogramObserve(double value) {};
-    virtual void CacheUsageGaugeIncrement(double value = 1) {};
-    virtual void CacheUsageGaugeDecrement(double value = 1) {};
+
     virtual void CacheUsageGaugeSet(double value) {};
-    virtual void MetaVisitTotalIncrement(double value = 1) {};
-    virtual void MetaVisitDurationSecondsHistogramObserve(double value) {};
-    virtual void MemUsagePercentGaugeSet(double value) {};
-    virtual void MemUsagePercentGaugeIncrement(double value = 1) {};
-    virtual void MemUsagePercentGaugeDecrement(double value = 1) {};
-    virtual void MemUsageTotalGaugeSet(double value) {};
-    virtual void MemUsageTotalGaugeIncrement(double value = 1) {};
-    virtual void MemUsageTotalGaugeDecrement(double value = 1) {};
+
     virtual void MetaAccessTotalIncrement(double value = 1) {};
     virtual void MetaAccessDurationSecondsHistogramObserve(double value) {};
     virtual void FaissDiskLoadDurationSecondsHistogramObserve(double value) {};
     virtual void FaissDiskLoadSizeBytesHistogramObserve(double value) {};
-    virtual void FaissDiskLoadIOSpeedHistogramObserve(double value) {};
     virtual void CacheAccessTotalIncrement(double value = 1) {};
     virtual void MemTableMergeDurationSecondsHistogramObserve(double value) {};
     virtual void SearchIndexDataDurationSecondsHistogramObserve(double value) {};

cpp/src/metrics/PrometheusMetrics.cpp

@@ -60,9 +60,6 @@ PrometheusMetrics::GPUPercentGaugeSet() {
     if(!startup_) return;
     int numDevide = server::SystemInfo::GetInstance().num_device();
     std::vector<unsigned int> values = server::SystemInfo::GetInstance().GPUPercent();
-//    for (int i = 0; i < numDevide; ++i) {
-//        GPU_percent_gauges_[i].Set(static_cast<double>(values[i]));
-//    }
     if(numDevide >= 1) GPU0_percent_gauge_.Set(static_cast<double>(values[0]));
     if(numDevide >= 2) GPU1_percent_gauge_.Set(static_cast<double>(values[1]));
     if(numDevide >= 3) GPU2_percent_gauge_.Set(static_cast<double>(values[2]));
@@ -77,13 +74,10 @@ PrometheusMetrics::GPUPercentGaugeSet() {
 
 void PrometheusMetrics::GPUMemoryUsageGaugeSet() {
     if(!startup_) return;
-    int numDevide = server::SystemInfo::GetInstance().num_device();
     std::vector<unsigned long long> values = server::SystemInfo::GetInstance().GPUMemoryUsed();
     constexpr unsigned long long MtoB = 1024*1024;
     int numDevice = values.size();
-//    for (int i = 0; i < numDevice; ++i) {
-//        GPU_memory_usage_gauges_[i].Set(values[i]/MtoB);
-//    }
+
     if(numDevice >=1) GPU0_memory_usage_gauge_.Set(values[0]/MtoB);
     if(numDevice >=2) GPU1_memory_usage_gauge_.Set(values[1]/MtoB);
     if(numDevice >=3) GPU2_memory_usage_gauge_.Set(values[2]/MtoB);
@@ -146,21 +140,6 @@ void PrometheusMetrics::OctetsSet() {
     outoctets_gauge_.Set((in_and_out_octets.second-old_outoctets)/total_second);
 }
 
-//void PrometheusMetrics::GpuPercentInit() {
-//    int num_device = SystemInfo::GetInstance().num_device();
-//    constexpr char device_number[] = "DeviceNum";
-//    for(int i = 0; i < num_device; ++ i) {
-//        GPU_percent_gauges_.emplace_back(GPU_percent_.Add({{device_number,std::to_string(i)}}));
-//    }
-//
-//}
-//void PrometheusMetrics::GpuMemoryInit() {
-//    int num_device = SystemInfo::GetInstance().num_device();
-//    constexpr char device_number[] = "DeviceNum";
-//    for(int i = 0; i < num_device; ++ i) {
-//        GPU_memory_usage_gauges_.emplace_back(GPU_memory_usage_.Add({{device_number,std::to_string(i)}}));
-//    }
-//}
 
 
 }

cpp/src/metrics/PrometheusMetrics.h

@@ -34,10 +34,6 @@ class PrometheusMetrics: public MetricsBase {
  public:
     static PrometheusMetrics &
     GetInstance() {
-//        switch(MetricCollectorType) {
-//            case: prometheus::
-//                static
-//        }
         static PrometheusMetrics instance;
         return instance;
     }
@@ -49,46 +45,21 @@ class PrometheusMetrics: public MetricsBase {
     std::shared_ptr<prometheus::Exposer> exposer_ptr_;
     std::shared_ptr<prometheus::Registry> registry_ = std::make_shared<prometheus::Registry>();
     bool startup_ = false;
-//    void GpuPercentInit();
-//    void GpuMemoryInit();
  public:
-
-    void AddGroupSuccessTotalIncrement(double value = 1.0) override { if(startup_) add_group_success_total_.Increment(value);};
-    void AddGroupFailTotalIncrement(double value = 1.0) override { if(startup_) add_group_fail_total_.Increment(value);};
-    void HasGroupSuccessTotalIncrement(double value = 1.0) override { if(startup_) has_group_success_total_.Increment(value);};
-    void HasGroupFailTotalIncrement(double value = 1.0) override { if(startup_) has_group_fail_total_.Increment(value);};
-    void GetGroupSuccessTotalIncrement(double value = 1.0) override { if(startup_) get_group_success_total_.Increment(value);};
-    void GetGroupFailTotalIncrement(double value = 1.0) override { if(startup_) get_group_fail_total_.Increment(value);};
-    void GetGroupFilesSuccessTotalIncrement(double value = 1.0) override { if(startup_) get_group_files_success_total_.Increment(value);};
-    void GetGroupFilesFailTotalIncrement(double value = 1.0) override { if(startup_) get_group_files_fail_total_.Increment(value);};
+    void SetStartup(bool startup) {startup_ = startup;};
     void AddVectorsSuccessTotalIncrement(double value = 1.0) override { if(startup_) add_vectors_success_total_.Increment(value);};
     void AddVectorsFailTotalIncrement(double value = 1.0) override { if(startup_) add_vectors_fail_total_.Increment(value);};
     void AddVectorsDurationHistogramOberve(double value) override { if(startup_) add_vectors_duration_histogram_.Observe(value);};
-    void SearchSuccessTotalIncrement(double value = 1.0) override { if(startup_) search_success_total_.Increment(value);};
-    void SearchFailTotalIncrement(double value = 1.0) override { if(startup_) search_fail_total_.Increment(value); };
-    void SearchDurationHistogramObserve(double value) override { if(startup_) search_duration_histogram_.Observe(value);};
     void RawFileSizeHistogramObserve(double value) override { if(startup_) raw_files_size_histogram_.Observe(value);};
     void IndexFileSizeHistogramObserve(double value) override { if(startup_) index_files_size_histogram_.Observe(value);};
     void BuildIndexDurationSecondsHistogramObserve(double value) override { if(startup_) build_index_duration_seconds_histogram_.Observe(value);};
-    void AllBuildIndexDurationSecondsHistogramObserve(double value) override { if(startup_) all_build_index_duration_seconds_histogram_.Observe(value);};
-    void CacheUsageGaugeIncrement(double value = 1.0) override { if(startup_) cache_usage_gauge_.Increment(value);};
-    void CacheUsageGaugeDecrement(double value = 1.0) override { if(startup_) cache_usage_gauge_.Decrement(value);};
     void CacheUsageGaugeSet(double value) override { if(startup_) cache_usage_gauge_.Set(value);};
-//    void MetaVisitTotalIncrement(double value = 1.0) override { meta_visit_total_.Increment(value);};
-//    void MetaVisitDurationSecondsHistogramObserve(double value) override { meta_visit_duration_seconds_histogram_.Observe(value);};
-    void MemUsagePercentGaugeSet(double value) override { if(startup_) mem_usage_percent_gauge_.Set(value);};
-    void MemUsagePercentGaugeIncrement(double value = 1.0) override { if(startup_) mem_usage_percent_gauge_.Increment(value);};
-    void MemUsagePercentGaugeDecrement(double value = 1.0) override { if(startup_) mem_usage_percent_gauge_.Decrement(value);};
-    void MemUsageTotalGaugeSet(double value) override { if(startup_) mem_usage_total_gauge_.Set(value);};
-    void MemUsageTotalGaugeIncrement(double value = 1.0) override { if(startup_) mem_usage_total_gauge_.Increment(value);};
-    void MemUsageTotalGaugeDecrement(double value = 1.0) override { if(startup_) mem_usage_total_gauge_.Decrement(value);};
 
     void MetaAccessTotalIncrement(double value = 1) override { if(startup_) meta_access_total_.Increment(value);};
     void MetaAccessDurationSecondsHistogramObserve(double value) override { if(startup_) meta_access_duration_seconds_histogram_.Observe(value);};
 
     void FaissDiskLoadDurationSecondsHistogramObserve(double value) override { if(startup_) faiss_disk_load_duration_seconds_histogram_.Observe(value);};
     void FaissDiskLoadSizeBytesHistogramObserve(double value) override { if(startup_) faiss_disk_load_size_bytes_histogram_.Observe(value);};
-//    void FaissDiskLoadIOSpeedHistogramObserve(double value) { if(startup_) faiss_disk_load_IO_speed_histogram_.Observe(value);};
     void FaissDiskLoadIOSpeedGaugeSet(double value) override { if(startup_) faiss_disk_load_IO_speed_gauge_.Set(value);};
 
     void CacheAccessTotalIncrement(double value = 1) override { if(startup_) cache_access_total_.Increment(value);};

cpp/src/metrics/SystemInfo.cpp

@@ -135,46 +135,6 @@ SystemInfo::CPUPercent() {
     return percent;
 }
 
-//std::unordered_map<int,std::vector<double>>
-//SystemInfo::GetGPUMemPercent(){
-//    // return GPUID: MEM%
-//
-//    //write GPU info to a file
-//    system("nvidia-smi pmon -c 1 > GPUInfo.txt");
-//    int pid = (int)getpid();
-//
-//    //parse line
-//    std::ifstream read_file;
-//    read_file.open("GPUInfo.txt");
-//    std::string line;
-//    while(getline(read_file, line)){
-//        std::vector<std::string> words = split(line);
-//        //                    0      1     2    3   4    5    6      7
-//        //words stand for gpuindex, pid, type, sm, mem, enc, dec, command respectively
-//        if(std::stoi(words[1]) != pid) continue;
-//        int GPUindex = std::stoi(words[0]);
-//        double sm_percent = std::stod(words[3]);
-//        double mem_percent = std::stod(words[4]);
-//
-//    }
-//
-//}
-
-//std::vector<std::string>
-//SystemInfo::split(std::string input) {
-//    std::vector<std::string> words;
-//    input += " ";
-//    int word_start = 0;
-//    for (int i = 0; i < input.size(); ++i) {
-//        if(input[i] != ' ') continue;
-//        if(input[i] == ' ') {
-//            word_start = i + 1;
-//            continue;
-//        }
-//        words.push_back(input.substr(word_start,i-word_start));
-//    }
-//    return words;
-//}
 
 std::vector<unsigned int>
 SystemInfo::GPUPercent() {

cpp/src/metrics/SystemInfo.h

@@ -59,8 +59,6 @@ class SystemInfo {
     double MemoryPercent();
     double CPUPercent();
     std::pair<unsigned long long , unsigned long long > Octets();
-//    std::unordered_map<int,std::vector<double>> GetGPUMemPercent() {};
-//    std::vector<std::string> split(std::string input) {};
     std::vector<unsigned int> GPUPercent();
     std::vector<unsigned long long> GPUMemoryUsed();
 

cpp/src/sdk/src/client/ClientProxy.cpp

@@ -4,7 +4,6 @@
  * Proprietary and confidential.
  ******************************************************************************/
 #include "ClientProxy.h"
-#include "util/ConvertUtil.h"
 
 namespace milvus {
 

cpp/src/sdk/src/util/ConvertUtil.cpp

@@ -1,44 +0,0 @@
-/*******************************************************************************
- * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved
- * Unauthorized copying of this file, via any medium is strictly prohibited.
- * Proprietary and confidential.
- ******************************************************************************/
-#include "ConvertUtil.h"
-#include "Exception.h"
-
-#include <map>
-
-namespace milvus {
-
-static const std::string INDEX_RAW = "raw";
-static const std::string INDEX_IVFFLAT = "ivfflat";
-
-std::string ConvertUtil::IndexType2Str(IndexType index) {
-    static const std::map<IndexType, std::string> s_index2str = {
-            {IndexType::cpu_idmap, INDEX_RAW},
-            {IndexType::gpu_ivfflat, INDEX_IVFFLAT}
-    };
-
-    const auto& iter = s_index2str.find(index);
-    if(iter == s_index2str.end()) {
-        throw Exception(StatusCode::InvalidAgument, "Invalid index type");
-    }
-
-    return iter->second;
-}
-
-IndexType ConvertUtil::Str2IndexType(const std::string& type) {
-    static const std::map<std::string, IndexType> s_str2index = {
-            {INDEX_RAW, IndexType::cpu_idmap},
-            {INDEX_IVFFLAT, IndexType::gpu_ivfflat}
-    };
-
-    const auto& iter = s_str2index.find(type);
-    if(iter == s_str2index.end()) {
-        throw Exception(StatusCode::InvalidAgument, "Invalid index type");
-    }
-
-    return iter->second;
-}
-
-}

cpp/src/sdk/src/util/ConvertUtil.h

@@ -1,18 +0,0 @@
-/*******************************************************************************
- * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved
- * Unauthorized copying of this file, via any medium is strictly prohibited.
- * Proprietary and confidential.
- ******************************************************************************/
-#pragma once
-
-#include "MilvusApi.h"
-
-namespace milvus {
-
-class ConvertUtil {
-public:
-    static std::string IndexType2Str(IndexType index);
-    static IndexType Str2IndexType(const std::string& type);
-};
-
-}

cpp/src/server/DBWrapper.cpp

@@ -37,12 +37,16 @@ DBWrapper::DBWrapper() {
     opt.meta.archive_conf.SetCriterias(criterial);
 
     //create db root folder
-    CommonUtil::CreateDirectory(opt.meta.path);
+    ServerError err = CommonUtil::CreateDirectory(opt.meta.path);
+    if(err != SERVER_SUCCESS) {
+        std::cout << "ERROR! Failed to create database root path: " << opt.meta.path << std::endl;
+        kill(0, SIGUSR1);
+    }
 
     zilliz::milvus::engine::DB::Open(opt, &db_);
     if(db_ == nullptr) {
-        SERVER_LOG_ERROR << "Failed to open db";
-        throw ServerException(SERVER_NULL_POINTER, "Failed to open db");
+        std::cout << "ERROR! Failed to open database" << std::endl;
+        kill(0, SIGUSR1);
     }
 }
 

cpp/src/server/MilvusServer.cpp

@@ -23,6 +23,7 @@
 #include <thrift/concurrency/PosixThreadFactory.h>
 
 #include <thread>
+#include <iostream>
 
 namespace zilliz {
 namespace milvus {
@@ -92,7 +93,8 @@ MilvusServer::StartService() {
             return;
         }
     } catch (apache::thrift::TException& ex) {
-        //SERVER_LOG_ERROR << "Server encounter exception: " << ex.what();
+        std::cout << "ERROR! " << ex.what() << std::endl;
+        kill(0, SIGUSR1);
     }
 }
 

cpp/src/server/RequestScheduler.cpp

@@ -18,35 +18,35 @@ using namespace ::milvus;
 namespace {
     const std::map<ServerError, thrift::ErrorCode::type> &ErrorMap() {
         static const std::map<ServerError, thrift::ErrorCode::type> code_map = {
-                {SERVER_UNEXPECTED_ERROR,         thrift::ErrorCode::ILLEGAL_ARGUMENT},
-                {SERVER_NULL_POINTER,             thrift::ErrorCode::ILLEGAL_ARGUMENT},
+                {SERVER_UNEXPECTED_ERROR,         thrift::ErrorCode::UNEXPECTED_ERROR},
+                {SERVER_UNSUPPORTED_ERROR,        thrift::ErrorCode::UNEXPECTED_ERROR},
+                {SERVER_NULL_POINTER,             thrift::ErrorCode::UNEXPECTED_ERROR},
                 {SERVER_INVALID_ARGUMENT,         thrift::ErrorCode::ILLEGAL_ARGUMENT},
-                {SERVER_FILE_NOT_FOUND,           thrift::ErrorCode::ILLEGAL_ARGUMENT},
-                {SERVER_NOT_IMPLEMENT,            thrift::ErrorCode::ILLEGAL_ARGUMENT},
-                {SERVER_BLOCKING_QUEUE_EMPTY,     thrift::ErrorCode::ILLEGAL_ARGUMENT},
+                {SERVER_FILE_NOT_FOUND,           thrift::ErrorCode::FILE_NOT_FOUND},
+                {SERVER_NOT_IMPLEMENT,            thrift::ErrorCode::UNEXPECTED_ERROR},
+                {SERVER_BLOCKING_QUEUE_EMPTY,     thrift::ErrorCode::UNEXPECTED_ERROR},
+                {SERVER_CANNOT_CREATE_FOLDER,     thrift::ErrorCode::CANNOT_CREATE_FOLDER},
+                {SERVER_CANNOT_CREATE_FILE,       thrift::ErrorCode::CANNOT_CREATE_FILE},
+                {SERVER_CANNOT_DELETE_FOLDER,     thrift::ErrorCode::CANNOT_DELETE_FOLDER},
+                {SERVER_CANNOT_DELETE_FILE,       thrift::ErrorCode::CANNOT_DELETE_FILE},
                 {SERVER_TABLE_NOT_EXIST,          thrift::ErrorCode::TABLE_NOT_EXISTS},
+                {SERVER_INVALID_TABLE_NAME,       thrift::ErrorCode::ILLEGAL_TABLE_NAME},
+                {SERVER_INVALID_TABLE_DIMENSION,  thrift::ErrorCode::ILLEGAL_DIMENSION},
                 {SERVER_INVALID_TIME_RANGE,       thrift::ErrorCode::ILLEGAL_RANGE},
                 {SERVER_INVALID_VECTOR_DIMENSION, thrift::ErrorCode::ILLEGAL_DIMENSION},
+
+                {SERVER_INVALID_INDEX_TYPE,       thrift::ErrorCode::ILLEGAL_INDEX_TYPE},
+                {SERVER_INVALID_ROWRECORD,        thrift::ErrorCode::ILLEGAL_ROWRECORD},
+                {SERVER_INVALID_ROWRECORD_ARRAY,  thrift::ErrorCode::ILLEGAL_ROWRECORD},
+                {SERVER_INVALID_TOPK,             thrift::ErrorCode::ILLEGAL_TOPK},
+                {SERVER_ILLEGAL_VECTOR_ID,        thrift::ErrorCode::ILLEGAL_VECTOR_ID},
+                {SERVER_ILLEGAL_SEARCH_RESULT,    thrift::ErrorCode::ILLEGAL_SEARCH_RESULT},
+                {SERVER_CACHE_ERROR,              thrift::ErrorCode::CACHE_FAILED},
+                {DB_META_TRANSACTION_FAILED,      thrift::ErrorCode::META_FAILED},
         };
 
         return code_map;
     }
-
-    const std::map<ServerError, std::string> &ErrorMessage() {
-        static const std::map<ServerError, std::string> msg_map = {
-                {SERVER_UNEXPECTED_ERROR,         "unexpected error occurs"},
-                {SERVER_NULL_POINTER,             "null pointer error"},
-                {SERVER_INVALID_ARGUMENT,         "invalid argument"},
-                {SERVER_FILE_NOT_FOUND,           "file not found"},
-                {SERVER_NOT_IMPLEMENT,            "not implemented"},
-                {SERVER_BLOCKING_QUEUE_EMPTY,     "queue empty"},
-                {SERVER_TABLE_NOT_EXIST,          "table not exist"},
-                {SERVER_INVALID_TIME_RANGE,       "invalid time range"},
-                {SERVER_INVALID_VECTOR_DIMENSION, "invalid vector dimension"},
-        };
-
-        return msg_map;
-    }
 }
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
@@ -69,6 +69,14 @@ ServerError BaseTask::Execute() {
     return error_code_;
 }
 
+ServerError BaseTask::SetError(ServerError error_code, const std::string& error_msg) {
+    error_code_ = error_code;
+    error_msg_ = error_msg;
+
+    SERVER_LOG_ERROR << error_msg_;
+    return error_code_;
+}
+
 ServerError BaseTask::WaitToFinish() {
     std::unique_lock <std::mutex> lock(finish_mtx_);
     finish_cond_.wait(lock, [this] { return done_; });
@@ -102,7 +110,7 @@ void RequestScheduler::ExecTask(BaseTaskPtr& task_ptr) {
             ex.__set_code(ErrorMap().at(err));
             std::string msg = task_ptr->ErrorMsg();
             if(msg.empty()){
-                msg = ErrorMessage().at(err);
+                msg = "Error message not set";
             }
             ex.__set_reason(msg);
             throw ex;

cpp/src/server/RequestScheduler.h

@@ -34,6 +34,8 @@ public:
 protected:
     virtual ServerError OnExecute() = 0;
 
+    ServerError SetError(ServerError error_code, const std::string& msg);
+
 protected:
     mutable std::mutex finish_mtx_;
     std::condition_variable finish_cond_;

cpp/src/server/RequestTask.cpp

@@ -53,26 +53,27 @@ namespace {
         return map_type[type];
     }
 
-    ServerError
+    void
     ConvertRowRecordToFloatArray(const std::vector<thrift::RowRecord>& record_array,
                                  uint64_t dimension,
-                                 std::vector<float>& float_array) {
-        ServerError error_code;
+                                 std::vector<float>& float_array,
+                                 ServerError& error_code,
+                                 std::string& error_msg) {
         uint64_t vec_count = record_array.size();
         float_array.resize(vec_count*dimension);//allocate enough memory
         for(uint64_t i = 0; i < vec_count; i++) {
             const auto& record = record_array[i];
             if(record.vector_data.empty()) {
-                error_code = SERVER_INVALID_ARGUMENT;
-                SERVER_LOG_ERROR << "No vector provided in record";
-                return error_code;
+                error_code = SERVER_INVALID_ROWRECORD;
+                error_msg = "Rowrecord float array is empty";
+                return;
             }
             uint64_t vec_dim = record.vector_data.size()/sizeof(double);//how many double value?
             if(vec_dim != dimension) {
-                SERVER_LOG_ERROR << "Invalid vector dimension: " << vec_dim
-                                 << " vs. table dimension:" << dimension;
                 error_code = SERVER_INVALID_VECTOR_DIMENSION;
-                return error_code;
+                error_msg = "Invalid rowrecord dimension: " + std::to_string(vec_dim)
+                                 + " vs. table dimension:" + std::to_string(dimension);
+                return;
             }
 
             //convert double array to float array(thrift has no float type)
@@ -81,30 +82,29 @@ namespace {
                 float_array[i*vec_dim + d] = (float)(d_p[d]);
             }
         }
-
-        return SERVER_SUCCESS;
     }
 
     static constexpr long DAY_SECONDS = 86400;
 
-    ServerError
+    void
     ConvertTimeRangeToDBDates(const std::vector<thrift::Range> &range_array,
-                              std::vector<DB_DATE>& dates) {
+                              std::vector<DB_DATE>& dates,
+                              ServerError& error_code,
+                              std::string& error_msg) {
         dates.clear();
-        ServerError error_code;
         for(auto& range : range_array) {
             time_t tt_start, tt_end;
             tm tm_start, tm_end;
             if(!CommonUtil::TimeStrToTime(range.start_value, tt_start, tm_start)){
                 error_code = SERVER_INVALID_TIME_RANGE;
-                SERVER_LOG_ERROR << "Invalid time range: " << range.start_value;
-                return error_code;
+                error_msg = "Invalid time range: " + range.start_value;
+                return;
             }
 
             if(!CommonUtil::TimeStrToTime(range.end_value, tt_end, tm_end)){
                 error_code = SERVER_INVALID_TIME_RANGE;
-                SERVER_LOG_ERROR << "Invalid time range: " << range.end_value;
-                return error_code;
+                error_msg = "Invalid time range: " + range.start_value;
+                return;
             }
 
             long days = (tt_end > tt_start) ? (tt_end - tt_start)/DAY_SECONDS : (tt_start - tt_end)/DAY_SECONDS;
@@ -117,8 +117,6 @@ namespace {
                 dates.push_back(date);
             }
         }
-
-        return SERVER_SUCCESS;
     }
 }
 
@@ -138,19 +136,16 @@ ServerError CreateTableTask::OnExecute() {
     
     try {
         //step 1: check arguments
-        if(schema_.table_name.empty() || schema_.dimension <= 0) {
-            error_code_ = SERVER_INVALID_ARGUMENT;
-            error_msg_ = "Invalid table name or dimension";
-            SERVER_LOG_ERROR << error_msg_;
-            return error_code_;
+        if(schema_.table_name.empty()) {
+            return SetError(SERVER_INVALID_TABLE_NAME, "Empty table name");
+        }
+        if(schema_.dimension <= 0) {
+            return SetError(SERVER_INVALID_TABLE_DIMENSION, "Invalid table dimension: " + std::to_string(schema_.dimension));
         }
 
         engine::EngineType engine_type = EngineType(schema_.index_type);
         if(engine_type == engine::EngineType::INVALID) {
-            error_code_ = SERVER_INVALID_ARGUMENT;
-            error_msg_ = "Invalid index type";
-            SERVER_LOG_ERROR << error_msg_;
-            return error_code_;
+            return SetError(SERVER_INVALID_INDEX_TYPE, "Invalid index type: " + std::to_string(schema_.index_type));
         }
 
         //step 2: construct table schema
@@ -163,17 +158,11 @@ ServerError CreateTableTask::OnExecute() {
         //step 3: create table
         engine::Status stat = DBWrapper::DB()->CreateTable(table_info);
         if(!stat.ok()) {//table could exist
-            error_code_ = SERVER_UNEXPECTED_ERROR;
-            error_msg_ = "Engine failed: " + stat.ToString();
-            SERVER_LOG_ERROR << error_msg_;
-            return error_code_;
+            return SetError(DB_META_TRANSACTION_FAILED, "Engine failed: " + stat.ToString());
         }
 
     } catch (std::exception& ex) {
-        error_code_ = SERVER_UNEXPECTED_ERROR;
-        error_msg_ = ex.what();
-        SERVER_LOG_ERROR << error_msg_;
-        return error_code_;
+        return SetError(SERVER_UNEXPECTED_ERROR, ex.what());
     }
 
     rc.Record("done");
@@ -199,10 +188,7 @@ ServerError DescribeTableTask::OnExecute() {
     try {
         //step 1: check arguments
         if(table_name_.empty()) {
-            error_code_ = SERVER_INVALID_ARGUMENT;
-            error_msg_ = "Table name cannot be empty";
-            SERVER_LOG_ERROR << error_msg_;
-            return error_code_;
+            return SetError(SERVER_INVALID_TABLE_NAME, "Empty table name");
         }
 
         //step 2: get table info
@@ -210,10 +196,7 @@ ServerError DescribeTableTask::OnExecute() {
         table_info.table_id_ = table_name_;
         engine::Status stat = DBWrapper::DB()->DescribeTable(table_info);
         if(!stat.ok()) {
-            error_code_ = SERVER_TABLE_NOT_EXIST;
-            error_msg_ = "Engine failed: " + stat.ToString();
-            SERVER_LOG_ERROR << error_msg_;
-            return error_code_;
+            return SetError(DB_META_TRANSACTION_FAILED, "Engine failed: " + stat.ToString());
         }
 
         schema_.table_name = table_info.table_id_;
@@ -222,10 +205,7 @@ ServerError DescribeTableTask::OnExecute() {
         schema_.store_raw_vector = table_info.store_raw_data_;
 
     } catch (std::exception& ex) {
-        error_code_ = SERVER_UNEXPECTED_ERROR;
-        error_msg_ = ex.what();
-        SERVER_LOG_ERROR << error_msg_;
-        return SERVER_UNEXPECTED_ERROR;
+        return SetError(SERVER_UNEXPECTED_ERROR, ex.what());
     }
 
     rc.Record("done");
@@ -250,22 +230,19 @@ ServerError HasTableTask::OnExecute() {
         TimeRecorder rc("HasTableTask");
 
         //step 1: check arguments
-        if (table_name_.empty()) {
-            error_code_ = SERVER_INVALID_ARGUMENT;
-            error_msg_ = "Table name cannot be empty";
-            SERVER_LOG_ERROR << error_msg_;
-            return error_code_;
+        if(table_name_.empty()) {
+            return SetError(SERVER_INVALID_TABLE_NAME, "Empty table name");
         }
 
         //step 2: check table existence
         engine::Status stat = DBWrapper::DB()->HasTable(table_name_, has_table_);
+        if(!stat.ok()) {
+            return SetError(DB_META_TRANSACTION_FAILED, "Engine failed: " + stat.ToString());
+        }
 
         rc.Elapse("totally cost");
     } catch (std::exception& ex) {
-        error_code_ = SERVER_UNEXPECTED_ERROR;
-        error_msg_ = ex.what();
-        SERVER_LOG_ERROR << error_msg_;
-        return error_code_;
+        return SetError(SERVER_UNEXPECTED_ERROR, ex.what());
     }
 
     return SERVER_SUCCESS;
@@ -288,10 +265,7 @@ ServerError DeleteTableTask::OnExecute() {
 
         //step 1: check arguments
         if (table_name_.empty()) {
-            error_code_ = SERVER_INVALID_ARGUMENT;
-            error_msg_ = "Table name cannot be empty";
-            SERVER_LOG_ERROR << error_msg_;
-            return error_code_;
+            return SetError(SERVER_INVALID_TABLE_NAME, "Empty table name");
         }
 
         //step 2: check table existence
@@ -299,10 +273,11 @@ ServerError DeleteTableTask::OnExecute() {
         table_info.table_id_ = table_name_;
         engine::Status stat = DBWrapper::DB()->DescribeTable(table_info);
         if(!stat.ok()) {
-            error_code_ = SERVER_TABLE_NOT_EXIST;
-            error_msg_ = "Engine failed: " + stat.ToString();
-            SERVER_LOG_ERROR << error_msg_;
-            return error_code_;
+            if(stat.IsNotFound()) {
+                return SetError(SERVER_TABLE_NOT_EXIST, "Table " + table_name_ + " not exists");
+            } else {
+                return SetError(DB_META_TRANSACTION_FAILED, "Engine failed: " + stat.ToString());
+            }
         }
 
         rc.Record("check validation");
@@ -311,17 +286,13 @@ ServerError DeleteTableTask::OnExecute() {
         std::vector<DB_DATE> dates;
         stat = DBWrapper::DB()->DeleteTable(table_name_, dates);
         if(!stat.ok()) {
-            SERVER_LOG_ERROR << "Engine failed: " << stat.ToString();
-            return SERVER_UNEXPECTED_ERROR;
+            return SetError(DB_META_TRANSACTION_FAILED, "Engine failed: " + stat.ToString());
         }
 
         rc.Record("deleta table");
         rc.Elapse("totally cost");
     } catch (std::exception& ex) {
-        error_code_ = SERVER_UNEXPECTED_ERROR;
-        error_msg_ = ex.what();
-        SERVER_LOG_ERROR << error_msg_;
-        return error_code_;
+        return SetError(SERVER_UNEXPECTED_ERROR, ex.what());
     }
 
     return SERVER_SUCCESS;
@@ -342,10 +313,7 @@ ServerError ShowTablesTask::OnExecute() {
     std::vector<engine::meta::TableSchema> schema_array;
     engine::Status stat = DBWrapper::DB()->AllTables(schema_array);
     if(!stat.ok()) {
-        error_code_ = SERVER_UNEXPECTED_ERROR;
-        error_msg_ = "Engine failed: " + stat.ToString();
-        SERVER_LOG_ERROR << error_msg_;
-        return error_code_;
+        return SetError(DB_META_TRANSACTION_FAILED, "Engine failed: " + stat.ToString());
     }
 
     tables_.clear();
@@ -379,17 +347,11 @@ ServerError AddVectorTask::OnExecute() {
 
         //step 1: check arguments
         if (table_name_.empty()) {
-            error_code_ = SERVER_INVALID_ARGUMENT;
-            error_msg_ = "Table name cannot be empty";
-            SERVER_LOG_ERROR << error_msg_;
-            return error_code_;
+            return SetError(SERVER_INVALID_TABLE_NAME, "Empty table name");
         }
 
         if(record_array_.empty()) {
-            error_code_ = SERVER_INVALID_ARGUMENT;
-            error_msg_ = "Row record array is empty";
-            SERVER_LOG_ERROR << error_msg_;
-            return error_code_;
+            return SetError(SERVER_INVALID_ROWRECORD_ARRAY, "Row record array is empty");
         }
 
         //step 2: check table existence
@@ -397,20 +359,22 @@ ServerError AddVectorTask::OnExecute() {
         table_info.table_id_ = table_name_;
         engine::Status stat = DBWrapper::DB()->DescribeTable(table_info);
         if(!stat.ok()) {
-            error_code_ = SERVER_TABLE_NOT_EXIST;
-            error_msg_ = "Engine failed: " + stat.ToString();
-            SERVER_LOG_ERROR << error_msg_;
-            return error_code_;
+            if(stat.IsNotFound()) {
+                return SetError(SERVER_TABLE_NOT_EXIST, "Table " + table_name_ + " not exists");
+            } else {
+                return SetError(DB_META_TRANSACTION_FAILED, "Engine failed: " + stat.ToString());
+            }
         }
 
         rc.Record("check validation");
 
         //step 3: prepare float data
         std::vector<float> vec_f;
-        error_code_ = ConvertRowRecordToFloatArray(record_array_, table_info.dimension_, vec_f);
-        if(error_code_ != SERVER_SUCCESS) {
-            error_msg_ = "Invalid row record data";
-            return error_code_;
+        ServerError error_code = SERVER_SUCCESS;
+        std::string error_msg;
+        ConvertRowRecordToFloatArray(record_array_, table_info.dimension_, vec_f, error_code, error_msg);
+        if(error_code != SERVER_SUCCESS) {
+            return SetError(error_code, error_msg);
         }
 
         rc.Record("prepare vectors data");
@@ -420,25 +384,20 @@ ServerError AddVectorTask::OnExecute() {
         stat = DBWrapper::DB()->InsertVectors(table_name_, vec_count, vec_f.data(), record_ids_);
         rc.Record("add vectors to engine");
         if(!stat.ok()) {
-            error_code_ = SERVER_UNEXPECTED_ERROR;
-            error_msg_ = "Engine failed: " + stat.ToString();
-            SERVER_LOG_ERROR << error_msg_;
-            return error_code_;
+            return SetError(SERVER_CACHE_ERROR, "Cache error: " + stat.ToString());
         }
 
         if(record_ids_.size() != vec_count) {
-            SERVER_LOG_ERROR << "Vector ID not returned";
-            return SERVER_UNEXPECTED_ERROR;
+            std::string msg = "Add " + std::to_string(vec_count) + " vectors but only return "
+                    + std::to_string(record_ids_.size()) + " id";
+            return SetError(SERVER_ILLEGAL_VECTOR_ID, msg);
         }
 
         rc.Record("do insert");
         rc.Elapse("totally cost");
 
     } catch (std::exception& ex) {
-        error_code_ = SERVER_UNEXPECTED_ERROR;
-        error_msg_ = ex.what();
-        SERVER_LOG_ERROR << error_msg_;
-        return error_code_;
+        return SetError(SERVER_UNEXPECTED_ERROR, ex.what());
     }
 
     return SERVER_SUCCESS;
@@ -477,17 +436,14 @@ ServerError SearchVectorTask::OnExecute() {
 
         //step 1: check arguments
         if (table_name_.empty()) {
-            error_code_ = SERVER_INVALID_ARGUMENT;
-            error_msg_ = "Table name cannot be empty";
-            SERVER_LOG_ERROR << error_msg_;
-            return error_code_;
+            return SetError(SERVER_INVALID_TABLE_NAME, "Empty table name");
         }
 
-        if(top_k_ <= 0 || record_array_.empty()) {
-            error_code_ = SERVER_INVALID_ARGUMENT;
-            error_msg_ = "Invalid topk value, or query record array is empty";
-            SERVER_LOG_ERROR << error_msg_;
-            return error_code_;
+        if(top_k_ <= 0) {
+            return SetError(SERVER_INVALID_TOPK, "Invalid topk: " + std::to_string(top_k_));
+        }
+        if(record_array_.empty()) {
+            return SetError(SERVER_INVALID_ROWRECORD_ARRAY, "Row record array is empty");
         }
 
         //step 2: check table existence
@@ -495,28 +451,29 @@ ServerError SearchVectorTask::OnExecute() {
         table_info.table_id_ = table_name_;
         engine::Status stat = DBWrapper::DB()->DescribeTable(table_info);
         if(!stat.ok()) {
-            error_code_ = SERVER_TABLE_NOT_EXIST;
-            error_msg_ = "Engine failed: " + stat.ToString();
-            SERVER_LOG_ERROR << error_msg_;
-            return error_code_;
+            if(stat.IsNotFound()) {
+                return SetError(SERVER_TABLE_NOT_EXIST, "Table " + table_name_ + " not exists");
+            } else {
+                return SetError(DB_META_TRANSACTION_FAILED, "Engine failed: " + stat.ToString());
+            }
         }
 
         //step 3: check date range, and convert to db dates
         std::vector<DB_DATE> dates;
-        error_code_ = ConvertTimeRangeToDBDates(range_array_, dates);
-        if(error_code_ != SERVER_SUCCESS) {
-            error_msg_ = "Invalid query range";
-            return error_code_;
+        ServerError error_code = SERVER_SUCCESS;
+        std::string error_msg;
+        ConvertTimeRangeToDBDates(range_array_, dates, error_code, error_msg);
+        if(error_code != SERVER_SUCCESS) {
+            return SetError(error_code, error_msg);
         }
 
         rc.Record("check validation");
 
         //step 3: prepare float data
         std::vector<float> vec_f;
-        error_code_ = ConvertRowRecordToFloatArray(record_array_, table_info.dimension_, vec_f);
-        if(error_code_ != SERVER_SUCCESS) {
-            error_msg_ = "Invalid row record data";
-            return error_code_;
+        ConvertRowRecordToFloatArray(record_array_, table_info.dimension_, vec_f, error_code, error_msg);
+        if(error_code != SERVER_SUCCESS) {
+            return SetError(error_code, error_msg);
         }
 
         rc.Record("prepare vector data");
@@ -533,13 +490,17 @@ ServerError SearchVectorTask::OnExecute() {
 
         rc.Record("search vectors from engine");
         if(!stat.ok()) {
-            SERVER_LOG_ERROR << "Engine failed: " << stat.ToString();
-            return SERVER_UNEXPECTED_ERROR;
+            return SetError(DB_META_TRANSACTION_FAILED, "Engine failed: " + stat.ToString());
+        }
+
+        if(results.empty()) {
+            return SERVER_SUCCESS; //empty table
         }
 
         if(results.size() != record_count) {
-            SERVER_LOG_ERROR << "Search result not returned";
-            return SERVER_UNEXPECTED_ERROR;
+            std::string msg = "Search " + std::to_string(record_count) + " vectors but only return "
+                              + std::to_string(results.size()) + " results";
+            return SetError(SERVER_ILLEGAL_SEARCH_RESULT, msg);
         }
 
         rc.Record("do search");
@@ -564,10 +525,7 @@ ServerError SearchVectorTask::OnExecute() {
         rc.Elapse("totally cost");
 
     } catch (std::exception& ex) {
-        error_code_ = SERVER_UNEXPECTED_ERROR;
-        error_msg_ = ex.what();
-        SERVER_LOG_ERROR << error_msg_;
-        return error_code_;
+        return SetError(SERVER_UNEXPECTED_ERROR, ex.what());
     }
 
     return SERVER_SUCCESS;
@@ -591,20 +549,14 @@ ServerError GetTableRowCountTask::OnExecute() {
 
         //step 1: check arguments
         if (table_name_.empty()) {
-            error_code_ = SERVER_INVALID_ARGUMENT;
-            error_msg_ = "Table name cannot be empty";
-            SERVER_LOG_ERROR << error_msg_;
-            return error_code_;
+            return SetError(SERVER_INVALID_TABLE_NAME, "Empty table name");
         }
 
         //step 2: get row count
         uint64_t row_count = 0;
         engine::Status stat = DBWrapper::DB()->GetTableRowCount(table_name_, row_count);
         if (!stat.ok()) {
-            error_code_ = SERVER_UNEXPECTED_ERROR;
-            error_msg_ = "Engine failed: " + stat.ToString();
-            SERVER_LOG_ERROR << error_msg_;
-            return error_code_;
+            return SetError(DB_META_TRANSACTION_FAILED, "Engine failed: " + stat.ToString());
         }
 
         row_count_ = (int64_t) row_count;
@@ -612,10 +564,7 @@ ServerError GetTableRowCountTask::OnExecute() {
         rc.Elapse("totally cost");
 
     } catch (std::exception& ex) {
-        error_code_ = SERVER_UNEXPECTED_ERROR;
-        error_msg_ = ex.what();
-        SERVER_LOG_ERROR << error_msg_;
-        return error_code_;
+        return SetError(SERVER_UNEXPECTED_ERROR, ex.what());
     }
 
     return SERVER_SUCCESS;

cpp/src/server/RequestTask.h

@@ -7,7 +7,6 @@
 
 #include "RequestScheduler.h"
 #include "utils/Error.h"
-#include "utils/AttributeSerializer.h"
 #include "db/Types.h"
 
 #include "milvus_types.h"

cpp/src/storage/s3/S3ClientWrapper.cpp

@@ -1,159 +1,159 @@
-#include "S3ClientWrapper.h"
-
-#include <aws/s3/model/CreateBucketRequest.h>
-#include <aws/s3/model/DeleteBucketRequest.h>
-#include <aws/s3/model/PutObjectRequest.h>
-#include <aws/s3/model/GetObjectRequest.h>
-#include <aws/s3/model/DeleteObjectRequest.h>
-
-#include <iostream>
-#include <fstream>
-
-
-namespace zilliz {
-namespace milvus {
-namespace engine {
-namespace storage {
-
-Status
-S3ClientWrapper::Create(const std::string &ip_address,
-                        const std::string &port,
-                        const std::string &access_key,
-                        const std::string &secret_key) {
-    Aws::InitAPI(options_);
-    Aws::Client::ClientConfiguration cfg;
-
-    // TODO: ip_address need to be validated.
-
-    cfg.endpointOverride = ip_address + ":" + port; // S3 server ip address and port
-    cfg.scheme = Aws::Http::Scheme::HTTP;
-    cfg.verifySSL =
-        false; //Aws::Auth::AWSCredentials cred("RPW421T9GSIO4A45Y9ZR", "2owKYy9emSS90Q0pXuyqpX1OxBCyEDYodsiBemcq"); // 认证的Key
-    client_ =
-        new S3Client(Aws::Auth::AWSCredentials(access_key, secret_key),
-                     cfg,
-                     Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Always,
-                     false);
-    if (client_ == nullptr) {
-        std::string error = "Can't connect server.";
-        return Status::Error(error);
-    } else {
-        return Status::OK();
-    }
-}
-
-
-Status
-S3ClientWrapper::Close() {
-    if (client_ != nullptr) {
-        delete client_;
-        client_ = nullptr;
-    }
-    Aws::ShutdownAPI(options_);
-    return Status::OK();
-}
-
-Status
-S3ClientWrapper::CreateBucket(std::string& bucket_name) {
-    Aws::S3::Model::CreateBucketRequest request;
-    request.SetBucket(bucket_name);
-
-    auto outcome = client_->CreateBucket(request);
-
-    if (outcome.IsSuccess())
-    {
-        return Status::OK();
-    }
-    else
-    {
-        std::cout << "CreateBucket error: "
-                  << outcome.GetError().GetExceptionName() << std::endl
-                  << outcome.GetError().GetMessage() << std::endl;
-        switch(outcome.GetError().GetErrorType()) {
-            case Aws::S3::S3Errors::BUCKET_ALREADY_EXISTS:
-            case Aws::S3::S3Errors::BUCKET_ALREADY_OWNED_BY_YOU:
-                return Status::AlreadyExist(outcome.GetError().GetMessage());
-            default:
-                return Status::Error(outcome.GetError().GetMessage());
-        }
-    }
-}
-
-Status
-S3ClientWrapper::DeleteBucket(std::string& bucket_name) {
-    Aws::S3::Model::DeleteBucketRequest bucket_request;
-    bucket_request.SetBucket(bucket_name);
-
-    auto outcome = client_->DeleteBucket(bucket_request);
-
-    if (outcome.IsSuccess())
-    {
-        return Status::OK();
-    }
-    else
-    {
-        std::cout << "DeleteBucket error: "
-                  << outcome.GetError().GetExceptionName() << " - "
-                  << outcome.GetError().GetMessage() << std::endl;
-        return Status::Error(outcome.GetError().GetMessage());
-    }
-}
-
-Status
-S3ClientWrapper::UploadFile(std::string &BucketName, std::string &objectKey, std::string &pathkey) {
-
-    PutObjectRequest putObjectRequest;
-    putObjectRequest.WithBucket(BucketName.c_str()).WithKey(objectKey.c_str());
-
-    auto input_data = Aws::MakeShared<Aws::FStream>("PutObjectInputStream",
-                                                    pathkey.c_str(),
-                                                    std::ios_base::in | std::ios_base::binary);
-    putObjectRequest.SetBody(input_data);
-    auto put_object_result = client_->PutObject(putObjectRequest);
-    if (put_object_result.IsSuccess()) {
-        return Status::OK();
-    } else {
-        std::cout << "PutObject error: " << put_object_result.GetError().GetExceptionName() << " "
-                  << put_object_result.GetError().GetMessage() << std::endl;
-        return Status::Error(put_object_result.GetError().GetMessage());
-    }
-}
-
-Status
-S3ClientWrapper::DownloadFile(std::string &BucketName, std::string &objectKey, std::string &pathkey) {
-    GetObjectRequest object_request;
-    object_request.WithBucket(BucketName.c_str()).WithKey(objectKey.c_str());
-    auto get_object_outcome = client_->GetObject(object_request);
-    if (get_object_outcome.IsSuccess()) {
-        Aws::OFStream local_file(pathkey.c_str(), std::ios::out | std::ios::binary);
-        local_file << get_object_outcome.GetResult().GetBody().rdbuf();
-        return Status::OK();
-    } else {
-        std::cout << "GetObject error: " << get_object_outcome.GetError().GetExceptionName() << " "
-                  << get_object_outcome.GetError().GetMessage() << std::endl;
-        return Status::Error(get_object_outcome.GetError().GetMessage());
-    }
-}
-
-Status
-S3ClientWrapper::DeleteFile(std::string &bucket_name, std::string &object_key) {
-    Aws::S3::Model::DeleteObjectRequest object_request;
-    object_request.WithBucket(bucket_name).WithKey(object_key);
-
-    auto delete_object_outcome = client_->DeleteObject(object_request);
-
-    if (delete_object_outcome.IsSuccess()) {
-        return Status::OK();
-    } else {
-        std::cout << "DeleteObject error: " <<
-                  delete_object_outcome.GetError().GetExceptionName() << " " <<
-                  delete_object_outcome.GetError().GetMessage() << std::endl;
-
-        return Status::Error(delete_object_outcome.GetError().GetMessage());
-    }
-}
-
-}
-}
-}
-}
+//#include "S3ClientWrapper.h"
+//
+//#include <aws/s3/model/CreateBucketRequest.h>
+//#include <aws/s3/model/DeleteBucketRequest.h>
+//#include <aws/s3/model/PutObjectRequest.h>
+//#include <aws/s3/model/GetObjectRequest.h>
+//#include <aws/s3/model/DeleteObjectRequest.h>
+//
+//#include <iostream>
+//#include <fstream>
+//
+//
+//namespace zilliz {
+//namespace milvus {
+//namespace engine {
+//namespace storage {
+//
+//Status
+//S3ClientWrapper::Create(const std::string &ip_address,
+//                        const std::string &port,
+//                        const std::string &access_key,
+//                        const std::string &secret_key) {
+//    Aws::InitAPI(options_);
+//    Aws::Client::ClientConfiguration cfg;
+//
+//    // TODO: ip_address need to be validated.
+//
+//    cfg.endpointOverride = ip_address + ":" + port; // S3 server ip address and port
+//    cfg.scheme = Aws::Http::Scheme::HTTP;
+//    cfg.verifySSL =
+//        false; //Aws::Auth::AWSCredentials cred("RPW421T9GSIO4A45Y9ZR", "2owKYy9emSS90Q0pXuyqpX1OxBCyEDYodsiBemcq"); // 认证的Key
+//    client_ =
+//        new S3Client(Aws::Auth::AWSCredentials(access_key, secret_key),
+//                     cfg,
+//                     Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Always,
+//                     false);
+//    if (client_ == nullptr) {
+//        std::string error = "Can't connect server.";
+//        return Status::Error(error);
+//    } else {
+//        return Status::OK();
+//    }
+//}
+//
+//
+//Status
+//S3ClientWrapper::Close() {
+//    if (client_ != nullptr) {
+//        delete client_;
+//        client_ = nullptr;
+//    }
+//    Aws::ShutdownAPI(options_);
+//    return Status::OK();
+//}
+//
+//Status
+//S3ClientWrapper::CreateBucket(std::string& bucket_name) {
+//    Aws::S3::Model::CreateBucketRequest request;
+//    request.SetBucket(bucket_name);
+//
+//    auto outcome = client_->CreateBucket(request);
+//
+//    if (outcome.IsSuccess())
+//    {
+//        return Status::OK();
+//    }
+//    else
+//    {
+//        std::cout << "CreateBucket error: "
+//                  << outcome.GetError().GetExceptionName() << std::endl
+//                  << outcome.GetError().GetMessage() << std::endl;
+//        switch(outcome.GetError().GetErrorType()) {
+//            case Aws::S3::S3Errors::BUCKET_ALREADY_EXISTS:
+//            case Aws::S3::S3Errors::BUCKET_ALREADY_OWNED_BY_YOU:
+//                return Status::AlreadyExist(outcome.GetError().GetMessage());
+//            default:
+//                return Status::Error(outcome.GetError().GetMessage());
+//        }
+//    }
+//}
+//
+//Status
+//S3ClientWrapper::DeleteBucket(std::string& bucket_name) {
+//    Aws::S3::Model::DeleteBucketRequest bucket_request;
+//    bucket_request.SetBucket(bucket_name);
+//
+//    auto outcome = client_->DeleteBucket(bucket_request);
+//
+//    if (outcome.IsSuccess())
+//    {
+//        return Status::OK();
+//    }
+//    else
+//    {
+//        std::cout << "DeleteBucket error: "
+//                  << outcome.GetError().GetExceptionName() << " - "
+//                  << outcome.GetError().GetMessage() << std::endl;
+//        return Status::Error(outcome.GetError().GetMessage());
+//    }
+//}
+//
+//Status
+//S3ClientWrapper::UploadFile(std::string &BucketName, std::string &objectKey, std::string &pathkey) {
+//
+//    PutObjectRequest putObjectRequest;
+//    putObjectRequest.WithBucket(BucketName.c_str()).WithKey(objectKey.c_str());
+//
+//    auto input_data = Aws::MakeShared<Aws::FStream>("PutObjectInputStream",
+//                                                    pathkey.c_str(),
+//                                                    std::ios_base::in | std::ios_base::binary);
+//    putObjectRequest.SetBody(input_data);
+//    auto put_object_result = client_->PutObject(putObjectRequest);
+//    if (put_object_result.IsSuccess()) {
+//        return Status::OK();
+//    } else {
+//        std::cout << "PutObject error: " << put_object_result.GetError().GetExceptionName() << " "
+//                  << put_object_result.GetError().GetMessage() << std::endl;
+//        return Status::Error(put_object_result.GetError().GetMessage());
+//    }
+//}
+//
+//Status
+//S3ClientWrapper::DownloadFile(std::string &BucketName, std::string &objectKey, std::string &pathkey) {
+//    GetObjectRequest object_request;
+//    object_request.WithBucket(BucketName.c_str()).WithKey(objectKey.c_str());
+//    auto get_object_outcome = client_->GetObject(object_request);
+//    if (get_object_outcome.IsSuccess()) {
+//        Aws::OFStream local_file(pathkey.c_str(), std::ios::out | std::ios::binary);
+//        local_file << get_object_outcome.GetResult().GetBody().rdbuf();
+//        return Status::OK();
+//    } else {
+//        std::cout << "GetObject error: " << get_object_outcome.GetError().GetExceptionName() << " "
+//                  << get_object_outcome.GetError().GetMessage() << std::endl;
+//        return Status::Error(get_object_outcome.GetError().GetMessage());
+//    }
+//}
+//
+//Status
+//S3ClientWrapper::DeleteFile(std::string &bucket_name, std::string &object_key) {
+//    Aws::S3::Model::DeleteObjectRequest object_request;
+//    object_request.WithBucket(bucket_name).WithKey(object_key);
+//
+//    auto delete_object_outcome = client_->DeleteObject(object_request);
+//
+//    if (delete_object_outcome.IsSuccess()) {
+//        return Status::OK();
+//    } else {
+//        std::cout << "DeleteObject error: " <<
+//                  delete_object_outcome.GetError().GetExceptionName() << " " <<
+//                  delete_object_outcome.GetError().GetMessage() << std::endl;
+//
+//        return Status::Error(delete_object_outcome.GetError().GetMessage());
+//    }
+//}
+//
+//}
+//}
+//}
+//}

cpp/src/storage/s3/S3ClientWrapper.h

@@ -1,45 +1,45 @@
-#pragma once
-
-#include "storage/IStorage.h"
-
-
-#include <aws/s3/S3Client.h>
-#include <aws/core/Aws.h>
-#include <aws/core/auth/AWSCredentialsProvider.h>
-
-
-using namespace Aws::S3;
-using namespace Aws::S3::Model;
-
-namespace zilliz {
-namespace milvus {
-namespace engine {
-namespace storage {
-
-class S3ClientWrapper : public IStorage {
- public:
-
-    S3ClientWrapper() = default;
-    ~S3ClientWrapper() = default;
-
-    Status Create(const std::string &ip_address,
-                const std::string &port,
-                const std::string &access_key,
-                const std::string &secret_key) override;
-    Status Close() override;
-
-    Status CreateBucket(std::string& bucket_name) override;
-    Status DeleteBucket(std::string& bucket_name) override;
-    Status UploadFile(std::string &BucketName, std::string &objectKey, std::string &pathkey) override;
-    Status DownloadFile(std::string &BucketName, std::string &objectKey, std::string &pathkey) override;
-    Status DeleteFile(std::string &bucket_name, std::string &object_key) override;
-
- private:
-    S3Client *client_ = nullptr;
-    Aws::SDKOptions options_;
-};
-
-}
-}
-}
-}
+//#pragma once
+//
+//#include "storage/IStorage.h"
+//
+//
+//#include <aws/s3/S3Client.h>
+//#include <aws/core/Aws.h>
+//#include <aws/core/auth/AWSCredentialsProvider.h>
+//
+//
+//using namespace Aws::S3;
+//using namespace Aws::S3::Model;
+//
+//namespace zilliz {
+//namespace milvus {
+//namespace engine {
+//namespace storage {
+//
+//class S3ClientWrapper : public IStorage {
+// public:
+//
+//    S3ClientWrapper() = default;
+//    ~S3ClientWrapper() = default;
+//
+//    Status Create(const std::string &ip_address,
+//                const std::string &port,
+//                const std::string &access_key,
+//                const std::string &secret_key) override;
+//    Status Close() override;
+//
+//    Status CreateBucket(std::string& bucket_name) override;
+//    Status DeleteBucket(std::string& bucket_name) override;
+//    Status UploadFile(std::string &BucketName, std::string &objectKey, std::string &pathkey) override;
+//    Status DownloadFile(std::string &BucketName, std::string &objectKey, std::string &pathkey) override;
+//    Status DeleteFile(std::string &bucket_name, std::string &object_key) override;
+//
+// private:
+//    S3Client *client_ = nullptr;
+//    Aws::SDKOptions options_;
+//};
+//
+//}
+//}
+//}
+//}

cpp/src/thrift/gen-cpp/milvus_types.cpp

@@ -15,23 +15,51 @@ namespace milvus { namespace thrift {
 
 int _kErrorCodeValues[] = {
   ErrorCode::SUCCESS,
+  ErrorCode::UNEXPECTED_ERROR,
   ErrorCode::CONNECT_FAILED,
   ErrorCode::PERMISSION_DENIED,
   ErrorCode::TABLE_NOT_EXISTS,
   ErrorCode::ILLEGAL_ARGUMENT,
   ErrorCode::ILLEGAL_RANGE,
-  ErrorCode::ILLEGAL_DIMENSION
+  ErrorCode::ILLEGAL_DIMENSION,
+  ErrorCode::ILLEGAL_INDEX_TYPE,
+  ErrorCode::ILLEGAL_TABLE_NAME,
+  ErrorCode::ILLEGAL_TOPK,
+  ErrorCode::ILLEGAL_ROWRECORD,
+  ErrorCode::ILLEGAL_VECTOR_ID,
+  ErrorCode::ILLEGAL_SEARCH_RESULT,
+  ErrorCode::FILE_NOT_FOUND,
+  ErrorCode::META_FAILED,
+  ErrorCode::CACHE_FAILED,
+  ErrorCode::CANNOT_CREATE_FOLDER,
+  ErrorCode::CANNOT_CREATE_FILE,
+  ErrorCode::CANNOT_DELETE_FOLDER,
+  ErrorCode::CANNOT_DELETE_FILE
 };
 const char* _kErrorCodeNames[] = {
   "SUCCESS",
+  "UNEXPECTED_ERROR",
   "CONNECT_FAILED",
   "PERMISSION_DENIED",
   "TABLE_NOT_EXISTS",
   "ILLEGAL_ARGUMENT",
   "ILLEGAL_RANGE",
-  "ILLEGAL_DIMENSION"
+  "ILLEGAL_DIMENSION",
+  "ILLEGAL_INDEX_TYPE",
+  "ILLEGAL_TABLE_NAME",
+  "ILLEGAL_TOPK",
+  "ILLEGAL_ROWRECORD",
+  "ILLEGAL_VECTOR_ID",
+  "ILLEGAL_SEARCH_RESULT",
+  "FILE_NOT_FOUND",
+  "META_FAILED",
+  "CACHE_FAILED",
+  "CANNOT_CREATE_FOLDER",
+  "CANNOT_CREATE_FILE",
+  "CANNOT_DELETE_FOLDER",
+  "CANNOT_DELETE_FILE"
 };
-const std::map<int, const char*> _ErrorCode_VALUES_TO_NAMES(::apache::thrift::TEnumIterator(7, _kErrorCodeValues, _kErrorCodeNames), ::apache::thrift::TEnumIterator(-1, NULL, NULL));
+const std::map<int, const char*> _ErrorCode_VALUES_TO_NAMES(::apache::thrift::TEnumIterator(21, _kErrorCodeValues, _kErrorCodeNames), ::apache::thrift::TEnumIterator(-1, NULL, NULL));
 
 std::ostream& operator<<(std::ostream& out, const ErrorCode::type& val) {
   std::map<int, const char*>::const_iterator it = _ErrorCode_VALUES_TO_NAMES.find(val);

cpp/src/thrift/gen-cpp/milvus_types.h

@@ -23,12 +23,26 @@ namespace milvus { namespace thrift {
 struct ErrorCode {
   enum type {
     SUCCESS = 0,
-    CONNECT_FAILED = 1,
-    PERMISSION_DENIED = 2,
-    TABLE_NOT_EXISTS = 3,
-    ILLEGAL_ARGUMENT = 4,
-    ILLEGAL_RANGE = 5,
-    ILLEGAL_DIMENSION = 6
+    UNEXPECTED_ERROR = 1,
+    CONNECT_FAILED = 2,
+    PERMISSION_DENIED = 3,
+    TABLE_NOT_EXISTS = 4,
+    ILLEGAL_ARGUMENT = 5,
+    ILLEGAL_RANGE = 6,
+    ILLEGAL_DIMENSION = 7,
+    ILLEGAL_INDEX_TYPE = 8,
+    ILLEGAL_TABLE_NAME = 9,
+    ILLEGAL_TOPK = 10,
+    ILLEGAL_ROWRECORD = 11,
+    ILLEGAL_VECTOR_ID = 12,
+    ILLEGAL_SEARCH_RESULT = 13,
+    FILE_NOT_FOUND = 14,
+    META_FAILED = 15,
+    CACHE_FAILED = 16,
+    CANNOT_CREATE_FOLDER = 17,
+    CANNOT_CREATE_FILE = 18,
+    CANNOT_DELETE_FOLDER = 19,
+    CANNOT_DELETE_FILE = 20
   };
 };
 

cpp/src/thrift/milvus.thrift

@@ -15,12 +15,26 @@ namespace netcore milvus.thrift
 
 enum ErrorCode {
     SUCCESS = 0,
+    UNEXPECTED_ERROR,
     CONNECT_FAILED,
     PERMISSION_DENIED,
     TABLE_NOT_EXISTS,
     ILLEGAL_ARGUMENT,
     ILLEGAL_RANGE,
     ILLEGAL_DIMENSION,
+    ILLEGAL_INDEX_TYPE,
+    ILLEGAL_TABLE_NAME,
+    ILLEGAL_TOPK,
+    ILLEGAL_ROWRECORD,
+    ILLEGAL_VECTOR_ID,
+    ILLEGAL_SEARCH_RESULT,
+    FILE_NOT_FOUND,
+    META_FAILED,
+    CACHE_FAILED,
+    CANNOT_CREATE_FOLDER,
+    CANNOT_CREATE_FILE,
+    CANNOT_DELETE_FOLDER,
+    CANNOT_DELETE_FILE,
 }
 
 exception Exception {

cpp/src/utils/AttributeSerializer.cpp

@@ -1,50 +0,0 @@
-/*******************************************************************************
- * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved
- * Unauthorized copying of this file, via any medium is strictly prohibited.
- * Proprietary and confidential.
- ******************************************************************************/
-
-#include "AttributeSerializer.h"
-#include "StringHelpFunctions.h"
-
-namespace zilliz {
-namespace milvus {
-namespace server {
-
-
-ServerError AttributeSerializer::Encode(const AttribMap& attrib_map, std::string& attrib_str) {
-    attrib_str = "";
-    for(auto iter : attrib_map) {
-        attrib_str += iter.first;
-        attrib_str += ":\"";
-        attrib_str += iter.second;
-        attrib_str += "\";";
-    }
-
-    return SERVER_SUCCESS;
-}
-
-ServerError AttributeSerializer::Decode(const std::string& attrib_str, AttribMap& attrib_map) {
-    attrib_map.clear();
-
-    std::vector<std::string> kv_pairs;
-    StringHelpFunctions::SplitStringByQuote(attrib_str, ";", "\"", kv_pairs);
-    for(std::string& str : kv_pairs) {
-        std::string key, val;
-        size_t index = str.find_first_of(":", 0);
-        if (index != std::string::npos) {
-            key = str.substr(0, index);
-            val = str.substr(index + 1);
-        } else {
-            key = str;
-        }
-
-        attrib_map.insert(std::make_pair(key, val));
-    }
-
-    return SERVER_SUCCESS;
-}
-
-}
-}
-}

cpp/src/utils/AttributeSerializer.h

@@ -1,27 +0,0 @@
-/*******************************************************************************
- * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved
- * Unauthorized copying of this file, via any medium is strictly prohibited.
- * Proprietary and confidential.
- ******************************************************************************/
-#pragma once
-
-#include <map>
-
-#include "Error.h"
-
-namespace zilliz {
-namespace milvus {
-namespace server {
-
-using AttribMap = std::map<std::string, std::string>;
-
-class AttributeSerializer {
-public:
-    static ServerError Encode(const AttribMap& attrib_map, std::string& attrib_str);
-    static ServerError Decode(const std::string& attrib_str, AttribMap& attrib_map);
-};
-
-
-}
-}
-}

cpp/src/utils/CommonUtil.cpp

@@ -51,7 +51,7 @@ bool CommonUtil::GetSystemAvailableThreads(unsigned int &threadCnt) {
     return true;
 }
 
-bool CommonUtil::IsDirectoryExit(const std::string &path)
+bool CommonUtil::IsDirectoryExist(const std::string &path)
 {
     DIR *dp = nullptr;
     if ((dp = opendir(path.c_str())) == nullptr) {
@@ -182,7 +182,7 @@ void CommonUtil::ConvertTime(time_t time_integer, tm &time_struct) {
     memcpy(&time_struct, t_m, sizeof(tm));
 }
 
-void ConvertTime(tm time_struct, time_t &time_integer) {
+void CommonUtil::ConvertTime(tm time_struct, time_t &time_integer) {
     time_integer = mktime(&time_struct);
 }
 

cpp/src/utils/CommonUtil.h

@@ -20,7 +20,7 @@ class CommonUtil {
     static bool GetSystemAvailableThreads(unsigned int &threadCnt);
 
     static bool IsFileExist(const std::string &path);
-    static bool IsDirectoryExit(const std::string &path);
+    static bool IsDirectoryExist(const std::string &path);
     static ServerError CreateDirectory(const std::string &path);
     static ServerError DeleteDirectory(const std::string &path);
 

cpp/src/utils/Error.h

@@ -24,18 +24,35 @@ ToGlobalServerErrorCode(const ServerError error_code) {
     return SERVER_ERROR_CODE_BASE + error_code;
 }
 
-constexpr ServerError SERVER_UNEXPECTED_ERROR = ToGlobalServerErrorCode(0x001);
-constexpr ServerError SERVER_UNSUPPORTED_ERROR = ToGlobalServerErrorCode(0x002);
-constexpr ServerError SERVER_NULL_POINTER = ToGlobalServerErrorCode(0x003);
-constexpr ServerError SERVER_INVALID_ARGUMENT = ToGlobalServerErrorCode(0x004);
-constexpr ServerError SERVER_FILE_NOT_FOUND = ToGlobalServerErrorCode(0x005);
-constexpr ServerError SERVER_NOT_IMPLEMENT = ToGlobalServerErrorCode(0x006);
-constexpr ServerError SERVER_BLOCKING_QUEUE_EMPTY = ToGlobalServerErrorCode(0x007);
-constexpr ServerError SERVER_TABLE_NOT_EXIST = ToGlobalServerErrorCode(0x008);
-constexpr ServerError SERVER_INVALID_TIME_RANGE = ToGlobalServerErrorCode(0x009);
-constexpr ServerError SERVER_INVALID_VECTOR_DIMENSION = ToGlobalServerErrorCode(0x00a);
-constexpr ServerError SERVER_LICENSE_VALIDATION_FAIL = ToGlobalServerErrorCode(0x00b);
-constexpr ServerError SERVER_LICENSE_FILE_NOT_EXIST = ToGlobalServerErrorCode(0x00c);
+constexpr ServerError SERVER_UNEXPECTED_ERROR = ToGlobalServerErrorCode(1);
+constexpr ServerError SERVER_UNSUPPORTED_ERROR = ToGlobalServerErrorCode(2);
+constexpr ServerError SERVER_NULL_POINTER = ToGlobalServerErrorCode(3);
+constexpr ServerError SERVER_INVALID_ARGUMENT = ToGlobalServerErrorCode(4);
+constexpr ServerError SERVER_FILE_NOT_FOUND = ToGlobalServerErrorCode(5);
+constexpr ServerError SERVER_NOT_IMPLEMENT = ToGlobalServerErrorCode(6);
+constexpr ServerError SERVER_BLOCKING_QUEUE_EMPTY = ToGlobalServerErrorCode(7);
+constexpr ServerError SERVER_CANNOT_CREATE_FOLDER = ToGlobalServerErrorCode(8);
+constexpr ServerError SERVER_CANNOT_CREATE_FILE = ToGlobalServerErrorCode(9);
+constexpr ServerError SERVER_CANNOT_DELETE_FOLDER = ToGlobalServerErrorCode(10);
+constexpr ServerError SERVER_CANNOT_DELETE_FILE = ToGlobalServerErrorCode(11);
+
+constexpr ServerError SERVER_TABLE_NOT_EXIST = ToGlobalServerErrorCode(100);
+constexpr ServerError SERVER_INVALID_TABLE_NAME = ToGlobalServerErrorCode(101);
+constexpr ServerError SERVER_INVALID_TABLE_DIMENSION = ToGlobalServerErrorCode(102);
+constexpr ServerError SERVER_INVALID_TIME_RANGE = ToGlobalServerErrorCode(103);
+constexpr ServerError SERVER_INVALID_VECTOR_DIMENSION = ToGlobalServerErrorCode(104);
+constexpr ServerError SERVER_INVALID_INDEX_TYPE = ToGlobalServerErrorCode(105);
+constexpr ServerError SERVER_INVALID_ROWRECORD = ToGlobalServerErrorCode(106);
+constexpr ServerError SERVER_INVALID_ROWRECORD_ARRAY = ToGlobalServerErrorCode(107);
+constexpr ServerError SERVER_INVALID_TOPK = ToGlobalServerErrorCode(108);
+constexpr ServerError SERVER_ILLEGAL_VECTOR_ID = ToGlobalServerErrorCode(109);
+constexpr ServerError SERVER_ILLEGAL_SEARCH_RESULT = ToGlobalServerErrorCode(110);
+constexpr ServerError SERVER_CACHE_ERROR = ToGlobalServerErrorCode(111);
+
+constexpr ServerError SERVER_LICENSE_FILE_NOT_EXIST = ToGlobalServerErrorCode(500);
+constexpr ServerError SERVER_LICENSE_VALIDATION_FAIL = ToGlobalServerErrorCode(501);
+
+constexpr ServerError DB_META_TRANSACTION_FAILED = ToGlobalServerErrorCode(1000);
 
 class ServerException : public std::exception {
 public:

cpp/src/utils/StringHelpFunctions.cpp

@@ -9,13 +9,6 @@ namespace zilliz {
 namespace milvus {
 namespace server {
 
-void StringHelpFunctions::TrimStringLineBreak(std::string &string) {
-    if (!string.empty()) {
-        static std::string s_format("\n\r");
-        string.erase(string.find_last_not_of(s_format) + 1);
-    }
-}
-
 void StringHelpFunctions::TrimStringBlank(std::string &string) {
     if (!string.empty()) {
         static std::string s_format(" \n\r\t");

cpp/src/utils/StringHelpFunctions.h

@@ -18,8 +18,6 @@ private:
     StringHelpFunctions() = default;
 
 public:
-    static void TrimStringLineBreak(std::string &string);
-
     static void TrimStringBlank(std::string &string);
 
     static void TrimStringQuote(std::string &string, const std::string &qoute);

cpp/unittest/CMakeLists.txt

@@ -46,6 +46,6 @@ set(unittest_libs
 add_subdirectory(server)
 add_subdirectory(db)
 add_subdirectory(faiss_wrapper)
-add_subdirectory(license)
+#add_subdirectory(license)
 add_subdirectory(metrics)
 add_subdirectory(storage)

cpp/unittest/db/meta_tests.cpp

@@ -17,39 +17,39 @@
 
 using namespace zilliz::milvus::engine;
 
-TEST_F(MetaTest, GROUP_TEST) {
-    auto table_id = "meta_test_group";
+TEST_F(MetaTest, TABLE_TEST) {
+    auto table_id = "meta_test_table";
 
-    meta::TableSchema group;
-    group.table_id_ = table_id;
-    auto status = impl_->CreateTable(group);
+    meta::TableSchema table;
+    table.table_id_ = table_id;
+    auto status = impl_->CreateTable(table);
     ASSERT_TRUE(status.ok());
 
-    auto gid = group.id_;
-    group.id_ = -1;
-    status = impl_->DescribeTable(group);
+    auto gid = table.id_;
+    table.id_ = -1;
+    status = impl_->DescribeTable(table);
     ASSERT_TRUE(status.ok());
-    ASSERT_EQ(group.id_, gid);
-    ASSERT_EQ(group.table_id_, table_id);
+    ASSERT_EQ(table.id_, gid);
+    ASSERT_EQ(table.table_id_, table_id);
 
-    group.table_id_ = "not_found";
-    status = impl_->DescribeTable(group);
+    table.table_id_ = "not_found";
+    status = impl_->DescribeTable(table);
     ASSERT_TRUE(!status.ok());
 
-    group.table_id_ = table_id;
-    status = impl_->CreateTable(group);
-    ASSERT_TRUE(!status.ok());
+    table.table_id_ = table_id;
+    status = impl_->CreateTable(table);
+    ASSERT_TRUE(status.ok());
 }
 
-TEST_F(MetaTest, table_file_TEST) {
-    auto table_id = "meta_test_group";
+TEST_F(MetaTest, TABLE_FILE_TEST) {
+    auto table_id = "meta_test_table";
 
-    meta::TableSchema group;
-    group.table_id_ = table_id;
-    auto status = impl_->CreateTable(group);
+    meta::TableSchema table;
+    table.table_id_ = table_id;
+    auto status = impl_->CreateTable(table);
 
     meta::TableFileSchema table_file;
-    table_file.table_id_ = group.table_id_;
+    table_file.table_id_ = table.table_id_;
     status = impl_->CreateTableFile(table_file);
     ASSERT_TRUE(status.ok());
     ASSERT_EQ(table_file.file_type_, meta::TableFileSchema::NEW);
@@ -104,15 +104,15 @@ TEST_F(MetaTest, ARCHIVE_TEST_DAYS) {
     options.archive_conf = ArchiveConf("delete", ss.str());
 
     auto impl = meta::DBMetaImpl(options);
-    auto table_id = "meta_test_group";
+    auto table_id = "meta_test_table";
 
-    meta::TableSchema group;
-    group.table_id_ = table_id;
-    auto status = impl.CreateTable(group);
+    meta::TableSchema table;
+    table.table_id_ = table_id;
+    auto status = impl.CreateTable(table);
 
     meta::TableFilesSchema files;
     meta::TableFileSchema table_file;
-    table_file.table_id_ = group.table_id_;
+    table_file.table_id_ = table.table_id_;
 
     auto cnt = 100;
     long ts = utils::GetMicroSecTimeStamp();
@@ -156,13 +156,13 @@ TEST_F(MetaTest, ARCHIVE_TEST_DISK) {
     auto impl = meta::DBMetaImpl(options);
     auto table_id = "meta_test_group";
 
-    meta::TableSchema group;
-    group.table_id_ = table_id;
-    auto status = impl.CreateTable(group);
+    meta::TableSchema table;
+    table.table_id_ = table_id;
+    auto status = impl.CreateTable(table);
 
     meta::TableFilesSchema files;
     meta::TableFileSchema table_file;
-    table_file.table_id_ = group.table_id_;
+    table_file.table_id_ = table.table_id_;
 
     auto cnt = 10;
     auto each_size = 2UL;
@@ -198,9 +198,9 @@ TEST_F(MetaTest, ARCHIVE_TEST_DISK) {
 TEST_F(MetaTest, TABLE_FILES_TEST) {
     auto table_id = "meta_test_group";
 
-    meta::TableSchema group;
-    group.table_id_ = table_id;
-    auto status = impl_->CreateTable(group);
+    meta::TableSchema table;
+    table.table_id_ = table_id;
+    auto status = impl_->CreateTable(table);
 
     int new_files_cnt = 4;
     int raw_files_cnt = 5;
@@ -208,7 +208,7 @@ TEST_F(MetaTest, TABLE_FILES_TEST) {
     int index_files_cnt = 7;
 
     meta::TableFileSchema table_file;
-    table_file.table_id_ = group.table_id_;
+    table_file.table_id_ = table.table_id_;
 
     for (auto i=0; i<new_files_cnt; ++i) {
         status = impl_->CreateTableFile(table_file);
@@ -241,7 +241,7 @@ TEST_F(MetaTest, TABLE_FILES_TEST) {
     ASSERT_EQ(files.size(), to_index_files_cnt);
 
     meta::DatePartionedTableFilesSchema dated_files;
-    status = impl_->FilesToMerge(group.table_id_, dated_files);
+    status = impl_->FilesToMerge(table.table_id_, dated_files);
     ASSERT_TRUE(status.ok());
     ASSERT_EQ(dated_files[table_file.date_].size(), raw_files_cnt);
 

cpp/unittest/license/license_check_test.cpp

@@ -1,184 +1,184 @@
+////
+//// Created by zilliz on 19-5-13.
+////
 //
-// Created by zilliz on 19-5-13.
+//#include "utils/Log.h"
+//#include "license/LicenseCheck.h"
+//#include "utils/Error.h"
+//
+//#include <gtest/gtest.h>
+//#include <iostream>
+//
+//
+//using namespace zilliz::milvus;
+//
+//TEST(LicenseLibraryTest, CHECK_TEST) {
+//
+//
+//    server::ServerError err;
+//    // 1. Set license file name
+//    std::string license_file_path("/tmp/milvus/abc.license");
+//
+//    // 2. Legality check
+//    err = server::LicenseCheck::LegalityCheck(license_file_path);
+//    ASSERT_EQ(err, server::SERVER_SUCCESS);
+//
+//}
+//
+//TEST(LicenseLibraryTest, CHECK_ERROR1_TEST){
+//
+//    server::ServerError err;
+//    // 1. Set license file name
+//    std::string license_file_path("/tmp/milvus/abc");
+//
+//    // 2. Legality check
+//    err = server::LicenseCheck::LegalityCheck(license_file_path);
+//    ASSERT_EQ(err, server::SERVER_SUCCESS);
+//}
+//
+//TEST(LicenseLibraryTest, CHECK_ERROR2_TEST){
+//
+//    server::ServerError err;
+//    // 1. Set license file name
+//    std::string license_file_path("/tmp/milvus/abc.license");
+//
+//    // 2. Define output var
+//    int device_count;
+//    std::map<int, std::string> uuid_encryption_map;
+//    time_t starting_time;
+//    time_t end_time;
+//
+//    // 3. Read License File
+//    err = server::LicenseLibrary::LicenseFileDeserialization(license_file_path,
+//                                                             device_count,
+//                                                             uuid_encryption_map,
+//                                                             starting_time,
+//                                                             end_time);
+//    ASSERT_EQ(err, server::SERVER_SUCCESS);
+//
+//    // 4. Change device count
+//    ++device_count;
+//    err = server::LicenseLibrary::LicenseFileSerialization(license_file_path,
+//                                                           device_count,
+//                                                           uuid_encryption_map,
+//                                                           starting_time,
+//                                                           end_time);
+//    ASSERT_EQ(err, server::SERVER_SUCCESS);
+//
+//    // 5. Legality check
+//    err = server::LicenseCheck::LegalityCheck(license_file_path);
+//    ASSERT_EQ(err, server::SERVER_SUCCESS);
+//}
+//
+//TEST(LicenseLibraryTest, CHECK_ERROR3_TEST){
+//
+//    server::ServerError err;
+//    // 1. Set license file name
+//    std::string license_file_path("/tmp/milvus/abc.license");
+//
+//    // 2. Define output var
+//    int device_count;
+//    std::map<int, std::string> uuid_encryption_map;
+//    time_t starting_time;
+//    time_t end_time;
+//
+//    // 3. Read License File
+//    err = server::LicenseLibrary::LicenseFileDeserialization(license_file_path,
+//                                                             device_count,
+//                                                             uuid_encryption_map,
+//                                                             starting_time,
+//                                                             end_time);
+//    ASSERT_EQ(err, server::SERVER_SUCCESS);
+//
+//    // 4. Change device count
+//    if(device_count) uuid_encryption_map[0]+="u";
+//    err = server::LicenseLibrary::LicenseFileSerialization(license_file_path,
+//                                                           device_count,
+//                                                           uuid_encryption_map,
+//                                                           starting_time,
+//                                                           end_time);
+//    ASSERT_EQ(err, server::SERVER_SUCCESS);
+//
+//    // 5. Legality check
+//    err = server::LicenseCheck::LegalityCheck(license_file_path);
+//    ASSERT_EQ(err, server::SERVER_SUCCESS);
+//}
+//
+//TEST(LicenseLibraryTest, CHECK_ERROR4_1_TEST){
+//
+//    server::ServerError err;
+//    // 1. Set license file name
+//    std::string license_file_path("/tmp/milvus/abc.license");
+//
+//    // 2. Define output var
+//    int device_count;
+//    std::map<int, std::string> uuid_encryption_map;
+//    time_t starting_time;
+//    time_t end_time;
+//
+//    // 3. Read License File
+//    err = server::LicenseLibrary::LicenseFileDeserialization(license_file_path,
+//                                                             device_count,
+//                                                             uuid_encryption_map,
+//                                                             starting_time,
+//                                                             end_time);
+//    ASSERT_EQ(err, server::SERVER_SUCCESS);
+//
+//    // 4. Change starting time
+//    time_t system_time;
+//    err = server::LicenseLibrary::GetSystemTime(system_time);
+//    ASSERT_EQ(err, server::SERVER_SUCCESS);
+//
+//    system_time+=60*60*24;
+//
+//    err = server::LicenseLibrary::LicenseFileSerialization(license_file_path,
+//                                                           device_count,
+//                                                           uuid_encryption_map,
+//                                                           system_time,
+//                                                           end_time);
+//    ASSERT_EQ(err, server::SERVER_SUCCESS);
+//
+//    // 5. Legality check
+//    err = server::LicenseCheck::LegalityCheck(license_file_path);
+//    ASSERT_EQ(err, server::SERVER_SUCCESS);
+//}
+//
+//TEST(LicenseLibraryTest, CHECK_ERROR4_2_TEST){
+//
+//    server::ServerError err;
+//    // 1. Set license file name
+//    std::string license_file_path("/tmp/milvus/abc.license");
+//
+//    // 2. Define output var
+//    int device_count;
+//    std::map<int, std::string> uuid_encryption_map;
+//    time_t starting_time;
+//    time_t end_time;
+//
+//    // 3. Read License File
+//    err = server::LicenseLibrary::LicenseFileDeserialization(license_file_path,
+//                                                             device_count,
+//                                                             uuid_encryption_map,
+//                                                             starting_time,
+//                                                             end_time);
+//    ASSERT_EQ(err, server::SERVER_SUCCESS);
+//
+//    // 4. Change end time
+//    time_t system_time;
+//    err = server::LicenseLibrary::GetSystemTime(system_time);
+//    ASSERT_EQ(err, server::SERVER_SUCCESS);
+//
+//    system_time-=100;
+//
+//    err = server::LicenseLibrary::LicenseFileSerialization(license_file_path,
+//                                                           device_count,
+//                                                           uuid_encryption_map,
+//                                                           starting_time,
+//                                                           system_time);
+//    ASSERT_EQ(err, server::SERVER_SUCCESS);
+//
+//    // 5. Legality check
+//    err = server::LicenseCheck::LegalityCheck(license_file_path);
+//    ASSERT_EQ(err, server::SERVER_SUCCESS);
+//}
 //
-
-#include "utils/Log.h"
-#include "license/LicenseCheck.h"
-#include "utils/Error.h"
-
-#include <gtest/gtest.h>
-#include <iostream>
-
-
-using namespace zilliz::milvus;
-
-TEST(LicenseLibraryTest, CHECK_TEST) {
-
-
-    server::ServerError err;
-    // 1. Set license file name
-    std::string license_file_path("/tmp/milvus/abc.license");
-
-    // 2. Legality check
-    err = server::LicenseCheck::LegalityCheck(license_file_path);
-    ASSERT_EQ(err, server::SERVER_SUCCESS);
-
-}
-
-TEST(LicenseLibraryTest, CHECK_ERROR1_TEST){
-
-    server::ServerError err;
-    // 1. Set license file name
-    std::string license_file_path("/tmp/milvus/abc");
-
-    // 2. Legality check
-    err = server::LicenseCheck::LegalityCheck(license_file_path);
-    ASSERT_EQ(err, server::SERVER_SUCCESS);
-}
-
-TEST(LicenseLibraryTest, CHECK_ERROR2_TEST){
-
-    server::ServerError err;
-    // 1. Set license file name
-    std::string license_file_path("/tmp/milvus/abc.license");
-
-    // 2. Define output var
-    int device_count;
-    std::map<int, std::string> uuid_encryption_map;
-    time_t starting_time;
-    time_t end_time;
-
-    // 3. Read License File
-    err = server::LicenseLibrary::LicenseFileDeserialization(license_file_path,
-                                                             device_count,
-                                                             uuid_encryption_map,
-                                                             starting_time,
-                                                             end_time);
-    ASSERT_EQ(err, server::SERVER_SUCCESS);
-
-    // 4. Change device count
-    ++device_count;
-    err = server::LicenseLibrary::LicenseFileSerialization(license_file_path,
-                                                           device_count,
-                                                           uuid_encryption_map,
-                                                           starting_time,
-                                                           end_time);
-    ASSERT_EQ(err, server::SERVER_SUCCESS);
-
-    // 5. Legality check
-    err = server::LicenseCheck::LegalityCheck(license_file_path);
-    ASSERT_EQ(err, server::SERVER_SUCCESS);
-}
-
-TEST(LicenseLibraryTest, CHECK_ERROR3_TEST){
-
-    server::ServerError err;
-    // 1. Set license file name
-    std::string license_file_path("/tmp/milvus/abc.license");
-
-    // 2. Define output var
-    int device_count;
-    std::map<int, std::string> uuid_encryption_map;
-    time_t starting_time;
-    time_t end_time;
-
-    // 3. Read License File
-    err = server::LicenseLibrary::LicenseFileDeserialization(license_file_path,
-                                                             device_count,
-                                                             uuid_encryption_map,
-                                                             starting_time,
-                                                             end_time);
-    ASSERT_EQ(err, server::SERVER_SUCCESS);
-
-    // 4. Change device count
-    if(device_count) uuid_encryption_map[0]+="u";
-    err = server::LicenseLibrary::LicenseFileSerialization(license_file_path,
-                                                           device_count,
-                                                           uuid_encryption_map,
-                                                           starting_time,
-                                                           end_time);
-    ASSERT_EQ(err, server::SERVER_SUCCESS);
-
-    // 5. Legality check
-    err = server::LicenseCheck::LegalityCheck(license_file_path);
-    ASSERT_EQ(err, server::SERVER_SUCCESS);
-}
-
-TEST(LicenseLibraryTest, CHECK_ERROR4_1_TEST){
-
-    server::ServerError err;
-    // 1. Set license file name
-    std::string license_file_path("/tmp/milvus/abc.license");
-
-    // 2. Define output var
-    int device_count;
-    std::map<int, std::string> uuid_encryption_map;
-    time_t starting_time;
-    time_t end_time;
-
-    // 3. Read License File
-    err = server::LicenseLibrary::LicenseFileDeserialization(license_file_path,
-                                                             device_count,
-                                                             uuid_encryption_map,
-                                                             starting_time,
-                                                             end_time);
-    ASSERT_EQ(err, server::SERVER_SUCCESS);
-
-    // 4. Change starting time
-    time_t system_time;
-    err = server::LicenseLibrary::GetSystemTime(system_time);
-    ASSERT_EQ(err, server::SERVER_SUCCESS);
-
-    system_time+=60*60*24;
-
-    err = server::LicenseLibrary::LicenseFileSerialization(license_file_path,
-                                                           device_count,
-                                                           uuid_encryption_map,
-                                                           system_time,
-                                                           end_time);
-    ASSERT_EQ(err, server::SERVER_SUCCESS);
-
-    // 5. Legality check
-    err = server::LicenseCheck::LegalityCheck(license_file_path);
-    ASSERT_EQ(err, server::SERVER_SUCCESS);
-}
-
-TEST(LicenseLibraryTest, CHECK_ERROR4_2_TEST){
-
-    server::ServerError err;
-    // 1. Set license file name
-    std::string license_file_path("/tmp/milvus/abc.license");
-
-    // 2. Define output var
-    int device_count;
-    std::map<int, std::string> uuid_encryption_map;
-    time_t starting_time;
-    time_t end_time;
-
-    // 3. Read License File
-    err = server::LicenseLibrary::LicenseFileDeserialization(license_file_path,
-                                                             device_count,
-                                                             uuid_encryption_map,
-                                                             starting_time,
-                                                             end_time);
-    ASSERT_EQ(err, server::SERVER_SUCCESS);
-
-    // 4. Change end time
-    time_t system_time;
-    err = server::LicenseLibrary::GetSystemTime(system_time);
-    ASSERT_EQ(err, server::SERVER_SUCCESS);
-
-    system_time-=100;
-
-    err = server::LicenseLibrary::LicenseFileSerialization(license_file_path,
-                                                           device_count,
-                                                           uuid_encryption_map,
-                                                           starting_time,
-                                                           system_time);
-    ASSERT_EQ(err, server::SERVER_SUCCESS);
-
-    // 5. Legality check
-    err = server::LicenseCheck::LegalityCheck(license_file_path);
-    ASSERT_EQ(err, server::SERVER_SUCCESS);
-}
-

cpp/unittest/license/license_library_tests.cpp

@@ -1,214 +1,214 @@
-/*******************************************************************************
- * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved
- * Unauthorized copying of this file, via any medium is strictly prohibited.
- * Proprietary and confidential.
- ******************************************************************************/
-
-
-#include "utils/Log.h"
-#include "license/LicenseLibrary.h"
-#include "utils/Error.h"
-
-#include <gtest/gtest.h>
-#include <iostream>
-
-
-using namespace zilliz::milvus;
-
-TEST(LicenseLibraryTest, FILE_EXISTENT_TEST) {
-
-    std::string hosts_file = "/etc/hosts";
-    ASSERT_EQ(server::LicenseLibrary::IsFileExistent(hosts_file), true);
-
-    std::string no_exist_file = "/temp/asdaasd";
-    ASSERT_EQ(server::LicenseLibrary::IsFileExistent(no_exist_file), false);
-
-    std::string directory = "/tmp";
-    ASSERT_EQ(server::LicenseLibrary::IsFileExistent(directory), false);
-}
-
-TEST(LicenseLibraryTest, GPU_INFO_TEST) {
-
-    int device_count = 0;
-    server::ServerError err = server::LicenseLibrary::GetDeviceCount(device_count);
-    ASSERT_EQ(err, server::SERVER_SUCCESS);
-    std::cout << "Device Count: " << device_count << std::endl;
-
-    std::vector<std::string> uuid_array;
-    err = server::LicenseLibrary::GetUUID(device_count, uuid_array);
-    ASSERT_EQ(err, server::SERVER_SUCCESS);
-    for (long i = 0; i < device_count; ++i) {
-        std::cout << "Device Id: " << i << ", UUID: " << uuid_array[i] << std::endl;
-    }
-
-    std::vector<std::string> uuid_md5_array;
-    err = server::LicenseLibrary::GetUUIDMD5(device_count, uuid_array, uuid_md5_array);
-    ASSERT_EQ(err, server::SERVER_SUCCESS);
-    for (long i = 0; i < device_count; ++i) {
-        std::cout << "Device Id: " << i << ", UUID: " << uuid_array[i] << ", UUID_MD5: " << uuid_md5_array[i]
-                  << std::endl;
-    }
-
-    std::vector<std::string> uuid_sha256_array;
-    err = server::LicenseLibrary::GetUUIDSHA256(device_count, uuid_array, uuid_sha256_array);
-    ASSERT_EQ(err, server::SERVER_SUCCESS);
-    for (long i = 0; i < device_count; ++i) {
-        std::cout << "Device Id: " << i << ", UUID: " << uuid_array[i] << ", UUID_SHA256: "
-                  << uuid_sha256_array[i] << std::endl;
-    }
-
-    time_t systemtime;
-    err = server::LicenseLibrary::GetSystemTime(systemtime);
-    ASSERT_EQ(err, server::SERVER_SUCCESS);
-    std::cout << "System Time: " << systemtime << std::endl;
-
-}
-
-TEST(LicenseLibraryTest, LICENSE_FILE_TEST) {
-
-    // 0. File check
-    std::string test("/tmp/a.test");
-    bool is = server::LicenseLibrary::IsFileExistent(test);
-    ASSERT_EQ(is, false);
-
-    // 1. Get Device Count
-    int device_count = 0;
-    server::ServerError err = server::LicenseLibrary::GetDeviceCount(device_count);
-    ASSERT_EQ(err, server::SERVER_SUCCESS);
-
-    // 2. Get All GPU UUID
-    std::vector<std::string> uuid_array;
-    err = server::LicenseLibrary::GetUUID(device_count, uuid_array);
-    ASSERT_EQ(err, server::SERVER_SUCCESS);
-
-    // 3. Get UUID SHA256
-    std::vector<std::string> uuid_sha256_array;
-    err = server::LicenseLibrary::GetUUIDSHA256(device_count, uuid_array, uuid_sha256_array);
-    ASSERT_EQ(err, server::SERVER_SUCCESS);
-
-    // 4. Generate GPU ID map with GPU UUID
-    std::map<int, std::string> uuid_encrption_map;
-    for (int i = 0; i < device_count; ++i) {
-        uuid_encrption_map[i] = uuid_sha256_array[i];
-    }
-
-    // 5.GPU_info File
-    std::string GPU_info_file_path("/tmp/milvus.info");
-
-
-    // 6. Generate GPU_info File
-    err = server::LicenseLibrary::GPUinfoFileSerialization(GPU_info_file_path,
-                                                           device_count,
-                                                           uuid_encrption_map);
-    ASSERT_EQ(err, server::SERVER_SUCCESS);
-
-    // 7. Define output var
-    int output_info_device_count = 0;
-    std::map<int, std::string> output_info_uuid_encrption_map;
-
-    // 8. Read GPU_info File
-    err = server::LicenseLibrary::GPUinfoFileDeserialization(GPU_info_file_path,
-                                                             output_info_device_count,
-                                                             output_info_uuid_encrption_map);
-    ASSERT_EQ(err, server::SERVER_SUCCESS);
-
-    ASSERT_EQ(device_count, output_info_device_count);
-    for (int i = 0; i < device_count; ++i) {
-        ASSERT_EQ(uuid_encrption_map[i], output_info_uuid_encrption_map[i]);
-    }
-
-    // 9. Set license file name
-    std::string license_file_path("/tmp/milvus.license");
-
-    // 10. Get System Time/starting_time ans End Time
-    time_t sysyem_time;
-    err = server::LicenseLibrary::GetSystemTime(sysyem_time);
-    ASSERT_EQ(err, server::SERVER_SUCCESS);
-
-    // 11.GetDateTime
-    time_t starting_time;
-    time_t end_time;
-    const char *string_starting_time = "2019-05-10";
-    const char *string_end_time = "2022-05-10";
-    err = server::LicenseLibrary::GetDateTime(string_starting_time, starting_time);
-    ASSERT_EQ(err, server::SERVER_SUCCESS);
-    err = server::LicenseLibrary::GetDateTime(string_end_time, end_time);
-    ASSERT_EQ(err, server::SERVER_SUCCESS);
-
-    // 12. Generate License File
-    err = server::LicenseLibrary::LicenseFileSerialization(license_file_path,
-                                                           device_count,
-                                                           uuid_encrption_map,
-                                                           starting_time,
-                                                           end_time);
-    ASSERT_EQ(err, server::SERVER_SUCCESS);
-
-    // 13. Define output var
-    int output_device_count = 0;
-    std::map<int, std::string> output_uuid_encrption_map;
-    time_t output_starting_time;
-    time_t output_end_time;
-
-    // 14. Read License File
-    err = server::LicenseLibrary::LicenseFileDeserialization(license_file_path,
-                                                             output_device_count,
-                                                             output_uuid_encrption_map,
-                                                             output_starting_time,
-                                                             output_end_time);
-    ASSERT_EQ(err, server::SERVER_SUCCESS);
-
-    ASSERT_EQ(device_count, output_device_count);
-    ASSERT_EQ(starting_time, output_starting_time);
-    ASSERT_EQ(end_time, output_end_time);
-
-    for (int i = 0; i < device_count; ++i) {
-        ASSERT_EQ(uuid_encrption_map[i], output_uuid_encrption_map[i]);
-    }
-
-//    // 15. Get License File Attribute
-//    time_t update_time;
-//    off_t file_size;
-//    err = server::LicenseLibrary::GetFileUpdateTimeAndSize(license_file_path, update_time, file_size);
+///*******************************************************************************
+// * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved
+// * Unauthorized copying of this file, via any medium is strictly prohibited.
+// * Proprietary and confidential.
+// ******************************************************************************/
+//
+//
+//#include "utils/Log.h"
+//#include "license/LicenseLibrary.h"
+//#include "utils/Error.h"
+//
+//#include <gtest/gtest.h>
+//#include <iostream>
+//
+//
+//using namespace zilliz::milvus;
+//
+//TEST(LicenseLibraryTest, FILE_EXISTENT_TEST) {
+//
+//    std::string hosts_file = "/etc/hosts";
+//    ASSERT_EQ(server::LicenseLibrary::IsFileExistent(hosts_file), true);
+//
+//    std::string no_exist_file = "/temp/asdaasd";
+//    ASSERT_EQ(server::LicenseLibrary::IsFileExistent(no_exist_file), false);
+//
+//    std::string directory = "/tmp";
+//    ASSERT_EQ(server::LicenseLibrary::IsFileExistent(directory), false);
+//}
+//
+//TEST(LicenseLibraryTest, GPU_INFO_TEST) {
+//
+//    int device_count = 0;
+//    server::ServerError err = server::LicenseLibrary::GetDeviceCount(device_count);
+//    ASSERT_EQ(err, server::SERVER_SUCCESS);
+//    std::cout << "Device Count: " << device_count << std::endl;
+//
+//    std::vector<std::string> uuid_array;
+//    err = server::LicenseLibrary::GetUUID(device_count, uuid_array);
+//    ASSERT_EQ(err, server::SERVER_SUCCESS);
+//    for (long i = 0; i < device_count; ++i) {
+//        std::cout << "Device Id: " << i << ", UUID: " << uuid_array[i] << std::endl;
+//    }
+//
+//    std::vector<std::string> uuid_md5_array;
+//    err = server::LicenseLibrary::GetUUIDMD5(device_count, uuid_array, uuid_md5_array);
+//    ASSERT_EQ(err, server::SERVER_SUCCESS);
+//    for (long i = 0; i < device_count; ++i) {
+//        std::cout << "Device Id: " << i << ", UUID: " << uuid_array[i] << ", UUID_MD5: " << uuid_md5_array[i]
+//                  << std::endl;
+//    }
+//
+//    std::vector<std::string> uuid_sha256_array;
+//    err = server::LicenseLibrary::GetUUIDSHA256(device_count, uuid_array, uuid_sha256_array);
+//    ASSERT_EQ(err, server::SERVER_SUCCESS);
+//    for (long i = 0; i < device_count; ++i) {
+//        std::cout << "Device Id: " << i << ", UUID: " << uuid_array[i] << ", UUID_SHA256: "
+//                  << uuid_sha256_array[i] << std::endl;
+//    }
+//
+//    time_t systemtime;
+//    err = server::LicenseLibrary::GetSystemTime(systemtime);
+//    ASSERT_EQ(err, server::SERVER_SUCCESS);
+//    std::cout << "System Time: " << systemtime << std::endl;
+//
+//}
+//
+//TEST(LicenseLibraryTest, LICENSE_FILE_TEST) {
+//
+//    // 0. File check
+//    std::string test("/tmp/a.test");
+//    bool is = server::LicenseLibrary::IsFileExistent(test);
+//    ASSERT_EQ(is, false);
+//
+//    // 1. Get Device Count
+//    int device_count = 0;
+//    server::ServerError err = server::LicenseLibrary::GetDeviceCount(device_count);
 //    ASSERT_EQ(err, server::SERVER_SUCCESS);
 //
-//    // 16. Get License File MD5
-//    std::string file_md5;
-//    err = server::LicenseLibrary::GetFileMD5(license_file_path, file_md5);
+//    // 2. Get All GPU UUID
+//    std::vector<std::string> uuid_array;
+//    err = server::LicenseLibrary::GetUUID(device_count, uuid_array);
 //    ASSERT_EQ(err, server::SERVER_SUCCESS);
-
-
-
-//    // 17. Generate Secret File
-//    std::string secret_file_path("/tmp/milvus.secret");
-//    err = server::LicenseLibrary::SecretFileSerialization(secret_file_path,
-//                                                          update_time,
-//                                                          file_size,
-//                                                          starting_time,
-//                                                          end_time,
-//                                                          file_md5);
+//
+//    // 3. Get UUID SHA256
+//    std::vector<std::string> uuid_sha256_array;
+//    err = server::LicenseLibrary::GetUUIDSHA256(device_count, uuid_array, uuid_sha256_array);
 //    ASSERT_EQ(err, server::SERVER_SUCCESS);
-
-//   // 18. Define output var
-//    time_t output_update_time;
-//    off_t output_file_size;
+//
+//    // 4. Generate GPU ID map with GPU UUID
+//    std::map<int, std::string> uuid_encrption_map;
+//    for (int i = 0; i < device_count; ++i) {
+//        uuid_encrption_map[i] = uuid_sha256_array[i];
+//    }
+//
+//    // 5.GPU_info File
+//    std::string GPU_info_file_path("/tmp/milvus.info");
+//
+//
+//    // 6. Generate GPU_info File
+//    err = server::LicenseLibrary::GPUinfoFileSerialization(GPU_info_file_path,
+//                                                           device_count,
+//                                                           uuid_encrption_map);
+//    ASSERT_EQ(err, server::SERVER_SUCCESS);
+//
+//    // 7. Define output var
+//    int output_info_device_count = 0;
+//    std::map<int, std::string> output_info_uuid_encrption_map;
+//
+//    // 8. Read GPU_info File
+//    err = server::LicenseLibrary::GPUinfoFileDeserialization(GPU_info_file_path,
+//                                                             output_info_device_count,
+//                                                             output_info_uuid_encrption_map);
+//    ASSERT_EQ(err, server::SERVER_SUCCESS);
+//
+//    ASSERT_EQ(device_count, output_info_device_count);
+//    for (int i = 0; i < device_count; ++i) {
+//        ASSERT_EQ(uuid_encrption_map[i], output_info_uuid_encrption_map[i]);
+//    }
+//
+//    // 9. Set license file name
+//    std::string license_file_path("/tmp/milvus.license");
+//
+//    // 10. Get System Time/starting_time ans End Time
+//    time_t sysyem_time;
+//    err = server::LicenseLibrary::GetSystemTime(sysyem_time);
+//    ASSERT_EQ(err, server::SERVER_SUCCESS);
+//
+//    // 11.GetDateTime
+//    time_t starting_time;
+//    time_t end_time;
+//    const char *string_starting_time = "2019-05-10";
+//    const char *string_end_time = "2022-05-10";
+//    err = server::LicenseLibrary::GetDateTime(string_starting_time, starting_time);
+//    ASSERT_EQ(err, server::SERVER_SUCCESS);
+//    err = server::LicenseLibrary::GetDateTime(string_end_time, end_time);
+//    ASSERT_EQ(err, server::SERVER_SUCCESS);
+//
+//    // 12. Generate License File
+//    err = server::LicenseLibrary::LicenseFileSerialization(license_file_path,
+//                                                           device_count,
+//                                                           uuid_encrption_map,
+//                                                           starting_time,
+//                                                           end_time);
+//    ASSERT_EQ(err, server::SERVER_SUCCESS);
+//
+//    // 13. Define output var
+//    int output_device_count = 0;
+//    std::map<int, std::string> output_uuid_encrption_map;
 //    time_t output_starting_time;
 //    time_t output_end_time;
-//    std::string output_file_md5;
-
-//   // 19. Read License File
-//    err = server::LicenseLibrary::SecretFileDeserialization(secret_file_path,
-//                                                            output_update_time,
-//                                                            output_file_size,
-//                                                            output_starting_time,
-//                                                            output_end_time,
-//                                                            output_file_md5);
+//
+//    // 14. Read License File
+//    err = server::LicenseLibrary::LicenseFileDeserialization(license_file_path,
+//                                                             output_device_count,
+//                                                             output_uuid_encrption_map,
+//                                                             output_starting_time,
+//                                                             output_end_time);
 //    ASSERT_EQ(err, server::SERVER_SUCCESS);
 //
-//    ASSERT_EQ(update_time, output_update_time);
-//    ASSERT_EQ(file_size, output_file_size);
+//    ASSERT_EQ(device_count, output_device_count);
 //    ASSERT_EQ(starting_time, output_starting_time);
 //    ASSERT_EQ(end_time, output_end_time);
-//    ASSERT_EQ(file_md5, output_file_md5);
-
-
-}
+//
+//    for (int i = 0; i < device_count; ++i) {
+//        ASSERT_EQ(uuid_encrption_map[i], output_uuid_encrption_map[i]);
+//    }
+//
+////    // 15. Get License File Attribute
+////    time_t update_time;
+////    off_t file_size;
+////    err = server::LicenseLibrary::GetFileUpdateTimeAndSize(license_file_path, update_time, file_size);
+////    ASSERT_EQ(err, server::SERVER_SUCCESS);
+////
+////    // 16. Get License File MD5
+////    std::string file_md5;
+////    err = server::LicenseLibrary::GetFileMD5(license_file_path, file_md5);
+////    ASSERT_EQ(err, server::SERVER_SUCCESS);
+//
+//
+//
+////    // 17. Generate Secret File
+////    std::string secret_file_path("/tmp/milvus.secret");
+////    err = server::LicenseLibrary::SecretFileSerialization(secret_file_path,
+////                                                          update_time,
+////                                                          file_size,
+////                                                          starting_time,
+////                                                          end_time,
+////                                                          file_md5);
+////    ASSERT_EQ(err, server::SERVER_SUCCESS);
+//
+////   // 18. Define output var
+////    time_t output_update_time;
+////    off_t output_file_size;
+////    time_t output_starting_time;
+////    time_t output_end_time;
+////    std::string output_file_md5;
+//
+////   // 19. Read License File
+////    err = server::LicenseLibrary::SecretFileDeserialization(secret_file_path,
+////                                                            output_update_time,
+////                                                            output_file_size,
+////                                                            output_starting_time,
+////                                                            output_end_time,
+////                                                            output_file_md5);
+////    ASSERT_EQ(err, server::SERVER_SUCCESS);
+////
+////    ASSERT_EQ(update_time, output_update_time);
+////    ASSERT_EQ(file_size, output_file_size);
+////    ASSERT_EQ(starting_time, output_starting_time);
+////    ASSERT_EQ(end_time, output_end_time);
+////    ASSERT_EQ(file_md5, output_file_md5);
+//
+//
+//}

cpp/unittest/metrics/CMakeLists.txt

@@ -55,8 +55,9 @@ set(count_test_src
         ${wrapper_src}
         ${metrics_src}
         metrics_test.cpp
+        prometheus_test.cpp
         ../db/utils.cpp
-        )
+        metricbase_test.cpp)
 
 
 add_executable(metrics_test ${count_test_src} ${require_files} )

cpp/unittest/metrics/metricbase_test.cpp

@@ -0,0 +1,56 @@
+/*******************************************************************************
+ * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved
+ * Unauthorized copying of this file, via any medium is strictly prohibited.
+ * Proprietary and confidential.
+ ******************************************************************************/
+
+#include "metrics/Metrics.h"
+
+#include <gtest/gtest.h>
+#include <iostream>
+
+using namespace zilliz::milvus;
+
+TEST(MetricbaseTest, Metricbase_Test){
+    server::MetricsBase instance = server::MetricsBase::GetInstance();
+    instance.Init();
+    server::SystemInfo::GetInstance().Init();
+    instance.AddVectorsSuccessTotalIncrement();
+    instance.AddVectorsFailTotalIncrement();
+    instance.AddVectorsDurationHistogramOberve(1.0);
+    instance.RawFileSizeHistogramObserve(1.0);
+    instance.IndexFileSizeHistogramObserve(1.0);
+    instance.BuildIndexDurationSecondsHistogramObserve(1.0);
+    instance.CacheUsageGaugeSet(1.0);
+    instance.MetaAccessTotalIncrement();
+    instance.MetaAccessDurationSecondsHistogramObserve(1.0);
+    instance.FaissDiskLoadDurationSecondsHistogramObserve(1.0);
+    instance.FaissDiskLoadSizeBytesHistogramObserve(1.0);
+    instance.FaissDiskLoadIOSpeedGaugeSet(1.0);
+    instance.CacheAccessTotalIncrement();
+    instance.MemTableMergeDurationSecondsHistogramObserve(1.0);
+    instance.SearchIndexDataDurationSecondsHistogramObserve(1.0);
+    instance.SearchRawDataDurationSecondsHistogramObserve(1.0);
+    instance.IndexFileSizeTotalIncrement();
+    instance.RawFileSizeTotalIncrement();
+    instance.IndexFileSizeGaugeSet(1.0);
+    instance.RawFileSizeGaugeSet(1.0);
+    instance.QueryResponseSummaryObserve(1.0);
+    instance.DiskStoreIOSpeedGaugeSet(1.0);
+    instance.DataFileSizeGaugeSet(1.0);
+    instance.AddVectorsSuccessGaugeSet(1.0);
+    instance.AddVectorsFailGaugeSet(1.0);
+    instance.QueryVectorResponseSummaryObserve(1.0, 1);
+    instance.QueryVectorResponsePerSecondGaugeSet(1.0);
+    instance.CPUUsagePercentSet();
+    instance.RAMUsagePercentSet();
+    instance.QueryResponsePerSecondGaugeSet(1.0);
+    instance.GPUPercentGaugeSet();
+    instance.GPUMemoryUsageGaugeSet();
+    instance.AddVectorsPerSecondGaugeSet(1,1,1);
+    instance.QueryIndexTypePerSecondSet("IVF", 1.0);
+    instance.ConnectionGaugeIncrement();
+    instance.ConnectionGaugeDecrement();
+    instance.KeepingAliveCounterIncrement();
+    instance.OctetsSet();
+}

cpp/unittest/metrics/metrics_test.cpp

@@ -24,7 +24,6 @@
 using namespace zilliz::milvus;
 
 
-
 TEST_F(DBTest, Metric_Tes) {
 
     server::SystemInfo::GetInstance().Init();

cpp/unittest/metrics/prometheus_test.cpp

@@ -0,0 +1,57 @@
+/*******************************************************************************
+ * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved
+ * Unauthorized copying of this file, via any medium is strictly prohibited.
+ * Proprietary and confidential.
+ ******************************************************************************/
+
+#include "metrics/PrometheusMetrics.h"
+
+#include <gtest/gtest.h>
+#include <iostream>
+
+using namespace zilliz::milvus;
+
+TEST(PrometheusTest, Prometheus_Test){
+    server::PrometheusMetrics instance = server::PrometheusMetrics::GetInstance();
+    instance.Init();
+    instance.SetStartup(true);
+    server::SystemInfo::GetInstance().Init();
+    instance.AddVectorsSuccessTotalIncrement();
+    instance.AddVectorsFailTotalIncrement();
+    instance.AddVectorsDurationHistogramOberve(1.0);
+    instance.RawFileSizeHistogramObserve(1.0);
+    instance.IndexFileSizeHistogramObserve(1.0);
+    instance.BuildIndexDurationSecondsHistogramObserve(1.0);
+    instance.CacheUsageGaugeSet(1.0);
+    instance.MetaAccessTotalIncrement();
+    instance.MetaAccessDurationSecondsHistogramObserve(1.0);
+    instance.FaissDiskLoadDurationSecondsHistogramObserve(1.0);
+    instance.FaissDiskLoadSizeBytesHistogramObserve(1.0);
+    instance.FaissDiskLoadIOSpeedGaugeSet(1.0);
+    instance.CacheAccessTotalIncrement();
+    instance.MemTableMergeDurationSecondsHistogramObserve(1.0);
+    instance.SearchIndexDataDurationSecondsHistogramObserve(1.0);
+    instance.SearchRawDataDurationSecondsHistogramObserve(1.0);
+    instance.IndexFileSizeTotalIncrement();
+    instance.RawFileSizeTotalIncrement();
+    instance.IndexFileSizeGaugeSet(1.0);
+    instance.RawFileSizeGaugeSet(1.0);
+    instance.QueryResponseSummaryObserve(1.0);
+    instance.DiskStoreIOSpeedGaugeSet(1.0);
+    instance.DataFileSizeGaugeSet(1.0);
+    instance.AddVectorsSuccessGaugeSet(1.0);
+    instance.AddVectorsFailGaugeSet(1.0);
+    instance.QueryVectorResponseSummaryObserve(1.0, 1);
+    instance.QueryVectorResponsePerSecondGaugeSet(1.0);
+    instance.CPUUsagePercentSet();
+    instance.RAMUsagePercentSet();
+    instance.QueryResponsePerSecondGaugeSet(1.0);
+    instance.GPUPercentGaugeSet();
+    instance.GPUMemoryUsageGaugeSet();
+    instance.AddVectorsPerSecondGaugeSet(1,1,1);
+    instance.QueryIndexTypePerSecondSet("IVF", 1.0);
+    instance.ConnectionGaugeIncrement();
+    instance.ConnectionGaugeDecrement();
+    instance.KeepingAliveCounterIncrement();
+    instance.OctetsSet();
+}

cpp/unittest/server/CMakeLists.txt

@@ -14,9 +14,11 @@ aux_source_directory(../../src/cache cache_srcs)
 aux_source_directory(../../src/wrapper wrapper_src)
 aux_source_directory(./ test_srcs)
 
-set(server_src_files
+set(utils_srcs
         ${MILVUS_ENGINE_SRC}/utils/StringHelpFunctions.cpp
-        ${MILVUS_ENGINE_SRC}/utils/AttributeSerializer.cpp
+        ${MILVUS_ENGINE_SRC}/utils/TimeRecorder.cpp
+        ${MILVUS_ENGINE_SRC}/utils/CommonUtil.cpp
+        ${MILVUS_ENGINE_SRC}/utils/LogUtil.cpp
         )
 
 cuda_add_executable(server_test
@@ -25,7 +27,7 @@ cuda_add_executable(server_test
         ${cache_srcs}
         ${wrapper_src}
         ${test_srcs}
-        ${server_src_files}
+        ${utils_srcs}
         ${require_files}
         )
 
@@ -53,3 +55,11 @@ target_link_libraries(server_test
         )
 
 install(TARGETS server_test DESTINATION bin)
+
+configure_file(appendix/server_config.yaml
+        "${CMAKE_CURRENT_BINARY_DIR}/milvus/conf/server_config.yaml"
+        COPYONLY)
+
+configure_file(appendix/log_config.conf
+        "${CMAKE_CURRENT_BINARY_DIR}/milvus/conf/log_config.conf"
+        COPYONLY)

cpp/unittest/server/appendix/log_config.conf

@@ -20,8 +20,8 @@
     TO_STANDARD_OUTPUT      =   false
 ## Error logs
 * ERROR:
-    ENABLED                 =   false
+    ENABLED                 =   true
     FILENAME                =   "/tmp/milvus/logs/milvus-%datetime{%H:%m}-error.log"
 * FATAL:
-    ENABLED                 =   false
-    FILENAME                =   "/tmp/milvus/logs/milvus-%datetime{%H:%m}-fatal.log"
+    ENABLED                 =   true
+    FILENAME                =   "/tmp/milvus/logs/milvus-%datetime{%H:%m}-fatal.log"

cpp/unittest/server/appendix/server_config.yaml

@@ -22,7 +22,7 @@ metric_config:
 
 
 license_config:         # license configure
-  license_path: "/tmp/system.license" # license file path
+  license_path: "/tmp/milvus/system.license" # license file path
 
 cache_config:           # cache configure
-  cpu_cache_capacity: 16   # how many memory are used as cache, unit: GB, range: 0 ~ less than total memory
+  cpu_cache_capacity: 16   # how many memory are used as cache, unit: GB, range: 0 ~ less than total memory

cpp/unittest/server/cache_test.cpp

@@ -11,18 +11,35 @@
 
 using namespace zilliz::milvus;
 
-TEST(CacheTest, CACHE_TEST) {
-    cache::CacheMgr* cpu_mgr = cache::CpuCacheMgr::GetInstance();
+namespace {
+
+class InvalidCacheMgr : public cache::CacheMgr {
+public:
+    InvalidCacheMgr() {
+    }
+};
+
+class LessItemCacheMgr : public cache::CacheMgr {
+public:
+    LessItemCacheMgr() {
+        cache_ = std::make_shared<cache::Cache>(1UL << 12, 10);
+    }
+};
+
+}
+
+TEST(CacheTest, CPU_CACHE_TEST) {
+    cache::CacheMgr *cpu_mgr = cache::CpuCacheMgr::GetInstance();
 
     const int64_t gbyte = 1 << 30;
     int64_t g_num = 16;
-    int64_t cap = g_num*gbyte;
+    int64_t cap = g_num * gbyte;
     cpu_mgr->SetCapacity(cap);
     ASSERT_EQ(cpu_mgr->CacheCapacity(), cap);
 
     const int dim = 256;
 
-    for(int i = 0; i < 20; i++) {
+    for (int i = 0; i < 20; i++) {
         std::shared_ptr<faiss::Index> raw_index(faiss::index_factory(dim, "IDMap,Flat"));
         engine::Index_ptr index = std::make_shared<engine::Index>(raw_index);
         index->ntotal = 1000000;//less 1G per index
@@ -31,6 +48,12 @@ TEST(CacheTest, CACHE_TEST) {
     }
     ASSERT_LT(cpu_mgr->ItemCount(), g_num);
 
+    auto obj = cpu_mgr->GetIndex("index_0");
+    ASSERT_TRUE(obj == nullptr);
+
+    obj = cpu_mgr->GetIndex("index_19");
+    ASSERT_TRUE(obj != nullptr);
+
     {
         std::string item = "index_15";
         ASSERT_TRUE(cpu_mgr->ItemExists(item));
@@ -49,4 +72,58 @@ TEST(CacheTest, CACHE_TEST) {
         cpu_mgr->InsertItem("index_6g", index);
         ASSERT_EQ(cpu_mgr->ItemCount(), 0);//data greater than capacity can not be inserted sucessfully
     }
+
+    cpu_mgr->PrintInfo();
+}
+
+TEST(CacheTest, GPU_CACHE_TEST) {
+    cache::CacheMgr* gpu_mgr = cache::GpuCacheMgr::GetInstance();
+
+    const int dim = 256;
+
+    for(int i = 0; i < 20; i++) {
+        std::shared_ptr<faiss::Index> raw_index(faiss::index_factory(dim, "IDMap,Flat"));
+        engine::Index_ptr index = std::make_shared<engine::Index>(raw_index);
+        index->ntotal = 1000;
+
+        cache::DataObjPtr obj = std::make_shared<cache::DataObj>(index);
+
+        gpu_mgr->InsertItem("index_" + std::to_string(i), obj);
+    }
+
+    auto obj = gpu_mgr->GetItem("index_0");
+
+    gpu_mgr->ClearCache();
+    ASSERT_EQ(gpu_mgr->ItemCount(), 0);
+}
+
+TEST(CacheTest, INVALID_TEST) {
+    {
+        InvalidCacheMgr mgr;
+        ASSERT_EQ(mgr.ItemCount(), 0);
+        ASSERT_FALSE(mgr.ItemExists("test"));
+        ASSERT_EQ(mgr.GetItem("test"), nullptr);
+
+        mgr.InsertItem("test", cache::DataObjPtr());
+        mgr.InsertItem("test", engine::Index_ptr(nullptr));
+        mgr.EraseItem("test");
+        mgr.PrintInfo();
+        mgr.ClearCache();
+        mgr.SetCapacity(100);
+        ASSERT_EQ(mgr.CacheCapacity(), 0);
+        ASSERT_EQ(mgr.CacheUsage(), 0);
+    }
+
+    {
+        LessItemCacheMgr mgr;
+        for(int i = 0; i < 20; i++) {
+            std::shared_ptr<faiss::Index> raw_index(faiss::index_factory(2, "IDMap,Flat"));
+            engine::Index_ptr index = std::make_shared<engine::Index>(raw_index);
+            index->ntotal = 2;
+
+            cache::DataObjPtr obj = std::make_shared<cache::DataObj>(index);
+            mgr.InsertItem("index_" + std::to_string(i), obj);
+        }
+        ASSERT_EQ(mgr.GetItem("index_0"), nullptr);
+    }
 }

cpp/unittest/server/common_test.cpp

@@ -1,26 +0,0 @@
-////////////////////////////////////////////////////////////////////////////////
-// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved
-// Unauthorized copying of this file, via any medium is strictly prohibited.
-// Proprietary and confidential.
-////////////////////////////////////////////////////////////////////////////////
-#include <gtest/gtest.h>
-#include "utils/CommonUtil.h"
-#include "utils/Error.h"
-
-using namespace zilliz::milvus;
-
-
-TEST(CommonTest, COMMON_TEST) {
-    std::string path1 = "/tmp/milvus_test/";
-    std::string path2 = path1 + "common_test_12345/";
-    std::string path3 = path2 + "abcdef";
-    server::ServerError err = server::CommonUtil::CreateDirectory(path3);
-    ASSERT_EQ(err, server::SERVER_SUCCESS);
-
-    ASSERT_TRUE(server::CommonUtil::IsDirectoryExit(path3));
-
-    err = server::CommonUtil::DeleteDirectory(path1);
-    ASSERT_EQ(err, server::SERVER_SUCCESS);
-
-    ASSERT_FALSE(server::CommonUtil::IsDirectoryExit(path1));
-}

cpp/unittest/server/config_test.cpp

@@ -0,0 +1,105 @@
+////////////////////////////////////////////////////////////////////////////////
+// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved
+// Unauthorized copying of this file, via any medium is strictly prohibited.
+// Proprietary and confidential.
+////////////////////////////////////////////////////////////////////////////////
+#include <gtest/gtest.h>
+
+#include "config/IConfigMgr.h"
+#include "server/ServerConfig.h"
+
+using namespace zilliz::milvus;
+
+namespace {
+
+static const std::string CONFIG_FILE_PATH = "./milvus/conf/server_config.yaml";
+static const std::string LOG_FILE_PATH = "./milvus/conf/log_config.conf";
+
+}
+
+TEST(ConfigTest, CONFIG_TEST) {
+    server::IConfigMgr* config_mgr = server::IConfigMgr::GetInstance();
+
+    server::ServerError err = config_mgr->LoadConfigFile("");
+    ASSERT_EQ(err, server::SERVER_UNEXPECTED_ERROR);
+
+    err = config_mgr->LoadConfigFile(LOG_FILE_PATH);
+    ASSERT_EQ(err, server::SERVER_UNEXPECTED_ERROR);
+
+    err = config_mgr->LoadConfigFile(CONFIG_FILE_PATH);
+    ASSERT_EQ(err, server::SERVER_SUCCESS);
+
+    config_mgr->Print();
+
+    server::ConfigNode& root_config = config_mgr->GetRootNode();
+    server::ConfigNode& server_config = root_config.GetChild("server_config");
+    server::ConfigNode& db_config = root_config.GetChild("db_config");
+    server::ConfigNode& metric_config = root_config.GetChild("metric_config");
+    server::ConfigNode& cache_config = root_config.GetChild("cache_config");
+
+    std::string address = server_config.GetValue("address");
+    ASSERT_TRUE(!address.empty());
+    int64_t port = server_config.GetInt64Value("port");
+    ASSERT_TRUE(port != 0);
+
+    server_config.SetValue("test", "2.5");
+    double test = server_config.GetDoubleValue("test");
+    ASSERT_EQ(test, 2.5);
+
+    server::ConfigNode fake;
+    server_config.AddChild("fake", fake);
+    fake = server_config.GetChild("fake");
+    server::ConfigNodeArr arr;
+    server_config.GetChildren(arr);
+    ASSERT_EQ(arr.size(), 1UL);
+
+    server_config.ClearChildren();
+    auto children = server_config.GetChildren();
+    ASSERT_TRUE(children.empty());
+
+    server_config.ClearConfig();
+    auto configs = server_config.GetConfig();
+    ASSERT_TRUE(configs.empty());
+
+    server_config.AddSequenceItem("seq", "aaa");
+    server_config.AddSequenceItem("seq", "bbb");
+    auto seq = server_config.GetSequence("seq");
+    ASSERT_EQ(seq.size(), 2UL);
+
+    server::ConfigNode combine;
+    combine.Combine(server_config);
+
+    combine.PrintAll();
+    std::string all = combine.DumpString();
+    ASSERT_TRUE(!all.empty());
+
+    server_config.ClearSequences();
+    auto seqs = server_config.GetSequences();
+    ASSERT_TRUE(seqs.empty());
+
+    const server::ConfigNode const_node = root_config.GetChild("cache_config");
+    float flt = const_node.GetFloatValue("cpu_cache_capacity");
+    ASSERT_GT(flt, 0.0);
+}
+
+TEST(ConfigTest, SERVER_CONFIG_TEST) {
+    server::ServerConfig& config = server::ServerConfig::GetInstance();
+    server::ServerError err = config.LoadConfigFile(CONFIG_FILE_PATH);
+    ASSERT_EQ(err, server::SERVER_SUCCESS);
+
+    server::ConfigNode node1 = config.GetConfig("server_config");
+    server::ConfigNode& node2 = config.GetConfig("cache_config");
+    node1.Combine(node2);
+
+    int32_t cap = node1.GetInt32Value("cpu_cache_capacity");
+    ASSERT_GT(cap, 0);
+
+    node1.SetValue("bool", "true");
+    bool bt = node1.GetBoolValue("bool");
+    ASSERT_TRUE(bt);
+
+    config.PrintAll();
+
+    const server::ServerConfig const_config = config;
+    server::ConfigNode node = const_config.GetConfig("aaa");
+}

cpp/unittest/server/util_test.cpp

@@ -4,31 +4,176 @@
 // Proprietary and confidential.
 ////////////////////////////////////////////////////////////////////////////////
 #include <gtest/gtest.h>
+#include <thread>
 
-#include "utils/AttributeSerializer.h"
+#include "utils/CommonUtil.h"
+#include "utils/Error.h"
 #include "utils/StringHelpFunctions.h"
+#include "utils/TimeRecorder.h"
+#include "utils/BlockingQueue.h"
+#include "utils/LogUtil.h"
 
 using namespace zilliz::milvus;
 
-TEST(AttribSerializeTest, ATTRIBSERIAL_TEST) {
-    std::map<std::string, std::string> attrib;
-    attrib["uid"] = "ABCDEF";
-    attrib["color"] = "red";
-    attrib["number"] = "9900";
-    attrib["comment"] = "please note: it is a car, not a ship";
-    attrib["address"] = " china;shanghai ";
+namespace {
 
-    std::string attri_str;
-    server::AttributeSerializer::Encode(attrib, attri_str);
+static const std::string LOG_FILE_PATH = "./milvus/conf/log_config.conf";
 
-    std::map<std::string, std::string> attrib_out;
-    server::ServerError err = server::AttributeSerializer::Decode(attri_str, attrib_out);
-    ASSERT_EQ(err, server::SERVER_SUCCESS);
-
-    ASSERT_EQ(attrib_out.size(), attrib.size());
-    for(auto iter : attrib) {
-        ASSERT_EQ(attrib_out[iter.first], attrib_out[iter.first]);
-    }
+using TimeUnit = server::TimeRecorder::TimeDisplayUnit;
+double TestTimeRecorder(TimeUnit unit, int64_t log_level, int64_t sleep_ms) {
+    server::TimeRecorder rc("test rc", unit, log_level);
+    rc.Record("begin");
+    std::this_thread::sleep_for(std::chrono::milliseconds(sleep_ms));
+    rc.Elapse("end");
+    return rc.Span();
+}
 
 }
 
+TEST(UtilTest, EXCEPTION_TEST) {
+    std::string err_msg = "failed";
+    server::ServerException ex(server::SERVER_UNEXPECTED_ERROR, err_msg);
+    ASSERT_EQ(ex.error_code(), server::SERVER_UNEXPECTED_ERROR);
+    std::string msg = ex.what();
+    ASSERT_EQ(msg, err_msg);
+}
+
+TEST(UtilTest, COMMON_TEST) {
+    unsigned long total_mem = 0, free_mem = 0;
+    server::CommonUtil::GetSystemMemInfo(total_mem, free_mem);
+    ASSERT_GT(total_mem, 0);
+    ASSERT_GT(free_mem, 0);
+
+    unsigned int thread_cnt = 0;
+    server::CommonUtil::GetSystemAvailableThreads(thread_cnt);
+    ASSERT_GT(thread_cnt, 0);
+
+    std::string path1 = "/tmp/milvus_test/";
+    std::string path2 = path1 + "common_test_12345/";
+    std::string path3 = path2 + "abcdef";
+    server::ServerError err = server::CommonUtil::CreateDirectory(path3);
+    ASSERT_EQ(err, server::SERVER_SUCCESS);
+    //test again
+    err = server::CommonUtil::CreateDirectory(path3);
+    ASSERT_EQ(err, server::SERVER_SUCCESS);
+
+    ASSERT_TRUE(server::CommonUtil::IsDirectoryExist(path3));
+
+    err = server::CommonUtil::DeleteDirectory(path1);
+    ASSERT_EQ(err, server::SERVER_SUCCESS);
+    //test again
+    err = server::CommonUtil::DeleteDirectory(path1);
+    ASSERT_EQ(err, server::SERVER_SUCCESS);
+
+    ASSERT_FALSE(server::CommonUtil::IsDirectoryExist(path1));
+    ASSERT_FALSE(server::CommonUtil::IsFileExist(path1));
+
+    std::string exe_path = server::CommonUtil::GetExePath();
+    ASSERT_FALSE(exe_path.empty());
+
+    time_t tt;
+    time( &tt );
+    tm time_struct;
+    memset(&time_struct, 0, sizeof(tm));
+    server::CommonUtil::ConvertTime(tt, time_struct);
+    ASSERT_GT(time_struct.tm_year, 0);
+    ASSERT_GT(time_struct.tm_mon, 0);
+    ASSERT_GT(time_struct.tm_mday, 0);
+    server::CommonUtil::ConvertTime(time_struct, tt);
+    ASSERT_GT(tt, 0);
+
+    bool res = server::CommonUtil::TimeStrToTime("2019-03-23", tt, time_struct);
+    ASSERT_EQ(time_struct.tm_year, 119);
+    ASSERT_EQ(time_struct.tm_mon, 2);
+    ASSERT_EQ(time_struct.tm_mday, 23);
+    ASSERT_GT(tt, 0);
+    ASSERT_TRUE(res);
+}
+
+TEST(UtilTest, STRINGFUNCTIONS_TEST) {
+    std::string str = " test zilliz";
+    server::StringHelpFunctions::TrimStringBlank(str);
+    ASSERT_EQ(str, "test zilliz");
+
+    str = "\"test zilliz\"";
+    server::StringHelpFunctions::TrimStringQuote(str, "\"");
+    ASSERT_EQ(str, "test zilliz");
+
+    str = "a,b,c";
+    std::vector<std::string> result;
+    server::ServerError err = server::StringHelpFunctions::SplitStringByDelimeter(str , ",", result);
+    ASSERT_EQ(err, server::SERVER_SUCCESS);
+    ASSERT_EQ(result.size(), 3UL);
+
+    result.clear();
+    err = server::StringHelpFunctions::SplitStringByQuote(str , ",", "\"", result);
+    ASSERT_EQ(err, server::SERVER_SUCCESS);
+    ASSERT_EQ(result.size(), 3UL);
+
+    result.clear();
+    err = server::StringHelpFunctions::SplitStringByQuote(str , ",", "", result);
+    ASSERT_EQ(err, server::SERVER_SUCCESS);
+    ASSERT_EQ(result.size(), 3UL);
+
+    str = "55,\"aa,gg,yy\",b";
+    result.clear();
+    err = server::StringHelpFunctions::SplitStringByQuote(str , ",", "\"", result);
+    ASSERT_EQ(err, server::SERVER_SUCCESS);
+    ASSERT_EQ(result.size(), 3UL);
+
+
+}
+
+TEST(UtilTest, TIMERECORDER_TEST) {
+    double span = TestTimeRecorder(TimeUnit::eTimeAutoUnit, 0, 1001);
+    ASSERT_GT(span, 0.0);
+    span = TestTimeRecorder(TimeUnit::eTimeAutoUnit, 0, 101);
+    ASSERT_GT(span, 0.0);
+    span = TestTimeRecorder(TimeUnit::eTimeHourUnit, 1, 10);
+    ASSERT_GT(span, 0.0);
+    span = TestTimeRecorder(TimeUnit::eTimeMinuteUnit, 2, 10);
+    ASSERT_GT(span, 0.0);
+    span = TestTimeRecorder(TimeUnit::eTimeSecondUnit, 3, 10);
+    ASSERT_GT(span, 0.0);
+    span = TestTimeRecorder(TimeUnit::eTimeMilliSecUnit, 4, 10);
+    ASSERT_GT(span, 0.0);
+    span = TestTimeRecorder(TimeUnit::eTimeMicroSecUnit, -1, 10);
+    ASSERT_GT(span, 0.0);
+}
+
+TEST(UtilTest, BLOCKINGQUEUE_TEST) {
+    server::BlockingQueue<std::string> bq;
+
+    static const size_t count = 10;
+    bq.SetCapacity(count);
+
+    for(size_t i = 1; i <= count; i++) {
+        std::string id = "No." + std::to_string(i);
+        bq.Put(id);
+    }
+
+    ASSERT_EQ(bq.Size(), count);
+    ASSERT_FALSE(bq.Empty());
+
+    std::string str = bq.Front();
+    ASSERT_EQ(str, "No.1");
+
+    str = bq.Back();
+    ASSERT_EQ(str, "No." + std::to_string(count));
+
+    for(size_t i = 1; i <= count; i++) {
+        std::string id = "No." + std::to_string(i);
+        str = bq.Take();
+        ASSERT_EQ(id, str);
+    }
+
+    ASSERT_EQ(bq.Size(), 0);
+}
+
+TEST(UtilTest, LOG_TEST) {
+    int32_t res = server::InitLog(LOG_FILE_PATH);
+    ASSERT_EQ(res, 0);
+
+    std::string fname = server::GetFileName(LOG_FILE_PATH);
+    ASSERT_EQ(fname, "log_config.conf");
+}

cpp/unittest/storage/S3ClientTest.cpp

@@ -1,66 +1,66 @@
-////////////////////////////////////////////////////////////////////////////////
-// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved
-// Unauthorized copying of this file, via any medium is strictly prohibited.
-// Proprietary and confidential.
-////////////////////////////////////////////////////////////////////////////////
-
-#include "storage/IStorage.h"
-#include "storage/s3/S3ClientWrapper.h"
-#include <gtest/gtest.h>
-#include <memory.h>
-#include <fstream>
-
-
-using namespace zilliz::milvus::engine;
-
-TEST(s3_client_wrapper, CLIENT_WRAPPER_TEST) {
-
-    std::shared_ptr<storage::IStorage> storage_ptr = std::make_shared<storage::S3ClientWrapper>();
-
-    std::string ip_address = "127.0.0.1";
-    std::string port = "9000";
-    std::string access_key = "AKIAIOSFODNN7EXAMPLE";
-    std::string secret_key = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY";
-
-    Status status = storage_ptr->Create(ip_address, port, access_key, secret_key);
-    ASSERT_TRUE(status.ok());
-
-    std::string filename = "/tmp/s3_test_file";
-    std::string bucket_name = "bucktname";
-    std::string object_name = "test_file";
-
-    status = storage_ptr->CreateBucket(bucket_name);
-    std::cout << status.IsAlreadyExist() << std::endl;
-    if (status.IsAlreadyExist()) {
-        status = storage_ptr->DeleteBucket(bucket_name);
-        status = storage_ptr->CreateBucket(bucket_name);
-    }
-
-    ASSERT_TRUE(status.ok());
-
-    std::ofstream ofile(filename);
-    std::stringstream ss;
-    for (int i = 0; i < 1024; ++i) {
-        ss << i;
-    }
-    ofile << ss.str() << std::endl;
-    ofile.close();
-    status = storage_ptr->UploadFile(bucket_name, object_name, filename);
-    ASSERT_TRUE(status.ok());
-
-    status = storage_ptr->DownloadFile(bucket_name, object_name, filename);
-    std::ifstream infile(filename);
-    std::string in_buffer;
-    infile >> in_buffer;
-    ASSERT_STREQ(in_buffer.c_str(), ss.str().c_str());
-
-    status = storage_ptr->DeleteFile(bucket_name, object_name);
-    ASSERT_TRUE(status.ok());
-
-    status = storage_ptr->DeleteBucket(bucket_name);
-    ASSERT_TRUE(status.ok());
-
-    status = storage_ptr->Close();
-    ASSERT_TRUE(status.ok());
-}
-
+//////////////////////////////////////////////////////////////////////////////////
+//// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved
+//// Unauthorized copying of this file, via any medium is strictly prohibited.
+//// Proprietary and confidential.
+//////////////////////////////////////////////////////////////////////////////////
+//
+//#include "storage/IStorage.h"
+//#include "storage/s3/S3ClientWrapper.h"
+//#include <gtest/gtest.h>
+//#include <memory.h>
+//#include <fstream>
+//
+//
+//using namespace zilliz::milvus::engine;
+//
+//TEST(s3_client_wrapper, CLIENT_WRAPPER_TEST) {
+//
+//    std::shared_ptr<storage::IStorage> storage_ptr = std::make_shared<storage::S3ClientWrapper>();
+//
+//    std::string ip_address = "127.0.0.1";
+//    std::string port = "9000";
+//    std::string access_key = "AKIAIOSFODNN7EXAMPLE";
+//    std::string secret_key = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY";
+//
+//    Status status = storage_ptr->Create(ip_address, port, access_key, secret_key);
+//    ASSERT_TRUE(status.ok());
+//
+//    std::string filename = "/tmp/s3_test_file";
+//    std::string bucket_name = "bucktname";
+//    std::string object_name = "test_file";
+//
+//    status = storage_ptr->CreateBucket(bucket_name);
+//    std::cout << status.IsAlreadyExist() << std::endl;
+//    if (status.IsAlreadyExist()) {
+//        status = storage_ptr->DeleteBucket(bucket_name);
+//        status = storage_ptr->CreateBucket(bucket_name);
+//    }
+//
+//    ASSERT_TRUE(status.ok());
+//
+//    std::ofstream ofile(filename);
+//    std::stringstream ss;
+//    for (int i = 0; i < 1024; ++i) {
+//        ss << i;
+//    }
+//    ofile << ss.str() << std::endl;
+//    ofile.close();
+//    status = storage_ptr->UploadFile(bucket_name, object_name, filename);
+//    ASSERT_TRUE(status.ok());
+//
+//    status = storage_ptr->DownloadFile(bucket_name, object_name, filename);
+//    std::ifstream infile(filename);
+//    std::string in_buffer;
+//    infile >> in_buffer;
+//    ASSERT_STREQ(in_buffer.c_str(), ss.str().c_str());
+//
+//    status = storage_ptr->DeleteFile(bucket_name, object_name);
+//    ASSERT_TRUE(status.ok());
+//
+//    status = storage_ptr->DeleteBucket(bucket_name);
+//    ASSERT_TRUE(status.ok());
+//
+//    status = storage_ptr->Close();
+//    ASSERT_TRUE(status.ok());
+//}
+//