diff --git a/CHANGELOG.md b/CHANGELOG.md index 38615bbeff..c6af48271b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ Please mark all change in change log and use the issue from GitHub - \#715 - Milvus crash when searching and building index simultaneously using SQ8H - \#744 - Don't return partition table for show_tables - \#770 - Server unittest run failed on low-end server +- \#805 - IVFTest.gpu_seal_test unittest failed - \#831 - Judge branch error in CommonUtil.cpp ## Feature @@ -17,6 +18,8 @@ Please mark all change in change log and use the issue from GitHub - \#766 - If partition tag is similar, wrong partition is searched - \#771 - Add server build commit info interface - \#759 - Put C++ sdk out of milvus/core +- \#815 - Support MinIO storage +- \#910 - Change Milvus c++ standard to c++17 ## Improvement - \#738 - Use Openblas / lapack from apt install @@ -24,6 +27,8 @@ Please mark all change in change log and use the issue from GitHub - \#791 - Remove Arrow - \#834 - add cpu mode for built-in Faiss - \#848 - Add ready-to-use config files to the Milvus repo for enhanced user experince +- \#860 - Remove redundant checks in CacheMgr's constructor +- \#908 - Move "primary_path" and "secondary_path" to storage config ## Task diff --git a/core/CMakeLists.txt b/core/CMakeLists.txt index b4b221a74b..0e99d64e82 100644 --- a/core/CMakeLists.txt +++ b/core/CMakeLists.txt @@ -106,7 +106,7 @@ message(STATUS "Milvus version: " "${MILVUS_VERSION_MAJOR}.${MILVUS_VERSION_MINOR}.${MILVUS_VERSION_PATCH} " "(full: '${MILVUS_VERSION}')") -set(CMAKE_CXX_STANDARD 14) +set(CMAKE_CXX_STANDARD 17) set(CMAKE_CXX_STANDARD_REQUIRED on) if (CMAKE_SYSTEM_PROCESSOR MATCHES "(x86)|(X86)|(amd64)|(AMD64)") diff --git a/core/centos7_build_deps.sh b/core/centos7_build_deps.sh index 9cc5a25b66..c980d7560c 100755 --- a/core/centos7_build_deps.sh +++ b/core/centos7_build_deps.sh @@ -4,7 +4,7 @@ sudo yum install -y epel-release centos-release-scl-rh && sudo yum install -y wg sudo wget -qO- "https://cmake.org/files/v3.14/cmake-3.14.3-Linux-x86_64.tar.gz" | sudo tar --strip-components=1 -xz -C /usr/local && \ sudo yum install -y ccache make automake git python3-pip libcurl-devel python3-devel boost-static mysql-devel \ devtoolset-7-gcc devtoolset-7-gcc-c++ devtoolset-7-gcc-gfortran llvm-toolset-7.0-clang llvm-toolset-7.0-clang-tools-extra lcov \ -openblas-devel lapack-devel +openblas-devel lapack-devel openssl-devel echo "source scl_source enable devtoolset-7" | sudo tee -a /etc/profile.d/devtoolset-7.sh echo "source scl_source enable llvm-toolset-7.0" | sudo tee -a /etc/profile.d/llvm-toolset-7.sh diff --git a/core/cmake/DefineOptions.cmake b/core/cmake/DefineOptions.cmake index 164856866b..c51edcf835 100644 --- a/core/cmake/DefineOptions.cmake +++ b/core/cmake/DefineOptions.cmake @@ -89,6 +89,8 @@ define_option(MILVUS_WITH_OPENTRACING "Build with Opentracing" ON) define_option(MILVUS_WITH_FIU "Build with fiu" OFF) +define_option(MILVUS_WITH_AWS "Build with aws" ON) + #---------------------------------------------------------------------- set_option_category("Test and benchmark") diff --git a/core/cmake/ThirdPartyPackages.cmake b/core/cmake/ThirdPartyPackages.cmake index 6076c12157..2245495040 100644 --- a/core/cmake/ThirdPartyPackages.cmake +++ b/core/cmake/ThirdPartyPackages.cmake @@ -27,7 +27,8 @@ set(MILVUS_THIRDPARTY_DEPENDENCIES GRPC ZLIB Opentracing - fiu) + fiu + AWS) message(STATUS "Using ${MILVUS_DEPENDENCY_SOURCE} approach to find dependencies") @@ -63,6 +64,8 @@ macro(build_dependency DEPENDENCY_NAME) build_opentracing() elseif ("${DEPENDENCY_NAME}" STREQUAL "fiu") build_fiu() + elseif("${DEPENDENCY_NAME}" STREQUAL "AWS") + build_aws() else () message(FATAL_ERROR "Unknown thirdparty dependency to build: ${DEPENDENCY_NAME}") endif () @@ -327,6 +330,11 @@ else () "https://gitee.com/quicksilver/libfiu/repository/archive/${FIU_VERSION}.zip") endif () +if (DEFINED ENV{MILVUS_AWS_URL}) + set(AWS_SOURCE_URL "$ENV{MILVUS_AWS_URL}") +else () + set(AWS_SOURCE_URL "https://github.com/aws/aws-sdk-cpp/archive/${AWS_VERSION}.tar.gz") +endif () # ---------------------------------------------------------------------- # Google gtest @@ -1004,3 +1012,101 @@ resolve_dependency(fiu) get_target_property(FIU_INCLUDE_DIR fiu INTERFACE_INCLUDE_DIRECTORIES) include_directories(SYSTEM ${FIU_INCLUDE_DIR}) + +# ---------------------------------------------------------------------- +# aws +macro(build_aws) + message(STATUS "Building aws-${AWS_VERSION} from source") + set(AWS_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/aws_ep-prefix/src/aws_ep") + + set(AWS_CMAKE_ARGS + ${EP_COMMON_TOOLCHAIN} + "-DCMAKE_INSTALL_PREFIX=${AWS_PREFIX}" + -DCMAKE_BUILD_TYPE=Release + -DCMAKE_INSTALL_LIBDIR=lib + -DBUILD_ONLY=s3 + -DBUILD_SHARED_LIBS=off + -DENABLE_TESTING=off + -DENABLE_UNITY_BUILD=on + -DNO_ENCRYPTION=off) + + set(AWS_CPP_SDK_CORE_STATIC_LIB + "${AWS_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}aws-cpp-sdk-core${CMAKE_STATIC_LIBRARY_SUFFIX}") + set(AWS_CPP_SDK_S3_STATIC_LIB + "${AWS_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}aws-cpp-sdk-s3${CMAKE_STATIC_LIBRARY_SUFFIX}") + set(AWS_INCLUDE_DIR "${AWS_PREFIX}/include") + set(AWS_CMAKE_ARGS + ${AWS_CMAKE_ARGS} + -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} + -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} + -DCMAKE_C_FLAGS=${EP_C_FLAGS} + -DCMAKE_CXX_FLAGS=${EP_CXX_FLAGS}) + + externalproject_add(aws_ep + ${EP_LOG_OPTIONS} + CMAKE_ARGS + ${AWS_CMAKE_ARGS} + BUILD_COMMAND + ${MAKE} + ${MAKE_BUILD_ARGS} + INSTALL_DIR + ${AWS_PREFIX} + URL + ${AWS_SOURCE_URL} + BUILD_BYPRODUCTS + "${AWS_CPP_SDK_S3_STATIC_LIB}" + "${AWS_CPP_SDK_CORE_STATIC_LIB}") + + file(MAKE_DIRECTORY "${AWS_INCLUDE_DIR}") + add_library(aws-cpp-sdk-s3 STATIC IMPORTED) + add_library(aws-cpp-sdk-core STATIC IMPORTED) + + set_target_properties(aws-cpp-sdk-s3 + PROPERTIES + IMPORTED_LOCATION "${AWS_CPP_SDK_S3_STATIC_LIB}" + INTERFACE_INCLUDE_DIRECTORIES "${AWS_INCLUDE_DIR}" + ) + + set_target_properties(aws-cpp-sdk-core + PROPERTIES + IMPORTED_LOCATION "${AWS_CPP_SDK_CORE_STATIC_LIB}" + INTERFACE_INCLUDE_DIRECTORIES "${AWS_INCLUDE_DIR}" + ) + + if(REDHAT_FOUND) + set_target_properties(aws-cpp-sdk-s3 + PROPERTIES + INTERFACE_LINK_LIBRARIES + "${AWS_PREFIX}/lib64/libaws-c-event-stream.a;${AWS_PREFIX}/lib64/libaws-checksums.a;${AWS_PREFIX}/lib64/libaws-c-common.a") + set_target_properties(aws-cpp-sdk-core + PROPERTIES + INTERFACE_LINK_LIBRARIES + "${AWS_PREFIX}/lib64/libaws-c-event-stream.a;${AWS_PREFIX}/lib64/libaws-checksums.a;${AWS_PREFIX}/lib64/libaws-c-common.a") + else() + set_target_properties(aws-cpp-sdk-s3 + PROPERTIES + INTERFACE_LINK_LIBRARIES + "${AWS_PREFIX}/lib/libaws-c-event-stream.a;${AWS_PREFIX}/lib/libaws-checksums.a;${AWS_PREFIX}/lib/libaws-c-common.a") + set_target_properties(aws-cpp-sdk-core + PROPERTIES + INTERFACE_LINK_LIBRARIES + "${AWS_PREFIX}/lib/libaws-c-event-stream.a;${AWS_PREFIX}/lib/libaws-checksums.a;${AWS_PREFIX}/lib/libaws-c-common.a") + endif() + + add_dependencies(aws-cpp-sdk-s3 aws_ep) + add_dependencies(aws-cpp-sdk-core aws_ep) + +endmacro() + +if(MILVUS_WITH_AWS) + resolve_dependency(AWS) + + link_directories(SYSTEM ${AWS_PREFIX}/lib) + + get_target_property(AWS_CPP_SDK_S3_INCLUDE_DIR aws-cpp-sdk-s3 INTERFACE_INCLUDE_DIRECTORIES) + include_directories(SYSTEM ${AWS_CPP_SDK_S3_INCLUDE_DIR}) + + get_target_property(AWS_CPP_SDK_CORE_INCLUDE_DIR aws-cpp-sdk-core INTERFACE_INCLUDE_DIRECTORIES) + include_directories(SYSTEM ${AWS_CPP_SDK_CORE_INCLUDE_DIR}) + +endif() diff --git a/core/conf/demo/server_config.yaml b/core/conf/demo/server_config.yaml index 11ce92c4be..e26ac8413b 100644 --- a/core/conf/demo/server_config.yaml +++ b/core/conf/demo/server_config.yaml @@ -38,12 +38,6 @@ server_config: #----------------------+------------------------------------------------------------+------------+-----------------+ # DataBase Config | Description | Type | Default | #----------------------+------------------------------------------------------------+------------+-----------------+ -# primary_path | Primary directory used to save meta data, vector data and | Path | /var/lib/milvus | -# | index data. | | | -#----------------------+------------------------------------------------------------+------------+-----------------+ -# secondary_path | A semicolon-separated list of secondary directories used | Path | | -# | to save vector data and index data. | | | -#----------------------+------------------------------------------------------------+------------+-----------------+ # backend_url | URL for metadata storage, using SQLite (for single server | URL | sqlite://:@:/ | # | Milvus) or MySQL (for distributed cluster Milvus). | | | # | Format: dialect://username:password@host:port/database | | | @@ -59,12 +53,41 @@ server_config: # | '*' means preload all existing tables. | | | #----------------------+------------------------------------------------------------+------------+-----------------+ db_config: - primary_path: /var/lib/milvus - secondary_path: backend_url: sqlite://:@:/ insert_buffer_size: 1 preload_table: +#----------------------+------------------------------------------------------------+------------+-----------------+ +# Storage Config | Description | Type | Default | +#----------------------+------------------------------------------------------------+------------+-----------------+ +# primary_path | Primary directory used to save meta data, vector data and | Path | /var/lib/milvus | +# | index data. | | | +#----------------------+------------------------------------------------------------+------------+-----------------+ +# secondary_path | A semicolon-separated list of secondary directories used | Path | | +# | to save vector data and index data. | | | +#----------------------+------------------------------------------------------------+------------+-----------------+ +# minio_enable | Enable MinIO storage or not. | Boolean | false | +#----------------------+------------------------------------------------------------+------------+-----------------+ +# minio_address | MinIO storage service IP address. | String | 127.0.0.1 | +#----------------------+------------------------------------------------------------+------------+-----------------+ +# minio_port | MinIO storage service port. Port range (1024, 65535) | Integer | 9000 | +#----------------------+------------------------------------------------------------+------------+-----------------+ +# minio_access_key | MinIO storage service access key. | String | minioadmin | +#----------------------+------------------------------------------------------------+------------+-----------------+ +# minio_secret_key | MinIO storage service secret key. | String | minioadmin | +#----------------------+------------------------------------------------------------+------------+-----------------+ +# minio_bucket | MinIO storage service bucket name. | String | milvus-bucket | +#----------------------+------------------------------------------------------------+------------+-----------------+ +storage_config: + primary_path: /var/lib/milvus + secondary_path: + minio_enable: false + minio_address: 127.0.0.1 + minio_port: 9000 + minio_access_key: minioadmin + minio_secret_key: minioadmin + minio_bucket: milvus-bucket + #----------------------+------------------------------------------------------------+------------+-----------------+ # Metric Config | Description | Type | Default | #----------------------+------------------------------------------------------------+------------+-----------------+ diff --git a/core/conf/server_cpu_config.template b/core/conf/server_cpu_config.template index 1dd82464a5..0f56436774 100644 --- a/core/conf/server_cpu_config.template +++ b/core/conf/server_cpu_config.template @@ -38,12 +38,6 @@ server_config: #----------------------+------------------------------------------------------------+------------+-----------------+ # DataBase Config | Description | Type | Default | #----------------------+------------------------------------------------------------+------------+-----------------+ -# primary_path | Primary directory used to save meta data, vector data and | Path | /var/lib/milvus | -# | index data. | | | -#----------------------+------------------------------------------------------------+------------+-----------------+ -# secondary_path | A semicolon-separated list of secondary directories used | Path | | -# | to save vector data and index data. | | | -#----------------------+------------------------------------------------------------+------------+-----------------+ # backend_url | URL for metadata storage, using SQLite (for single server | URL | sqlite://:@:/ | # | Milvus) or MySQL (for distributed cluster Milvus). | | | # | Format: dialect://username:password@host:port/database | | | @@ -59,12 +53,41 @@ server_config: # | '*' means preload all existing tables. | | | #----------------------+------------------------------------------------------------+------------+-----------------+ db_config: - primary_path: @MILVUS_DB_PATH@ - secondary_path: backend_url: sqlite://:@:/ insert_buffer_size: 1 preload_table: +#----------------------+------------------------------------------------------------+------------+-----------------+ +# Storage Config | Description | Type | Default | +#----------------------+------------------------------------------------------------+------------+-----------------+ +# primary_path | Primary directory used to save meta data, vector data and | Path | /var/lib/milvus | +# | index data. | | | +#----------------------+------------------------------------------------------------+------------+-----------------+ +# secondary_path | A semicolon-separated list of secondary directories used | Path | | +# | to save vector data and index data. | | | +#----------------------+------------------------------------------------------------+------------+-----------------+ +# minio_enable | Enable MinIO storage or not. | Boolean | false | +#----------------------+------------------------------------------------------------+------------+-----------------+ +# minio_address | MinIO storage service IP address. | String | 127.0.0.1 | +#----------------------+------------------------------------------------------------+------------+-----------------+ +# minio_port | MinIO storage service port. Port range (1024, 65535) | Integer | 9000 | +#----------------------+------------------------------------------------------------+------------+-----------------+ +# minio_access_key | MinIO storage service access key. | String | minioadmin | +#----------------------+------------------------------------------------------------+------------+-----------------+ +# minio_secret_key | MinIO storage service secret key. | String | minioadmin | +#----------------------+------------------------------------------------------------+------------+-----------------+ +# minio_bucket | MinIO storage service bucket name. | String | milvus-bucket | +#----------------------+------------------------------------------------------------+------------+-----------------+ +storage_config: + primary_path: @MILVUS_DB_PATH@ + secondary_path: + minio_enable: false + minio_address: 127.0.0.1 + minio_port: 9000 + minio_access_key: minioadmin + minio_secret_key: minioadmin + minio_bucket: milvus-bucket + #----------------------+------------------------------------------------------------+------------+-----------------+ # Metric Config | Description | Type | Default | #----------------------+------------------------------------------------------------+------------+-----------------+ diff --git a/core/conf/server_gpu_config.template b/core/conf/server_gpu_config.template index 401a89d894..e622310a97 100644 --- a/core/conf/server_gpu_config.template +++ b/core/conf/server_gpu_config.template @@ -38,12 +38,6 @@ server_config: #----------------------+------------------------------------------------------------+------------+-----------------+ # DataBase Config | Description | Type | Default | #----------------------+------------------------------------------------------------+------------+-----------------+ -# primary_path | Primary directory used to save meta data, vector data and | Path | /var/lib/milvus | -# | index data. | | | -#----------------------+------------------------------------------------------------+------------+-----------------+ -# secondary_path | A semicolon-separated list of secondary directories used | Path | | -# | to save vector data and index data. | | | -#----------------------+------------------------------------------------------------+------------+-----------------+ # backend_url | URL for metadata storage, using SQLite (for single server | URL | sqlite://:@:/ | # | Milvus) or MySQL (for distributed cluster Milvus). | | | # | Format: dialect://username:password@host:port/database | | | @@ -59,12 +53,41 @@ server_config: # | '*' means preload all existing tables. | | | #----------------------+------------------------------------------------------------+------------+-----------------+ db_config: - primary_path: @MILVUS_DB_PATH@ - secondary_path: backend_url: sqlite://:@:/ insert_buffer_size: 1 preload_table: +#----------------------+------------------------------------------------------------+------------+-----------------+ +# Storage Config | Description | Type | Default | +#----------------------+------------------------------------------------------------+------------+-----------------+ +# primary_path | Primary directory used to save meta data, vector data and | Path | /var/lib/milvus | +# | index data. | | | +#----------------------+------------------------------------------------------------+------------+-----------------+ +# secondary_path | A semicolon-separated list of secondary directories used | Path | | +# | to save vector data and index data. | | | +#----------------------+------------------------------------------------------------+------------+-----------------+ +# minio_enable | Enable MinIO storage or not. | Boolean | false | +#----------------------+------------------------------------------------------------+------------+-----------------+ +# minio_address | MinIO storage service IP address. | String | 127.0.0.1 | +#----------------------+------------------------------------------------------------+------------+-----------------+ +# minio_port | MinIO storage service port. Port range (1024, 65535) | Integer | 9000 | +#----------------------+------------------------------------------------------------+------------+-----------------+ +# minio_access_key | MinIO storage service access key. | String | minioadmin | +#----------------------+------------------------------------------------------------+------------+-----------------+ +# minio_secret_key | MinIO storage service secret key. | String | minioadmin | +#----------------------+------------------------------------------------------------+------------+-----------------+ +# minio_bucket | MinIO storage service bucket name. | String | milvus-bucket | +#----------------------+------------------------------------------------------------+------------+-----------------+ +storage_config: + primary_path: @MILVUS_DB_PATH@ + secondary_path: + minio_enable: false + minio_address: 127.0.0.1 + minio_port: 9000 + minio_access_key: minioadmin + minio_secret_key: minioadmin + minio_bucket: milvus-bucket + #----------------------+------------------------------------------------------------+------------+-----------------+ # Metric Config | Description | Type | Default | #----------------------+------------------------------------------------------------+------------+-----------------+ diff --git a/core/src/CMakeLists.txt b/core/src/CMakeLists.txt index 5e41d8bb09..7b3799550e 100644 --- a/core/src/CMakeLists.txt +++ b/core/src/CMakeLists.txt @@ -92,6 +92,15 @@ set(grpc_server_files ${grpc_interceptor_files} ) +aux_source_directory(${MILVUS_ENGINE_SRC}/storage storage_main_files) +aux_source_directory(${MILVUS_ENGINE_SRC}/storage/file storage_file_files) +aux_source_directory(${MILVUS_ENGINE_SRC}/storage/s3 storage_s3_files) +set(storage_files + ${storage_main_files} + ${storage_file_files} + ${storage_s3_files} + ) + aux_source_directory(${MILVUS_ENGINE_SRC}/utils utils_files) aux_source_directory(${MILVUS_ENGINE_SRC}/wrapper wrapper_files) @@ -105,6 +114,7 @@ set(engine_files ${db_insert_files} ${db_meta_files} ${metrics_files} + ${storage_files} ${thirdparty_files} ${utils_files} ${wrapper_files} @@ -135,6 +145,11 @@ set(boost_lib libboost_serialization.a ) +set(s3_client_lib + aws-cpp-sdk-s3 + aws-cpp-sdk-core + ) + set(third_party_libs sqlite ${grpc_lib} @@ -143,6 +158,9 @@ set(third_party_libs zlib fiu ${boost_lib} + ${s3_client_lib} + curl + crypto ) if (MILVUS_GPU_VERSION) diff --git a/core/src/cache/CpuCacheMgr.cpp b/core/src/cache/CpuCacheMgr.cpp index 1f560cbf8d..0a685c6c1e 100644 --- a/core/src/cache/CpuCacheMgr.cpp +++ b/core/src/cache/CpuCacheMgr.cpp @@ -29,28 +29,17 @@ constexpr int64_t unit = 1024 * 1024 * 1024; } CpuCacheMgr::CpuCacheMgr() { + // All config values have been checked in Config::ValidateConfig() server::Config& config = server::Config::GetInstance(); - Status s; int64_t cpu_cache_cap; - s = config.GetCacheConfigCpuCacheCapacity(cpu_cache_cap); - if (!s.ok()) { - SERVER_LOG_ERROR << s.message(); - } + config.GetCacheConfigCpuCacheCapacity(cpu_cache_cap); int64_t cap = cpu_cache_cap * unit; cache_ = std::make_shared>(cap, 1UL << 32); float cpu_cache_threshold; - s = config.GetCacheConfigCpuCacheThreshold(cpu_cache_threshold); - if (!s.ok()) { - SERVER_LOG_ERROR << s.message(); - } - if (cpu_cache_threshold > 0.0 && cpu_cache_threshold <= 1.0) { - cache_->set_freemem_percent(cpu_cache_threshold); - } else { - SERVER_LOG_ERROR << "Invalid cpu_cache_threshold: " << cpu_cache_threshold << ", by default set to " - << cache_->freemem_percent(); - } + config.GetCacheConfigCpuCacheThreshold(cpu_cache_threshold); + cache_->set_freemem_percent(cpu_cache_threshold); } CpuCacheMgr* diff --git a/core/src/cache/GpuCacheMgr.cpp b/core/src/cache/GpuCacheMgr.cpp index 1802fb3935..8437304fc6 100644 --- a/core/src/cache/GpuCacheMgr.cpp +++ b/core/src/cache/GpuCacheMgr.cpp @@ -34,28 +34,17 @@ constexpr int64_t G_BYTE = 1024 * 1024 * 1024; } GpuCacheMgr::GpuCacheMgr() { + // All config values have been checked in Config::ValidateConfig() server::Config& config = server::Config::GetInstance(); - Status s; int64_t gpu_cache_cap; - s = config.GetGpuResourceConfigCacheCapacity(gpu_cache_cap); - if (!s.ok()) { - SERVER_LOG_ERROR << s.message(); - } + config.GetGpuResourceConfigCacheCapacity(gpu_cache_cap); int64_t cap = gpu_cache_cap * G_BYTE; cache_ = std::make_shared>(cap, 1UL << 32); float gpu_mem_threshold; - s = config.GetGpuResourceConfigCacheThreshold(gpu_mem_threshold); - if (!s.ok()) { - SERVER_LOG_ERROR << s.message(); - } - if (gpu_mem_threshold > 0.0 && gpu_mem_threshold <= 1.0) { - cache_->set_freemem_percent(gpu_mem_threshold); - } else { - SERVER_LOG_ERROR << "Invalid gpu_mem_threshold: " << gpu_mem_threshold << ", by default set to " - << cache_->freemem_percent(); - } + config.GetGpuResourceConfigCacheThreshold(gpu_mem_threshold); + cache_->set_freemem_percent(gpu_mem_threshold); } GpuCacheMgr* diff --git a/core/src/db/Utils.cpp b/core/src/db/Utils.cpp index 9689f496cf..62c71caa7c 100644 --- a/core/src/db/Utils.cpp +++ b/core/src/db/Utils.cpp @@ -16,6 +16,8 @@ // under the License. #include "db/Utils.h" +#include "server/Config.h" +#include "storage/s3/S3ClientWrapper.h" #include "utils/CommonUtil.h" #include "utils/Log.h" @@ -36,14 +38,14 @@ const char* TABLES_FOLDER = "/tables/"; uint64_t index_file_counter = 0; std::mutex index_file_counter_mutex; -std::string +static std::string ConstructParentFolder(const std::string& db_path, const meta::TableFileSchema& table_file) { std::string table_path = db_path + TABLES_FOLDER + table_file.table_id_; std::string partition_path = table_path + "/" + std::to_string(table_file.date_); return partition_path; } -std::string +static std::string GetTableFileParentFolder(const DBMetaOptions& options, const meta::TableFileSchema& table_file) { uint64_t path_count = options.slave_paths_.size() + 1; std::string target_path = options.path_; @@ -117,6 +119,20 @@ DeleteTablePath(const DBMetaOptions& options, const std::string& table_id, bool } } + bool minio_enable = false; + server::Config& config = server::Config::GetInstance(); + config.GetStorageConfigMinioEnable(minio_enable); + + if (minio_enable) { + std::string table_path = options.path_ + TABLES_FOLDER + table_id; + + auto storage_inst = milvus::storage::S3ClientWrapper::GetInstance(); + Status stat = storage_inst.DeleteObjects(table_path); + if (!stat.ok()) { + return stat; + } + } + return Status::OK(); } @@ -139,6 +155,16 @@ Status GetTableFilePath(const DBMetaOptions& options, meta::TableFileSchema& table_file) { std::string parent_path = ConstructParentFolder(options.path_, table_file); std::string file_path = parent_path + "/" + table_file.file_id_; + + bool minio_enable = false; + server::Config& config = server::Config::GetInstance(); + config.GetStorageConfigMinioEnable(minio_enable); + if (minio_enable) { + /* need not check file existence */ + table_file.location_ = file_path; + return Status::OK(); + } + if (boost::filesystem::exists(file_path)) { table_file.location_ = file_path; return Status::OK(); diff --git a/core/src/db/meta/MySQLMetaImpl.cpp b/core/src/db/meta/MySQLMetaImpl.cpp index 57cce25c42..66033f389b 100644 --- a/core/src/db/meta/MySQLMetaImpl.cpp +++ b/core/src/db/meta/MySQLMetaImpl.cpp @@ -254,7 +254,7 @@ MySQLMetaImpl::Initialize() { if (!ret) { std::string msg = "Failed to create db directory " + options_.path_; ENGINE_LOG_ERROR << msg; - return Status(DB_META_TRANSACTION_FAILED, msg); + throw Exception(DB_META_TRANSACTION_FAILED, msg); } } diff --git a/core/src/db/meta/SqliteMetaImpl.cpp b/core/src/db/meta/SqliteMetaImpl.cpp index 98a5505dd2..f7e3f3bb24 100644 --- a/core/src/db/meta/SqliteMetaImpl.cpp +++ b/core/src/db/meta/SqliteMetaImpl.cpp @@ -143,7 +143,7 @@ SqliteMetaImpl::Initialize() { if (!ret) { std::string msg = "Failed to create db directory " + options_.path_; ENGINE_LOG_ERROR << msg; - return Status(DB_INVALID_PATH, msg); + throw Exception(DB_INVALID_PATH, msg); } } diff --git a/core/src/index/CMakeLists.txt b/core/src/index/CMakeLists.txt index d020104d6d..6429df14d3 100644 --- a/core/src/index/CMakeLists.txt +++ b/core/src/index/CMakeLists.txt @@ -25,7 +25,7 @@ message(STATUS "Building using CMake version: ${CMAKE_VERSION}") set(KNOWHERE_VERSION "0.6.0") string(REGEX MATCH "^[0-9]+\\.[0-9]+\\.[0-9]+" KNOWHERE_BASE_VERSION "${KNOWHERE_VERSION}") project(knowhere VERSION "${KNOWHERE_BASE_VERSION}" LANGUAGES C CXX) -set(CMAKE_CXX_STANDARD 14) +set(CMAKE_CXX_STANDARD 17) set(KNOWHERE_VERSION_MAJOR "${knowhere_VERSION_MAJOR}") set(KNOWHERE_VERSION_MINOR "${knowhere_VERSION_MINOR}") diff --git a/core/src/index/knowhere/knowhere/index/vector_index/FaissBaseIndex.cpp b/core/src/index/knowhere/knowhere/index/vector_index/FaissBaseIndex.cpp index 89c478b8e3..45d5930137 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/FaissBaseIndex.cpp +++ b/core/src/index/knowhere/knowhere/index/vector_index/FaissBaseIndex.cpp @@ -70,6 +70,7 @@ FaissBaseIndex::SealImpl() { faiss::Index* index = index_.get(); auto idx = dynamic_cast(index); if (idx != nullptr) { + // To be deleted KNOWHERE_LOG_DEBUG << "Test before to_readonly:" << " IVF READONLY " << std::boolalpha << idx->is_readonly(); idx->to_readonly(); diff --git a/core/src/index/knowhere/knowhere/index/vector_index/IndexGPUIVFPQ.cpp b/core/src/index/knowhere/knowhere/index/vector_index/IndexGPUIVFPQ.cpp index cbd4f4f09c..360461a926 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/IndexGPUIVFPQ.cpp +++ b/core/src/index/knowhere/knowhere/index/vector_index/IndexGPUIVFPQ.cpp @@ -50,7 +50,7 @@ GPUIVFPQ::Train(const DatasetPtr& dataset, const Config& config) { host_index.reset(faiss::gpu::index_gpu_to_cpu(device_index)); return std::make_shared(host_index); } else { - KNOWHERE_THROW_MSG("Build IVFSQ can't get gpu resource"); + KNOWHERE_THROW_MSG("Build IVFPQ can't get gpu resource"); } } diff --git a/core/src/index/unittest/test_ivf.cpp b/core/src/index/unittest/test_ivf.cpp index d24fc0197c..8b42706728 100644 --- a/core/src/index/unittest/test_ivf.cpp +++ b/core/src/index/unittest/test_ivf.cpp @@ -265,7 +265,7 @@ TEST_P(IVFTest, clone_test) { #ifdef MILVUS_GPU_VERSION #ifdef CUSTOMIZATION TEST_P(IVFTest, gpu_seal_test) { - std::vector support_idx_vec{"GPUIVF", "GPUIVFSQ", "IVFSQHybrid"}; + std::vector support_idx_vec{"GPUIVF", "GPUIVFSQ"}; auto finder = std::find(support_idx_vec.cbegin(), support_idx_vec.cend(), index_type); if (finder == support_idx_vec.cend()) { return; diff --git a/core/src/main.cpp b/core/src/main.cpp index 4b1d16fde3..33864edcc9 100644 --- a/core/src/main.cpp +++ b/core/src/main.cpp @@ -16,17 +16,14 @@ // under the License. #include -#include -#include #include +#include #include #include #include "easyloggingpp/easylogging++.h" -#include "metrics/Metrics.h" #include "server/Server.h" #include "src/version.h" -#include "utils/CommonUtil.h" #include "utils/SignalUtil.h" INITIALIZE_EASYLOGGINGPP diff --git a/core/src/server/Config.cpp b/core/src/server/Config.cpp index 13c90262a7..cfc3959fb4 100644 --- a/core/src/server/Config.cpp +++ b/core/src/server/Config.cpp @@ -72,309 +72,169 @@ Config::LoadConfigFile(const std::string& filename) { Status Config::ValidateConfig() { - Status s; - std::string config_version; - s = GetConfigVersion(config_version); - if (!s.ok()) { - return s; - } + CONFIG_CHECK(GetConfigVersion(config_version)); /* server config */ std::string server_addr; - s = GetServerConfigAddress(server_addr); - if (!s.ok()) { - return s; - } + CONFIG_CHECK(GetServerConfigAddress(server_addr)); std::string server_port; - s = GetServerConfigPort(server_port); - if (!s.ok()) { - return s; - } + CONFIG_CHECK(GetServerConfigPort(server_port)); std::string server_mode; - s = GetServerConfigDeployMode(server_mode); - if (!s.ok()) { - return s; - } + CONFIG_CHECK(GetServerConfigDeployMode(server_mode)); std::string server_time_zone; - s = GetServerConfigTimeZone(server_time_zone); - if (!s.ok()) { - return s; - } + CONFIG_CHECK(GetServerConfigTimeZone(server_time_zone)); /* db config */ - std::string db_primary_path; - s = GetDBConfigPrimaryPath(db_primary_path); - if (!s.ok()) { - return s; - } - - std::string db_secondary_path; - s = GetDBConfigSecondaryPath(db_secondary_path); - if (!s.ok()) { - return s; - } - std::string db_backend_url; - s = GetDBConfigBackendUrl(db_backend_url); - if (!s.ok()) { - return s; - } + CONFIG_CHECK(GetDBConfigBackendUrl(db_backend_url)); int64_t db_archive_disk_threshold; - s = GetDBConfigArchiveDiskThreshold(db_archive_disk_threshold); - if (!s.ok()) { - return s; - } + CONFIG_CHECK(GetDBConfigArchiveDiskThreshold(db_archive_disk_threshold)); int64_t db_archive_days_threshold; - s = GetDBConfigArchiveDaysThreshold(db_archive_days_threshold); - if (!s.ok()) { - return s; - } + CONFIG_CHECK(GetDBConfigArchiveDaysThreshold(db_archive_days_threshold)); int64_t db_insert_buffer_size; - s = GetDBConfigInsertBufferSize(db_insert_buffer_size); - if (!s.ok()) { - return s; - } + CONFIG_CHECK(GetDBConfigInsertBufferSize(db_insert_buffer_size)); + + /* storage config */ + std::string storage_primary_path; + CONFIG_CHECK(GetStorageConfigPrimaryPath(storage_primary_path)); + + std::string storage_secondary_path; + CONFIG_CHECK(GetStorageConfigSecondaryPath(storage_secondary_path)); + + bool storage_minio_enable; + CONFIG_CHECK(GetStorageConfigMinioEnable(storage_minio_enable)); + std::cout << "MinIO " << (storage_minio_enable ? "ENABLED !" : "DISABLED !") << std::endl; + + std::string storage_minio_address; + CONFIG_CHECK(GetStorageConfigMinioAddress(storage_minio_address)); + + std::string storage_minio_port; + CONFIG_CHECK(GetStorageConfigMinioPort(storage_minio_port)); + + std::string storage_minio_access_key; + CONFIG_CHECK(GetStorageConfigMinioAccessKey(storage_minio_access_key)); + + std::string storage_minio_secret_key; + CONFIG_CHECK(GetStorageConfigMinioSecretKey(storage_minio_secret_key)); + + std::string storage_minio_bucket; + CONFIG_CHECK(GetStorageConfigMinioBucket(storage_minio_bucket)); /* metric config */ bool metric_enable_monitor; - s = GetMetricConfigEnableMonitor(metric_enable_monitor); - if (!s.ok()) { - return s; - } + CONFIG_CHECK(GetMetricConfigEnableMonitor(metric_enable_monitor)); std::string metric_collector; - s = GetMetricConfigCollector(metric_collector); - if (!s.ok()) { - return s; - } + CONFIG_CHECK(GetMetricConfigCollector(metric_collector)); std::string metric_prometheus_port; - s = GetMetricConfigPrometheusPort(metric_prometheus_port); - if (!s.ok()) { - return s; - } + CONFIG_CHECK(GetMetricConfigPrometheusPort(metric_prometheus_port)); /* cache config */ int64_t cache_cpu_cache_capacity; - s = GetCacheConfigCpuCacheCapacity(cache_cpu_cache_capacity); - if (!s.ok()) { - return s; - } + CONFIG_CHECK(GetCacheConfigCpuCacheCapacity(cache_cpu_cache_capacity)); float cache_cpu_cache_threshold; - s = GetCacheConfigCpuCacheThreshold(cache_cpu_cache_threshold); - if (!s.ok()) { - return s; - } + CONFIG_CHECK(GetCacheConfigCpuCacheThreshold(cache_cpu_cache_threshold)); bool cache_insert_data; - s = GetCacheConfigCacheInsertData(cache_insert_data); - if (!s.ok()) { - return s; - } + CONFIG_CHECK(GetCacheConfigCacheInsertData(cache_insert_data)); /* engine config */ int64_t engine_use_blas_threshold; - s = GetEngineConfigUseBlasThreshold(engine_use_blas_threshold); - if (!s.ok()) { - return s; - } + CONFIG_CHECK(GetEngineConfigUseBlasThreshold(engine_use_blas_threshold)); int64_t engine_omp_thread_num; - s = GetEngineConfigOmpThreadNum(engine_omp_thread_num); - if (!s.ok()) { - return s; - } + CONFIG_CHECK(GetEngineConfigOmpThreadNum(engine_omp_thread_num)); #ifdef MILVUS_GPU_VERSION int64_t engine_gpu_search_threshold; - s = GetEngineConfigGpuSearchThreshold(engine_gpu_search_threshold); - if (!s.ok()) { - return s; - } + CONFIG_CHECK(GetEngineConfigGpuSearchThreshold(engine_gpu_search_threshold)); +#endif /* gpu resource config */ +#ifdef MILVUS_GPU_VERSION bool gpu_resource_enable; - s = GetGpuResourceConfigEnable(gpu_resource_enable); - if (!s.ok()) { - return s; - } - + CONFIG_CHECK(GetGpuResourceConfigEnable(gpu_resource_enable)); std::cout << "GPU resources " << (gpu_resource_enable ? "ENABLED !" : "DISABLED !") << std::endl; + if (gpu_resource_enable) { int64_t resource_cache_capacity; - s = GetGpuResourceConfigCacheCapacity(resource_cache_capacity); - if (!s.ok()) { - return s; - } + CONFIG_CHECK(GetGpuResourceConfigCacheCapacity(resource_cache_capacity)); float resource_cache_threshold; - s = GetGpuResourceConfigCacheThreshold(resource_cache_threshold); - if (!s.ok()) { - return s; - } + CONFIG_CHECK(GetGpuResourceConfigCacheThreshold(resource_cache_threshold)); std::vector search_resources; - s = GetGpuResourceConfigSearchResources(search_resources); - if (!s.ok()) { - return s; - } + CONFIG_CHECK(GetGpuResourceConfigSearchResources(search_resources)); std::vector index_build_resources; - s = GetGpuResourceConfigBuildIndexResources(index_build_resources); - if (!s.ok()) { - return s; - } + CONFIG_CHECK(GetGpuResourceConfigBuildIndexResources(index_build_resources)); } #endif /* tracing config */ std::string tracing_config_path; - s = GetTracingConfigJsonConfigPath(tracing_config_path); - if (!s.ok()) { - return s; - } + CONFIG_CHECK(GetTracingConfigJsonConfigPath(tracing_config_path)); return Status::OK(); } Status Config::ResetDefaultConfig() { - Status s; - /* server config */ - s = SetServerConfigAddress(CONFIG_SERVER_ADDRESS_DEFAULT); - if (!s.ok()) { - return s; - } - - s = SetServerConfigPort(CONFIG_SERVER_PORT_DEFAULT); - if (!s.ok()) { - return s; - } - - s = SetServerConfigDeployMode(CONFIG_SERVER_DEPLOY_MODE_DEFAULT); - if (!s.ok()) { - return s; - } - - s = SetServerConfigTimeZone(CONFIG_SERVER_TIME_ZONE_DEFAULT); - if (!s.ok()) { - return s; - } + CONFIG_CHECK(SetServerConfigAddress(CONFIG_SERVER_ADDRESS_DEFAULT)); + CONFIG_CHECK(SetServerConfigPort(CONFIG_SERVER_PORT_DEFAULT)); + CONFIG_CHECK(SetServerConfigDeployMode(CONFIG_SERVER_DEPLOY_MODE_DEFAULT)); + CONFIG_CHECK(SetServerConfigTimeZone(CONFIG_SERVER_TIME_ZONE_DEFAULT)); /* db config */ - s = SetDBConfigPrimaryPath(CONFIG_DB_PRIMARY_PATH_DEFAULT); - if (!s.ok()) { - return s; - } + CONFIG_CHECK(SetDBConfigBackendUrl(CONFIG_DB_BACKEND_URL_DEFAULT)); + CONFIG_CHECK(SetDBConfigArchiveDiskThreshold(CONFIG_DB_ARCHIVE_DISK_THRESHOLD_DEFAULT)); + CONFIG_CHECK(SetDBConfigArchiveDaysThreshold(CONFIG_DB_ARCHIVE_DAYS_THRESHOLD_DEFAULT)); + CONFIG_CHECK(SetDBConfigInsertBufferSize(CONFIG_DB_INSERT_BUFFER_SIZE_DEFAULT)); - s = SetDBConfigSecondaryPath(CONFIG_DB_SECONDARY_PATH_DEFAULT); - if (!s.ok()) { - return s; - } - - s = SetDBConfigBackendUrl(CONFIG_DB_BACKEND_URL_DEFAULT); - if (!s.ok()) { - return s; - } - - s = SetDBConfigArchiveDiskThreshold(CONFIG_DB_ARCHIVE_DISK_THRESHOLD_DEFAULT); - if (!s.ok()) { - return s; - } - - s = SetDBConfigArchiveDaysThreshold(CONFIG_DB_ARCHIVE_DAYS_THRESHOLD_DEFAULT); - if (!s.ok()) { - return s; - } - - s = SetDBConfigInsertBufferSize(CONFIG_DB_INSERT_BUFFER_SIZE_DEFAULT); - if (!s.ok()) { - return s; - } + /* storage config */ + CONFIG_CHECK(SetStorageConfigPrimaryPath(CONFIG_STORAGE_PRIMARY_PATH_DEFAULT)); + CONFIG_CHECK(SetStorageConfigSecondaryPath(CONFIG_STORAGE_SECONDARY_PATH_DEFAULT)); + CONFIG_CHECK(SetStorageConfigMinioEnable(CONFIG_STORAGE_MINIO_ENABLE_DEFAULT)); + CONFIG_CHECK(SetStorageConfigMinioAddress(CONFIG_STORAGE_MINIO_ADDRESS_DEFAULT)); + CONFIG_CHECK(SetStorageConfigMinioPort(CONFIG_STORAGE_MINIO_PORT_DEFAULT)); + CONFIG_CHECK(SetStorageConfigMinioAccessKey(CONFIG_STORAGE_MINIO_ACCESS_KEY_DEFAULT)); + CONFIG_CHECK(SetStorageConfigMinioSecretKey(CONFIG_STORAGE_MINIO_SECRET_KEY_DEFAULT)); + CONFIG_CHECK(SetStorageConfigMinioBucket(CONFIG_STORAGE_MINIO_BUCKET_DEFAULT)); /* metric config */ - s = SetMetricConfigEnableMonitor(CONFIG_METRIC_ENABLE_MONITOR_DEFAULT); - if (!s.ok()) { - return s; - } - - s = SetMetricConfigCollector(CONFIG_METRIC_COLLECTOR_DEFAULT); - if (!s.ok()) { - return s; - } - - s = SetMetricConfigPrometheusPort(CONFIG_METRIC_PROMETHEUS_PORT_DEFAULT); - if (!s.ok()) { - return s; - } + CONFIG_CHECK(SetMetricConfigEnableMonitor(CONFIG_METRIC_ENABLE_MONITOR_DEFAULT)); + CONFIG_CHECK(SetMetricConfigCollector(CONFIG_METRIC_COLLECTOR_DEFAULT)); + CONFIG_CHECK(SetMetricConfigPrometheusPort(CONFIG_METRIC_PROMETHEUS_PORT_DEFAULT)); /* cache config */ - s = SetCacheConfigCpuCacheCapacity(CONFIG_CACHE_CPU_CACHE_CAPACITY_DEFAULT); - if (!s.ok()) { - return s; - } - - s = SetCacheConfigCpuCacheThreshold(CONFIG_CACHE_CPU_CACHE_THRESHOLD_DEFAULT); - if (!s.ok()) { - return s; - } - - s = SetCacheConfigCacheInsertData(CONFIG_CACHE_CACHE_INSERT_DATA_DEFAULT); - if (!s.ok()) { - return s; - } + CONFIG_CHECK(SetCacheConfigCpuCacheCapacity(CONFIG_CACHE_CPU_CACHE_CAPACITY_DEFAULT)); + CONFIG_CHECK(SetCacheConfigCpuCacheThreshold(CONFIG_CACHE_CPU_CACHE_THRESHOLD_DEFAULT)); + CONFIG_CHECK(SetCacheConfigCacheInsertData(CONFIG_CACHE_CACHE_INSERT_DATA_DEFAULT)); /* engine config */ - s = SetEngineConfigUseBlasThreshold(CONFIG_ENGINE_USE_BLAS_THRESHOLD_DEFAULT); - if (!s.ok()) { - return s; - } - - s = SetEngineConfigOmpThreadNum(CONFIG_ENGINE_OMP_THREAD_NUM_DEFAULT); - if (!s.ok()) { - return s; - } - + CONFIG_CHECK(SetEngineConfigUseBlasThreshold(CONFIG_ENGINE_USE_BLAS_THRESHOLD_DEFAULT)); + CONFIG_CHECK(SetEngineConfigOmpThreadNum(CONFIG_ENGINE_OMP_THREAD_NUM_DEFAULT)); #ifdef MILVUS_GPU_VERSION + CONFIG_CHECK(SetEngineConfigGpuSearchThreshold(CONFIG_ENGINE_GPU_SEARCH_THRESHOLD_DEFAULT)); +#endif + /* gpu resource config */ - s = SetEngineConfigGpuSearchThreshold(CONFIG_ENGINE_GPU_SEARCH_THRESHOLD_DEFAULT); - if (!s.ok()) { - return s; - } - - s = SetGpuResourceConfigEnable(CONFIG_GPU_RESOURCE_ENABLE_DEFAULT); - if (!s.ok()) { - return s; - } - - s = SetGpuResourceConfigCacheCapacity(CONFIG_GPU_RESOURCE_CACHE_CAPACITY_DEFAULT); - if (!s.ok()) { - return s; - } - - s = SetGpuResourceConfigCacheThreshold(CONFIG_GPU_RESOURCE_CACHE_THRESHOLD_DEFAULT); - if (!s.ok()) { - return s; - } - - s = SetGpuResourceConfigSearchResources(CONFIG_GPU_RESOURCE_SEARCH_RESOURCES_DEFAULT); - if (!s.ok()) { - return s; - } - - s = SetGpuResourceConfigBuildIndexResources(CONFIG_GPU_RESOURCE_BUILD_INDEX_RESOURCES_DEFAULT); - if (!s.ok()) { - return s; - } +#ifdef MILVUS_GPU_VERSION + CONFIG_CHECK(SetGpuResourceConfigEnable(CONFIG_GPU_RESOURCE_ENABLE_DEFAULT)); + CONFIG_CHECK(SetGpuResourceConfigCacheCapacity(CONFIG_GPU_RESOURCE_CACHE_CAPACITY_DEFAULT)); + CONFIG_CHECK(SetGpuResourceConfigCacheThreshold(CONFIG_GPU_RESOURCE_CACHE_THRESHOLD_DEFAULT)); + CONFIG_CHECK(SetGpuResourceConfigSearchResources(CONFIG_GPU_RESOURCE_SEARCH_RESOURCES_DEFAULT)); + CONFIG_CHECK(SetGpuResourceConfigBuildIndexResources(CONFIG_GPU_RESOURCE_BUILD_INDEX_RESOURCES_DEFAULT)); #endif return Status::OK(); @@ -405,6 +265,8 @@ Config::SetConfigCli(const std::string& parent_key, const std::string& child_key return Status(SERVER_UNSUPPORTED_ERROR, "Not support set server_config"); } else if (parent_key == CONFIG_DB) { return Status(SERVER_UNSUPPORTED_ERROR, "Not support set db_config"); + } else if (parent_key == CONFIG_STORAGE) { + return Status(SERVER_UNSUPPORTED_ERROR, "Not support set storage_config"); } else if (parent_key == CONFIG_METRIC) { return Status(SERVER_UNSUPPORTED_ERROR, "Not support set metric_config"); } else if (parent_key == CONFIG_CACHE) { @@ -488,6 +350,7 @@ Config::CheckConfigVersion(const std::string& value) { return Status::OK(); } +/* server config */ Status Config::CheckServerConfigAddress(const std::string& value) { if (!ValidationUtil::ValidateIpAddress(value).ok()) { @@ -507,7 +370,7 @@ Config::CheckServerConfigPort(const std::string& value) { int32_t port = std::stoi(value); if (!(port > 1024 && port < 65535)) { std::string msg = "Invalid server port: " + value + - ". Possible reason: server_config.port is not in range [1025, 65534]."; + ". Possible reason: server_config.port is not in range (1024, 65535)."; return Status(SERVER_INVALID_ARGUMENT, msg); } } @@ -541,19 +404,7 @@ Config::CheckServerConfigTimeZone(const std::string& value) { return Status::OK(); } -Status -Config::CheckDBConfigPrimaryPath(const std::string& value) { - if (value.empty()) { - return Status(SERVER_INVALID_ARGUMENT, "db_config.db_path is empty."); - } - return Status::OK(); -} - -Status -Config::CheckDBConfigSecondaryPath(const std::string& value) { - return Status::OK(); -} - +/* DB config */ Status Config::CheckDBConfigBackendUrl(const std::string& value) { if (!ValidationUtil::ValidateDbURI(value).ok()) { @@ -610,6 +461,81 @@ Config::CheckDBConfigInsertBufferSize(const std::string& value) { return Status::OK(); } +/* storage config */ +Status +Config::CheckStorageConfigPrimaryPath(const std::string& value) { + if (value.empty()) { + return Status(SERVER_INVALID_ARGUMENT, "storage_config.db_path is empty."); + } + return Status::OK(); +} + +Status +Config::CheckStorageConfigSecondaryPath(const std::string& value) { + return Status::OK(); +} + +Status +Config::CheckStorageConfigMinioEnable(const std::string& value) { + if (!ValidationUtil::ValidateStringIsBool(value).ok()) { + std::string msg = + "Invalid storage config: " + value + ". Possible reason: storage_config.minio_enable is not a boolean."; + return Status(SERVER_INVALID_ARGUMENT, msg); + } + return Status::OK(); +} + +Status +Config::CheckStorageConfigMinioAddress(const std::string& value) { + if (!ValidationUtil::ValidateIpAddress(value).ok()) { + std::string msg = + "Invalid minio address: " + value + ". Possible reason: storage_config.minio_address is invalid."; + return Status(SERVER_INVALID_ARGUMENT, msg); + } + return Status::OK(); +} + +Status +Config::CheckStorageConfigMinioPort(const std::string& value) { + if (!ValidationUtil::ValidateStringIsNumber(value).ok()) { + std::string msg = "Invalid minio port: " + value + ". Possible reason: storage_config.port is not a number."; + return Status(SERVER_INVALID_ARGUMENT, msg); + } else { + int32_t port = std::stoi(value); + if (!(port > 1024 && port < 65535)) { + std::string msg = "Invalid minio port: " + value + + ". Possible reason: storage_config.port is not in range (1024, 65535)."; + return Status(SERVER_INVALID_ARGUMENT, msg); + } + } + return Status::OK(); +} + +Status +Config::CheckStorageConfigMinioAccessKey(const std::string& value) { + if (value.empty()) { + return Status(SERVER_INVALID_ARGUMENT, "storage_config.minio_access_key is empty."); + } + return Status::OK(); +} + +Status +Config::CheckStorageConfigMinioSecretKey(const std::string& value) { + if (value.empty()) { + return Status(SERVER_INVALID_ARGUMENT, "storage_config.minio_secret_key is empty."); + } + return Status::OK(); +} + +Status +Config::CheckStorageConfigMinioBucket(const std::string& value) { + if (value.empty()) { + return Status(SERVER_INVALID_ARGUMENT, "storage_config.minio_bucket is empty."); + } + return Status::OK(); +} + +/* metric config */ Status Config::CheckMetricConfigEnableMonitor(const std::string& value) { if (!ValidationUtil::ValidateStringIsBool(value).ok()) { @@ -633,13 +559,21 @@ Config::CheckMetricConfigCollector(const std::string& value) { Status Config::CheckMetricConfigPrometheusPort(const std::string& value) { if (!ValidationUtil::ValidateStringIsNumber(value).ok()) { - std::string msg = "Invalid metric port: " + value + - ". Possible reason: metric_config.prometheus_config.port is not in range [1025, 65534]."; - return Status(SERVER_INVALID_ARGUMENT, "Invalid metric config prometheus_port: " + value); + std::string msg = "Invalid prometheus port: " + value + + ". Possible reason: metric_config.prometheus_config.port is not a number."; + return Status(SERVER_INVALID_ARGUMENT, msg); + } else { + int32_t port = std::stoi(value); + if (!(port > 1024 && port < 65535)) { + std::string msg = "Invalid prometheus port: " + value + + ". Possible reason: metric_config.prometheus_config.port is not in range (1024, 65535)."; + return Status(SERVER_INVALID_ARGUMENT, msg); + } } return Status::OK(); } +/* cache config */ Status Config::CheckCacheConfigCpuCacheCapacity(const std::string& value) { if (!ValidationUtil::ValidateStringIsNumber(value).ok()) { @@ -665,10 +599,7 @@ Config::CheckCacheConfigCpuCacheCapacity(const std::string& value) { } int64_t buffer_value; - Status s = GetDBConfigInsertBufferSize(buffer_value); - if (!s.ok()) { - return s; - } + CONFIG_CHECK(GetDBConfigInsertBufferSize(buffer_value)); int64_t insert_buffer_size = buffer_value * GB; if (insert_buffer_size + cpu_cache_capacity >= total_mem) { @@ -708,6 +639,7 @@ Config::CheckCacheConfigCacheInsertData(const std::string& value) { return Status::OK(); } +/* engine config */ Status Config::CheckEngineConfigUseBlasThreshold(const std::string& value) { if (!ValidationUtil::ValidateStringIsNumber(value).ok()) { @@ -748,6 +680,7 @@ Config::CheckEngineConfigGpuSearchThreshold(const std::string& value) { return Status::OK(); } +/* gpu resource config */ Status Config::CheckGpuResourceConfigEnable(const std::string& value) { if (!ValidationUtil::ValidateStringIsBool(value).ok()) { @@ -767,10 +700,7 @@ Config::CheckGpuResourceConfigCacheCapacity(const std::string& value) { } else { int64_t gpu_cache_capacity = std::stoll(value) * GB; std::vector gpu_ids; - Status s = GetGpuResourceConfigBuildIndexResources(gpu_ids); - if (!s.ok()) { - return s; - } + CONFIG_CHECK(GetGpuResourceConfigBuildIndexResources(gpu_ids)); for (int64_t gpu_id : gpu_ids) { size_t gpu_memory; @@ -841,10 +771,7 @@ Config::CheckGpuResourceConfigSearchResources(const std::vector& va } for (auto& resource : value) { - auto status = CheckGpuResource(resource); - if (!status.ok()) { - return Status(SERVER_INVALID_ARGUMENT, status.message()); - } + CONFIG_CHECK(CheckGpuResource(resource)); } return Status::OK(); } @@ -859,10 +786,7 @@ Config::CheckGpuResourceConfigBuildIndexResources(const std::vector } for (auto& resource : value) { - auto status = CheckGpuResource(resource); - if (!status.ok()) { - return Status(SERVER_INVALID_ARGUMENT, status.message()); - } + CONFIG_CHECK(CheckGpuResource(resource)); } return Status::OK(); @@ -943,6 +867,7 @@ Config::GetConfigVersion(std::string& value) { return CheckConfigVersion(value); } +/* server config */ Status Config::GetServerConfigAddress(std::string& value) { value = GetConfigStr(CONFIG_SERVER, CONFIG_SERVER_ADDRESS, CONFIG_SERVER_ADDRESS_DEFAULT); @@ -967,18 +892,7 @@ Config::GetServerConfigTimeZone(std::string& value) { return CheckServerConfigTimeZone(value); } -Status -Config::GetDBConfigPrimaryPath(std::string& value) { - value = GetConfigStr(CONFIG_DB, CONFIG_DB_PRIMARY_PATH, CONFIG_DB_PRIMARY_PATH_DEFAULT); - return CheckDBConfigPrimaryPath(value); -} - -Status -Config::GetDBConfigSecondaryPath(std::string& value) { - value = GetConfigStr(CONFIG_DB, CONFIG_DB_SECONDARY_PATH, CONFIG_DB_SECONDARY_PATH_DEFAULT); - return Status::OK(); -} - +/* DB config */ Status Config::GetDBConfigBackendUrl(std::string& value) { value = GetConfigStr(CONFIG_DB, CONFIG_DB_BACKEND_URL, CONFIG_DB_BACKEND_URL_DEFAULT); @@ -989,10 +903,7 @@ Status Config::GetDBConfigArchiveDiskThreshold(int64_t& value) { std::string str = GetConfigStr(CONFIG_DB, CONFIG_DB_ARCHIVE_DISK_THRESHOLD, CONFIG_DB_ARCHIVE_DISK_THRESHOLD_DEFAULT); - Status s = CheckDBConfigArchiveDiskThreshold(str); - if (!s.ok()) { - return s; - } + CONFIG_CHECK(CheckDBConfigArchiveDiskThreshold(str)); value = std::stoll(str); return Status::OK(); } @@ -1001,10 +912,7 @@ Status Config::GetDBConfigArchiveDaysThreshold(int64_t& value) { std::string str = GetConfigStr(CONFIG_DB, CONFIG_DB_ARCHIVE_DAYS_THRESHOLD, CONFIG_DB_ARCHIVE_DAYS_THRESHOLD_DEFAULT); - Status s = CheckDBConfigArchiveDaysThreshold(str); - if (!s.ok()) { - return s; - } + CONFIG_CHECK(CheckDBConfigArchiveDaysThreshold(str)); value = std::stoll(str); return Status::OK(); } @@ -1012,10 +920,7 @@ Config::GetDBConfigArchiveDaysThreshold(int64_t& value) { Status Config::GetDBConfigInsertBufferSize(int64_t& value) { std::string str = GetConfigStr(CONFIG_DB, CONFIG_DB_INSERT_BUFFER_SIZE, CONFIG_DB_INSERT_BUFFER_SIZE_DEFAULT); - Status s = CheckDBConfigInsertBufferSize(str); - if (!s.ok()) { - return s; - } + CONFIG_CHECK(CheckDBConfigInsertBufferSize(str)); value = std::stoll(str); return Status::OK(); } @@ -1026,13 +931,63 @@ Config::GetDBConfigPreloadTable(std::string& value) { return Status::OK(); } +/* storage config */ +Status +Config::GetStorageConfigPrimaryPath(std::string& value) { + value = GetConfigStr(CONFIG_DB, CONFIG_STORAGE_PRIMARY_PATH, CONFIG_STORAGE_PRIMARY_PATH_DEFAULT); + return CheckStorageConfigPrimaryPath(value); +} + +Status +Config::GetStorageConfigSecondaryPath(std::string& value) { + value = GetConfigStr(CONFIG_DB, CONFIG_STORAGE_SECONDARY_PATH, CONFIG_STORAGE_SECONDARY_PATH_DEFAULT); + return CheckStorageConfigSecondaryPath(value); +} + +Status +Config::GetStorageConfigMinioEnable(bool& value) { + std::string str = GetConfigStr(CONFIG_STORAGE, CONFIG_STORAGE_MINIO_ENABLE, CONFIG_STORAGE_MINIO_ENABLE_DEFAULT); + CONFIG_CHECK(CheckStorageConfigMinioEnable(str)); + std::transform(str.begin(), str.end(), str.begin(), ::tolower); + value = (str == "true" || str == "on" || str == "yes" || str == "1"); + return Status::OK(); +} + +Status +Config::GetStorageConfigMinioAddress(std::string& value) { + value = GetConfigStr(CONFIG_STORAGE, CONFIG_STORAGE_MINIO_ADDRESS, CONFIG_STORAGE_MINIO_ADDRESS_DEFAULT); + return CheckStorageConfigMinioAddress(value); +} + +Status +Config::GetStorageConfigMinioPort(std::string& value) { + value = GetConfigStr(CONFIG_STORAGE, CONFIG_STORAGE_MINIO_PORT, CONFIG_STORAGE_MINIO_PORT_DEFAULT); + return CheckStorageConfigMinioPort(value); +} + +Status +Config::GetStorageConfigMinioAccessKey(std::string& value) { + value = GetConfigStr(CONFIG_STORAGE, CONFIG_STORAGE_MINIO_ACCESS_KEY, CONFIG_STORAGE_MINIO_ACCESS_KEY_DEFAULT); + return Status::OK(); +} + +Status +Config::GetStorageConfigMinioSecretKey(std::string& value) { + value = GetConfigStr(CONFIG_STORAGE, CONFIG_STORAGE_MINIO_SECRET_KEY, CONFIG_STORAGE_MINIO_SECRET_KEY_DEFAULT); + return Status::OK(); +} + +Status +Config::GetStorageConfigMinioBucket(std::string& value) { + value = GetConfigStr(CONFIG_STORAGE, CONFIG_STORAGE_MINIO_BUCKET, CONFIG_STORAGE_MINIO_BUCKET_DEFAULT); + return Status::OK(); +} + +/* metric config */ Status Config::GetMetricConfigEnableMonitor(bool& value) { std::string str = GetConfigStr(CONFIG_METRIC, CONFIG_METRIC_ENABLE_MONITOR, CONFIG_METRIC_ENABLE_MONITOR_DEFAULT); - Status s = CheckMetricConfigEnableMonitor(str); - if (!s.ok()) { - return s; - } + CONFIG_CHECK(CheckMetricConfigEnableMonitor(str)); std::transform(str.begin(), str.end(), str.begin(), ::tolower); value = (str == "true" || str == "on" || str == "yes" || str == "1"); return Status::OK(); @@ -1050,14 +1005,12 @@ Config::GetMetricConfigPrometheusPort(std::string& value) { return CheckMetricConfigPrometheusPort(value); } +/* cache config */ Status Config::GetCacheConfigCpuCacheCapacity(int64_t& value) { std::string str = GetConfigStr(CONFIG_CACHE, CONFIG_CACHE_CPU_CACHE_CAPACITY, CONFIG_CACHE_CPU_CACHE_CAPACITY_DEFAULT); - Status s = CheckCacheConfigCpuCacheCapacity(str); - if (!s.ok()) { - return s; - } + CONFIG_CHECK(CheckCacheConfigCpuCacheCapacity(str)); value = std::stoll(str); return Status::OK(); } @@ -1066,10 +1019,7 @@ Status Config::GetCacheConfigCpuCacheThreshold(float& value) { std::string str = GetConfigStr(CONFIG_CACHE, CONFIG_CACHE_CPU_CACHE_THRESHOLD, CONFIG_CACHE_CPU_CACHE_THRESHOLD_DEFAULT); - Status s = CheckCacheConfigCpuCacheThreshold(str); - if (!s.ok()) { - return s; - } + CONFIG_CHECK(CheckCacheConfigCpuCacheThreshold(str)); value = std::stof(str); return Status::OK(); } @@ -1078,23 +1028,18 @@ Status Config::GetCacheConfigCacheInsertData(bool& value) { std::string str = GetConfigStr(CONFIG_CACHE, CONFIG_CACHE_CACHE_INSERT_DATA, CONFIG_CACHE_CACHE_INSERT_DATA_DEFAULT); - Status s = CheckCacheConfigCacheInsertData(str); - if (!s.ok()) { - return s; - } + CONFIG_CHECK(CheckCacheConfigCacheInsertData(str)); std::transform(str.begin(), str.end(), str.begin(), ::tolower); value = (str == "true" || str == "on" || str == "yes" || str == "1"); return Status::OK(); } +/* engine config */ Status Config::GetEngineConfigUseBlasThreshold(int64_t& value) { std::string str = GetConfigStr(CONFIG_ENGINE, CONFIG_ENGINE_USE_BLAS_THRESHOLD, CONFIG_ENGINE_USE_BLAS_THRESHOLD_DEFAULT); - Status s = CheckEngineConfigUseBlasThreshold(str); - if (!s.ok()) { - return s; - } + CONFIG_CHECK(CheckEngineConfigUseBlasThreshold(str)); value = std::stoll(str); return Status::OK(); } @@ -1102,10 +1047,7 @@ Config::GetEngineConfigUseBlasThreshold(int64_t& value) { Status Config::GetEngineConfigOmpThreadNum(int64_t& value) { std::string str = GetConfigStr(CONFIG_ENGINE, CONFIG_ENGINE_OMP_THREAD_NUM, CONFIG_ENGINE_OMP_THREAD_NUM_DEFAULT); - Status s = CheckEngineConfigOmpThreadNum(str); - if (!s.ok()) { - return s; - } + CONFIG_CHECK(CheckEngineConfigOmpThreadNum(str)); value = std::stoll(str); return Status::OK(); } @@ -1115,21 +1057,18 @@ Status Config::GetEngineConfigGpuSearchThreshold(int64_t& value) { std::string str = GetConfigStr(CONFIG_ENGINE, CONFIG_ENGINE_GPU_SEARCH_THRESHOLD, CONFIG_ENGINE_GPU_SEARCH_THRESHOLD_DEFAULT); - Status s = CheckEngineConfigGpuSearchThreshold(str); - if (!s.ok()) { - return s; - } + CONFIG_CHECK(CheckEngineConfigGpuSearchThreshold(str)); value = std::stoll(str); return Status::OK(); } +#endif +/* gpu resource config */ +#ifdef MILVUS_GPU_VERSION Status Config::GetGpuResourceConfigEnable(bool& value) { std::string str = GetConfigStr(CONFIG_GPU_RESOURCE, CONFIG_GPU_RESOURCE_ENABLE, CONFIG_GPU_RESOURCE_ENABLE_DEFAULT); - Status s = CheckGpuResourceConfigEnable(str); - if (!s.ok()) { - return s; - } + CONFIG_CHECK(CheckGpuResourceConfigEnable(str)); std::transform(str.begin(), str.end(), str.begin(), ::tolower); value = (str == "true" || str == "on" || str == "yes" || str == "1"); return Status::OK(); @@ -1138,20 +1077,14 @@ Config::GetGpuResourceConfigEnable(bool& value) { Status Config::GetGpuResourceConfigCacheCapacity(int64_t& value) { bool gpu_resource_enable = false; - Status s = GetGpuResourceConfigEnable(gpu_resource_enable); - if (!s.ok()) { - return s; - } + CONFIG_CHECK(GetGpuResourceConfigEnable(gpu_resource_enable)); if (!gpu_resource_enable) { std::string msg = "GPU not supported. Possible reason: gpu_resource_config.enable is set to false."; return Status(SERVER_UNSUPPORTED_ERROR, msg); } std::string str = GetConfigStr(CONFIG_GPU_RESOURCE, CONFIG_GPU_RESOURCE_CACHE_CAPACITY, CONFIG_GPU_RESOURCE_CACHE_CAPACITY_DEFAULT); - s = CheckGpuResourceConfigCacheCapacity(str); - if (!s.ok()) { - return s; - } + CONFIG_CHECK(CheckGpuResourceConfigCacheCapacity(str)); value = std::stoll(str); return Status::OK(); } @@ -1159,20 +1092,14 @@ Config::GetGpuResourceConfigCacheCapacity(int64_t& value) { Status Config::GetGpuResourceConfigCacheThreshold(float& value) { bool gpu_resource_enable = false; - Status s = GetGpuResourceConfigEnable(gpu_resource_enable); - if (!s.ok()) { - return s; - } + CONFIG_CHECK(GetGpuResourceConfigEnable(gpu_resource_enable)); if (!gpu_resource_enable) { std::string msg = "GPU not supported. Possible reason: gpu_resource_config.enable is set to false."; return Status(SERVER_UNSUPPORTED_ERROR, msg); } std::string str = GetConfigStr(CONFIG_GPU_RESOURCE, CONFIG_GPU_RESOURCE_CACHE_THRESHOLD, CONFIG_GPU_RESOURCE_CACHE_THRESHOLD_DEFAULT); - s = CheckGpuResourceConfigCacheThreshold(str); - if (!s.ok()) { - return s; - } + CONFIG_CHECK(CheckGpuResourceConfigCacheThreshold(str)); value = std::stof(str); return Status::OK(); } @@ -1180,10 +1107,7 @@ Config::GetGpuResourceConfigCacheThreshold(float& value) { Status Config::GetGpuResourceConfigSearchResources(std::vector& value) { bool gpu_resource_enable = false; - Status s = GetGpuResourceConfigEnable(gpu_resource_enable); - if (!s.ok()) { - return s; - } + CONFIG_CHECK(GetGpuResourceConfigEnable(gpu_resource_enable)); if (!gpu_resource_enable) { std::string msg = "GPU not supported. Possible reason: gpu_resource_config.enable is set to false."; return Status(SERVER_UNSUPPORTED_ERROR, msg); @@ -1192,10 +1116,7 @@ Config::GetGpuResourceConfigSearchResources(std::vector& value) { CONFIG_GPU_RESOURCE_DELIMITER, CONFIG_GPU_RESOURCE_SEARCH_RESOURCES_DEFAULT); std::vector res_vec; server::StringHelpFunctions::SplitStringByDelimeter(str, CONFIG_GPU_RESOURCE_DELIMITER, res_vec); - s = CheckGpuResourceConfigSearchResources(res_vec); - if (!s.ok()) { - return s; - } + CONFIG_CHECK(CheckGpuResourceConfigSearchResources(res_vec)); for (std::string& res : res_vec) { value.push_back(std::stoll(res.substr(3))); } @@ -1205,10 +1126,7 @@ Config::GetGpuResourceConfigSearchResources(std::vector& value) { Status Config::GetGpuResourceConfigBuildIndexResources(std::vector& value) { bool gpu_resource_enable = false; - Status s = GetGpuResourceConfigEnable(gpu_resource_enable); - if (!s.ok()) { - return s; - } + CONFIG_CHECK(GetGpuResourceConfigEnable(gpu_resource_enable)); if (!gpu_resource_enable) { std::string msg = "GPU not supported. Possible reason: gpu_resource_config.enable is set to false."; return Status(SERVER_UNSUPPORTED_ERROR, msg); @@ -1218,10 +1136,7 @@ Config::GetGpuResourceConfigBuildIndexResources(std::vector& value) { CONFIG_GPU_RESOURCE_DELIMITER, CONFIG_GPU_RESOURCE_BUILD_INDEX_RESOURCES_DEFAULT); std::vector res_vec; server::StringHelpFunctions::SplitStringByDelimeter(str, CONFIG_GPU_RESOURCE_DELIMITER, res_vec); - s = CheckGpuResourceConfigBuildIndexResources(res_vec); - if (!s.ok()) { - return s; - } + CONFIG_CHECK(CheckGpuResourceConfigBuildIndexResources(res_vec)); for (std::string& res : res_vec) { value.push_back(std::stoll(res.substr(3))); } @@ -1248,253 +1163,196 @@ Config::GetTracingConfigJsonConfigPath(std::string& value) { /* server config */ Status Config::SetServerConfigAddress(const std::string& value) { - Status s = CheckServerConfigAddress(value); - if (!s.ok()) { - return s; - } - SetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_ADDRESS, value); - return Status::OK(); + CONFIG_CHECK(CheckServerConfigAddress(value)); + return SetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_ADDRESS, value); } Status Config::SetServerConfigPort(const std::string& value) { - Status s = CheckServerConfigPort(value); - if (!s.ok()) { - return s; - } - SetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_PORT, value); - return Status::OK(); + CONFIG_CHECK(CheckServerConfigPort(value)); + return SetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_PORT, value); } Status Config::SetServerConfigDeployMode(const std::string& value) { - Status s = CheckServerConfigDeployMode(value); - if (!s.ok()) { - return s; - } - SetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_DEPLOY_MODE, value); - return Status::OK(); + CONFIG_CHECK(CheckServerConfigDeployMode(value)); + return SetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_DEPLOY_MODE, value); } Status Config::SetServerConfigTimeZone(const std::string& value) { - Status s = CheckServerConfigTimeZone(value); - if (!s.ok()) { - return s; - } - SetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_TIME_ZONE, value); - return Status::OK(); + CONFIG_CHECK(CheckServerConfigTimeZone(value)); + return SetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_TIME_ZONE, value); } /* db config */ -Status -Config::SetDBConfigPrimaryPath(const std::string& value) { - Status s = CheckDBConfigPrimaryPath(value); - if (!s.ok()) { - return s; - } - SetConfigValueInMem(CONFIG_DB, CONFIG_DB_PRIMARY_PATH, value); - return Status::OK(); -} - -Status -Config::SetDBConfigSecondaryPath(const std::string& value) { - Status s = CheckDBConfigSecondaryPath(value); - if (!s.ok()) { - return s; - } - SetConfigValueInMem(CONFIG_DB, CONFIG_DB_SECONDARY_PATH, value); - return Status::OK(); -} - Status Config::SetDBConfigBackendUrl(const std::string& value) { - Status s = CheckDBConfigBackendUrl(value); - if (!s.ok()) { - return s; - } - SetConfigValueInMem(CONFIG_DB, CONFIG_DB_BACKEND_URL, value); - return Status::OK(); + CONFIG_CHECK(CheckDBConfigBackendUrl(value)); + return SetConfigValueInMem(CONFIG_DB, CONFIG_DB_BACKEND_URL, value); } Status Config::SetDBConfigArchiveDiskThreshold(const std::string& value) { - Status s = CheckDBConfigArchiveDiskThreshold(value); - if (!s.ok()) { - return s; - } - SetConfigValueInMem(CONFIG_DB, CONFIG_DB_ARCHIVE_DISK_THRESHOLD, value); - return Status::OK(); + CONFIG_CHECK(CheckDBConfigArchiveDiskThreshold(value)); + return SetConfigValueInMem(CONFIG_DB, CONFIG_DB_ARCHIVE_DISK_THRESHOLD, value); } Status Config::SetDBConfigArchiveDaysThreshold(const std::string& value) { - Status s = CheckDBConfigArchiveDaysThreshold(value); - if (!s.ok()) { - return s; - } - SetConfigValueInMem(CONFIG_DB, CONFIG_DB_ARCHIVE_DAYS_THRESHOLD, value); - return Status::OK(); + CONFIG_CHECK(CheckDBConfigArchiveDaysThreshold(value)); + return SetConfigValueInMem(CONFIG_DB, CONFIG_DB_ARCHIVE_DAYS_THRESHOLD, value); } Status Config::SetDBConfigInsertBufferSize(const std::string& value) { - Status s = CheckDBConfigInsertBufferSize(value); - if (!s.ok()) { - return s; - } - SetConfigValueInMem(CONFIG_DB, CONFIG_DB_INSERT_BUFFER_SIZE, value); - return Status::OK(); + CONFIG_CHECK(CheckDBConfigInsertBufferSize(value)); + return SetConfigValueInMem(CONFIG_DB, CONFIG_DB_INSERT_BUFFER_SIZE, value); +} + +/* storage config */ +Status +Config::SetStorageConfigPrimaryPath(const std::string& value) { + CONFIG_CHECK(CheckStorageConfigPrimaryPath(value)); + return SetConfigValueInMem(CONFIG_DB, CONFIG_STORAGE_PRIMARY_PATH, value); +} + +Status +Config::SetStorageConfigSecondaryPath(const std::string& value) { + CONFIG_CHECK(CheckStorageConfigSecondaryPath(value)); + return SetConfigValueInMem(CONFIG_DB, CONFIG_STORAGE_SECONDARY_PATH, value); +} + +Status +Config::SetStorageConfigMinioEnable(const std::string& value) { + CONFIG_CHECK(CheckStorageConfigMinioEnable(value)); + return SetConfigValueInMem(CONFIG_METRIC, CONFIG_METRIC_ENABLE_MONITOR, value); +} + +Status +Config::SetStorageConfigMinioAddress(const std::string& value) { + CONFIG_CHECK(CheckStorageConfigMinioAddress(value)); + return SetConfigValueInMem(CONFIG_STORAGE, CONFIG_STORAGE_MINIO_ADDRESS, value); +} + +Status +Config::SetStorageConfigMinioPort(const std::string& value) { + CONFIG_CHECK(CheckStorageConfigMinioPort(value)); + return SetConfigValueInMem(CONFIG_STORAGE, CONFIG_STORAGE_MINIO_PORT, value); +} + +Status +Config::SetStorageConfigMinioAccessKey(const std::string& value) { + CONFIG_CHECK(CheckStorageConfigMinioAccessKey(value)); + return SetConfigValueInMem(CONFIG_STORAGE, CONFIG_STORAGE_MINIO_ACCESS_KEY, value); +} + +Status +Config::SetStorageConfigMinioSecretKey(const std::string& value) { + CONFIG_CHECK(CheckStorageConfigMinioSecretKey(value)); + return SetConfigValueInMem(CONFIG_STORAGE, CONFIG_STORAGE_MINIO_SECRET_KEY, value); +} + +Status +Config::SetStorageConfigMinioBucket(const std::string& value) { + CONFIG_CHECK(CheckStorageConfigMinioBucket(value)); + return SetConfigValueInMem(CONFIG_STORAGE, CONFIG_STORAGE_MINIO_BUCKET, value); } /* metric config */ Status Config::SetMetricConfigEnableMonitor(const std::string& value) { - Status s = CheckMetricConfigEnableMonitor(value); - if (!s.ok()) { - return s; - } - SetConfigValueInMem(CONFIG_METRIC, CONFIG_METRIC_ENABLE_MONITOR, value); - return Status::OK(); + CONFIG_CHECK(CheckMetricConfigEnableMonitor(value)); + return SetConfigValueInMem(CONFIG_METRIC, CONFIG_METRIC_ENABLE_MONITOR, value); } Status Config::SetMetricConfigCollector(const std::string& value) { - Status s = CheckMetricConfigCollector(value); - if (!s.ok()) { - return s; - } - SetConfigValueInMem(CONFIG_METRIC, CONFIG_METRIC_COLLECTOR, value); - return Status::OK(); + CONFIG_CHECK(CheckMetricConfigCollector(value)); + return SetConfigValueInMem(CONFIG_METRIC, CONFIG_METRIC_COLLECTOR, value); } Status Config::SetMetricConfigPrometheusPort(const std::string& value) { - Status s = CheckMetricConfigPrometheusPort(value); - if (!s.ok()) { - return s; - } - SetConfigValueInMem(CONFIG_METRIC, CONFIG_METRIC_PROMETHEUS_PORT, value); - return Status::OK(); + CONFIG_CHECK(CheckMetricConfigPrometheusPort(value)); + return SetConfigValueInMem(CONFIG_METRIC, CONFIG_METRIC_PROMETHEUS_PORT, value); } /* cache config */ Status Config::SetCacheConfigCpuCacheCapacity(const std::string& value) { - Status s = CheckCacheConfigCpuCacheCapacity(value); - if (!s.ok()) { - return s; - } - SetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_CPU_CACHE_CAPACITY, value); - return Status::OK(); + CONFIG_CHECK(CheckCacheConfigCpuCacheCapacity(value)); + return SetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_CPU_CACHE_CAPACITY, value); } Status Config::SetCacheConfigCpuCacheThreshold(const std::string& value) { - Status s = CheckCacheConfigCpuCacheThreshold(value); - if (!s.ok()) { - return s; - } - SetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_CPU_CACHE_THRESHOLD, value); - return Status::OK(); + CONFIG_CHECK(CheckCacheConfigCpuCacheThreshold(value)); + return SetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_CPU_CACHE_THRESHOLD, value); } Status Config::SetCacheConfigCacheInsertData(const std::string& value) { - Status s = CheckCacheConfigCacheInsertData(value); - if (!s.ok()) { - return s; - } - SetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_CACHE_INSERT_DATA, value); - return Status::OK(); + CONFIG_CHECK(CheckCacheConfigCacheInsertData(value)); + return SetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_CACHE_INSERT_DATA, value); } /* engine config */ Status Config::SetEngineConfigUseBlasThreshold(const std::string& value) { - Status s = CheckEngineConfigUseBlasThreshold(value); - if (!s.ok()) { - return s; - } - SetConfigValueInMem(CONFIG_ENGINE, CONFIG_ENGINE_USE_BLAS_THRESHOLD, value); - return Status::OK(); + CONFIG_CHECK(CheckEngineConfigUseBlasThreshold(value)); + return SetConfigValueInMem(CONFIG_ENGINE, CONFIG_ENGINE_USE_BLAS_THRESHOLD, value); } Status Config::SetEngineConfigOmpThreadNum(const std::string& value) { - Status s = CheckEngineConfigOmpThreadNum(value); - if (!s.ok()) { - return s; - } - SetConfigValueInMem(CONFIG_ENGINE, CONFIG_ENGINE_OMP_THREAD_NUM, value); - return Status::OK(); + CONFIG_CHECK(CheckEngineConfigOmpThreadNum(value)); + return SetConfigValueInMem(CONFIG_ENGINE, CONFIG_ENGINE_OMP_THREAD_NUM, value); } #ifdef MILVUS_GPU_VERSION -/* gpu resource config */ Status Config::SetEngineConfigGpuSearchThreshold(const std::string& value) { - Status s = CheckEngineConfigGpuSearchThreshold(value); - if (!s.ok()) { - return s; - } - SetConfigValueInMem(CONFIG_ENGINE, CONFIG_ENGINE_GPU_SEARCH_THRESHOLD, value); - return Status::OK(); + CONFIG_CHECK(CheckEngineConfigGpuSearchThreshold(value)); + return SetConfigValueInMem(CONFIG_ENGINE, CONFIG_ENGINE_GPU_SEARCH_THRESHOLD, value); } +#endif +/* gpu resource config */ +#ifdef MILVUS_GPU_VERSION Status Config::SetGpuResourceConfigEnable(const std::string& value) { - Status s = CheckGpuResourceConfigEnable(value); - if (!s.ok()) { - return s; - } - SetConfigValueInMem(CONFIG_GPU_RESOURCE, CONFIG_GPU_RESOURCE_ENABLE, value); - return Status::OK(); + CONFIG_CHECK(CheckGpuResourceConfigEnable(value)); + return SetConfigValueInMem(CONFIG_GPU_RESOURCE, CONFIG_GPU_RESOURCE_ENABLE, value); } Status Config::SetGpuResourceConfigCacheCapacity(const std::string& value) { - Status s = CheckGpuResourceConfigCacheCapacity(value); - if (!s.ok()) { - return s; - } - SetConfigValueInMem(CONFIG_GPU_RESOURCE, CONFIG_GPU_RESOURCE_CACHE_CAPACITY, value); - return Status::OK(); + CONFIG_CHECK(CheckGpuResourceConfigCacheCapacity(value)); + return SetConfigValueInMem(CONFIG_GPU_RESOURCE, CONFIG_GPU_RESOURCE_CACHE_CAPACITY, value); } Status Config::SetGpuResourceConfigCacheThreshold(const std::string& value) { - Status s = CheckGpuResourceConfigCacheThreshold(value); - if (!s.ok()) { - return s; - } - SetConfigValueInMem(CONFIG_GPU_RESOURCE, CONFIG_GPU_RESOURCE_CACHE_THRESHOLD, value); - return Status::OK(); + CONFIG_CHECK(CheckGpuResourceConfigCacheThreshold(value)); + return SetConfigValueInMem(CONFIG_GPU_RESOURCE, CONFIG_GPU_RESOURCE_CACHE_THRESHOLD, value); } Status Config::SetGpuResourceConfigSearchResources(const std::string& value) { std::vector res_vec; server::StringHelpFunctions::SplitStringByDelimeter(value, CONFIG_GPU_RESOURCE_DELIMITER, res_vec); - Status s = CheckGpuResourceConfigSearchResources(res_vec); - if (!s.ok()) { - return s; - } - SetConfigValueInMem(CONFIG_GPU_RESOURCE, CONFIG_GPU_RESOURCE_SEARCH_RESOURCES, value); - return Status::OK(); + CONFIG_CHECK(CheckGpuResourceConfigSearchResources(res_vec)); + return SetConfigValueInMem(CONFIG_GPU_RESOURCE, CONFIG_GPU_RESOURCE_SEARCH_RESOURCES, value); } Status Config::SetGpuResourceConfigBuildIndexResources(const std::string& value) { std::vector res_vec; server::StringHelpFunctions::SplitStringByDelimeter(value, CONFIG_GPU_RESOURCE_DELIMITER, res_vec); - Status s = CheckGpuResourceConfigBuildIndexResources(res_vec); - if (!s.ok()) { - return s; - } - SetConfigValueInMem(CONFIG_GPU_RESOURCE, CONFIG_GPU_RESOURCE_BUILD_INDEX_RESOURCES, value); - return Status::OK(); -} // namespace server + CONFIG_CHECK(CheckGpuResourceConfigBuildIndexResources(res_vec)); + return SetConfigValueInMem(CONFIG_GPU_RESOURCE, CONFIG_GPU_RESOURCE_BUILD_INDEX_RESOURCES, value); +} #endif } // namespace server diff --git a/core/src/server/Config.h b/core/src/server/Config.h index e8a10b525f..ebc121d4e3 100644 --- a/core/src/server/Config.h +++ b/core/src/server/Config.h @@ -28,6 +28,14 @@ namespace milvus { namespace server { +#define CONFIG_CHECK(func) \ + do { \ + Status s = func; \ + if (!s.ok()) { \ + return s; \ + } \ + } while (false) + static const char* CONFIG_NODE_DELIMITER = "."; static const char* CONFIG_VERSION = "version"; @@ -44,10 +52,6 @@ static const char* CONFIG_SERVER_TIME_ZONE_DEFAULT = "UTC+8"; /* db config */ static const char* CONFIG_DB = "db_config"; -static const char* CONFIG_DB_PRIMARY_PATH = "primary_path"; -static const char* CONFIG_DB_PRIMARY_PATH_DEFAULT = "/tmp/milvus"; -static const char* CONFIG_DB_SECONDARY_PATH = "secondary_path"; -static const char* CONFIG_DB_SECONDARY_PATH_DEFAULT = ""; static const char* CONFIG_DB_BACKEND_URL = "backend_url"; static const char* CONFIG_DB_BACKEND_URL_DEFAULT = "sqlite://:@:/"; static const char* CONFIG_DB_ARCHIVE_DISK_THRESHOLD = "archive_disk_threshold"; @@ -59,6 +63,25 @@ static const char* CONFIG_DB_INSERT_BUFFER_SIZE_DEFAULT = "1"; static const char* CONFIG_DB_PRELOAD_TABLE = "preload_table"; static const char* CONFIG_DB_PRELOAD_TABLE_DEFAULT = ""; +/* storage config */ +static const char* CONFIG_STORAGE = "storage_config"; +static const char* CONFIG_STORAGE_PRIMARY_PATH = "primary_path"; +static const char* CONFIG_STORAGE_PRIMARY_PATH_DEFAULT = "/tmp/milvus"; +static const char* CONFIG_STORAGE_SECONDARY_PATH = "secondary_path"; +static const char* CONFIG_STORAGE_SECONDARY_PATH_DEFAULT = ""; +static const char* CONFIG_STORAGE_MINIO_ENABLE = "minio_enable"; +static const char* CONFIG_STORAGE_MINIO_ENABLE_DEFAULT = "false"; +static const char* CONFIG_STORAGE_MINIO_ADDRESS = "minio_address"; +static const char* CONFIG_STORAGE_MINIO_ADDRESS_DEFAULT = "127.0.0.1"; +static const char* CONFIG_STORAGE_MINIO_PORT = "minio_port"; +static const char* CONFIG_STORAGE_MINIO_PORT_DEFAULT = "9000"; +static const char* CONFIG_STORAGE_MINIO_ACCESS_KEY = "minio_access_key"; +static const char* CONFIG_STORAGE_MINIO_ACCESS_KEY_DEFAULT = "minioadmin"; +static const char* CONFIG_STORAGE_MINIO_SECRET_KEY = "minio_secret_key"; +static const char* CONFIG_STORAGE_MINIO_SECRET_KEY_DEFAULT = "minioadmin"; +static const char* CONFIG_STORAGE_MINIO_BUCKET = "minio_bucket"; +static const char* CONFIG_STORAGE_MINIO_BUCKET_DEFAULT = "milvus-bucket"; + /* cache config */ static const char* CONFIG_CACHE = "cache_config"; static const char* CONFIG_CACHE_CPU_CACHE_CAPACITY = "cpu_cache_capacity"; @@ -157,10 +180,6 @@ class Config { /* db config */ Status - CheckDBConfigPrimaryPath(const std::string& value); - Status - CheckDBConfigSecondaryPath(const std::string& value); - Status CheckDBConfigBackendUrl(const std::string& value); Status CheckDBConfigArchiveDiskThreshold(const std::string& value); @@ -169,6 +188,24 @@ class Config { Status CheckDBConfigInsertBufferSize(const std::string& value); + /* storage config */ + Status + CheckStorageConfigPrimaryPath(const std::string& value); + Status + CheckStorageConfigSecondaryPath(const std::string& value); + Status + CheckStorageConfigMinioEnable(const std::string& value); + Status + CheckStorageConfigMinioAddress(const std::string& value); + Status + CheckStorageConfigMinioPort(const std::string& value); + Status + CheckStorageConfigMinioAccessKey(const std::string& value); + Status + CheckStorageConfigMinioSecretKey(const std::string& value); + Status + CheckStorageConfigMinioBucket(const std::string& value); + /* metric config */ Status CheckMetricConfigEnableMonitor(const std::string& value); @@ -229,10 +266,6 @@ class Config { /* db config */ Status - GetDBConfigPrimaryPath(std::string& value); - Status - GetDBConfigSecondaryPath(std::string& value); - Status GetDBConfigBackendUrl(std::string& value); Status GetDBConfigArchiveDiskThreshold(int64_t& value); @@ -243,6 +276,24 @@ class Config { Status GetDBConfigPreloadTable(std::string& value); + /* storage config */ + Status + GetStorageConfigPrimaryPath(std::string& value); + Status + GetStorageConfigSecondaryPath(std::string& value); + Status + GetStorageConfigMinioEnable(bool& value); + Status + GetStorageConfigMinioAddress(std::string& value); + Status + GetStorageConfigMinioPort(std::string& value); + Status + GetStorageConfigMinioAccessKey(std::string& value); + Status + GetStorageConfigMinioSecretKey(std::string& value); + Status + GetStorageConfigMinioBucket(std::string& value); + /* metric config */ Status GetMetricConfigEnableMonitor(bool& value); @@ -299,10 +350,6 @@ class Config { /* db config */ Status - SetDBConfigPrimaryPath(const std::string& value); - Status - SetDBConfigSecondaryPath(const std::string& value); - Status SetDBConfigBackendUrl(const std::string& value); Status SetDBConfigArchiveDiskThreshold(const std::string& value); @@ -311,6 +358,24 @@ class Config { Status SetDBConfigInsertBufferSize(const std::string& value); + /* storage config */ + Status + SetStorageConfigPrimaryPath(const std::string& value); + Status + SetStorageConfigSecondaryPath(const std::string& value); + Status + SetStorageConfigMinioEnable(const std::string& value); + Status + SetStorageConfigMinioAddress(const std::string& value); + Status + SetStorageConfigMinioPort(const std::string& value); + Status + SetStorageConfigMinioAccessKey(const std::string& value); + Status + SetStorageConfigMinioSecretKey(const std::string& value); + Status + SetStorageConfigMinioBucket(const std::string& value); + /* metric config */ Status SetMetricConfigEnableMonitor(const std::string& value); diff --git a/core/src/server/DBWrapper.cpp b/core/src/server/DBWrapper.cpp index f7a43d671c..e0b71205df 100644 --- a/core/src/server/DBWrapper.cpp +++ b/core/src/server/DBWrapper.cpp @@ -45,7 +45,7 @@ DBWrapper::StartService() { } std::string path; - s = config.GetDBConfigPrimaryPath(path); + s = config.GetStorageConfigPrimaryPath(path); if (!s.ok()) { std::cerr << s.ToString() << std::endl; return s; @@ -54,7 +54,7 @@ DBWrapper::StartService() { opt.meta_.path_ = path + "/db"; std::string db_slave_path; - s = config.GetDBConfigSecondaryPath(db_slave_path); + s = config.GetStorageConfigSecondaryPath(db_slave_path); if (!s.ok()) { std::cerr << s.ToString() << std::endl; return s; diff --git a/core/src/server/Server.cpp b/core/src/server/Server.cpp index 71f7d09f4a..85aeaccc20 100644 --- a/core/src/server/Server.cpp +++ b/core/src/server/Server.cpp @@ -27,6 +27,7 @@ #include "server/DBWrapper.h" #include "server/grpc_impl/GrpcServer.h" #include "src/version.h" +#include "storage/s3/S3ClientWrapper.h" #include "tracing/TracerUtil.h" #include "utils/Log.h" #include "utils/LogUtil.h" @@ -263,10 +264,12 @@ Server::StartService() { scheduler::StartSchedulerService(); DBWrapper::GetInstance().StartService(); grpc::GrpcServer::GetInstance().Start(); + storage::S3ClientWrapper::GetInstance().StartService(); } void Server::StopService() { + storage::S3ClientWrapper::GetInstance().StopService(); grpc::GrpcServer::GetInstance().Stop(); DBWrapper::GetInstance().StopService(); scheduler::StopSchedulerService(); diff --git a/core/src/storage/IOReader.h b/core/src/storage/IOReader.h new file mode 100644 index 0000000000..ecde54f891 --- /dev/null +++ b/core/src/storage/IOReader.h @@ -0,0 +1,45 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +namespace milvus { +namespace storage { + +class IOReader { + public: + explicit IOReader(const std::string& name) : name_(name) { + } + ~IOReader() = default; + + virtual void + read(void* ptr, size_t size) = 0; + + virtual void + seekg(size_t pos) = 0; + + virtual size_t + length() = 0; + + public: + std::string name_; +}; + +} // namespace storage +} // namespace milvus diff --git a/core/src/storage/IOWriter.h b/core/src/storage/IOWriter.h new file mode 100644 index 0000000000..f528ee4089 --- /dev/null +++ b/core/src/storage/IOWriter.h @@ -0,0 +1,43 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +namespace milvus { +namespace storage { + +class IOWriter { + public: + explicit IOWriter(const std::string& name) : name_(name), len_(0) { + } + ~IOWriter() = default; + + virtual void + write(void* ptr, size_t size) = 0; + + virtual size_t + length() = 0; + + public: + std::string name_; + size_t len_; +}; + +} // namespace storage +} // namespace milvus diff --git a/core/src/storage/IStorage.h b/core/src/storage/IStorage.h new file mode 100644 index 0000000000..21807deadd --- /dev/null +++ b/core/src/storage/IStorage.h @@ -0,0 +1,50 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include "utils/Status.h" + +namespace milvus { +namespace storage { + +class IStorage { + public: + virtual Status + CreateBucket() = 0; + virtual Status + DeleteBucket() = 0; + virtual Status + PutObjectFile(const std::string& object_name, const std::string& file_path) = 0; + virtual Status + PutObjectStr(const std::string& object_name, const std::string& content) = 0; + virtual Status + GetObjectFile(const std::string& object_name, const std::string& file_path) = 0; + virtual Status + GetObjectStr(const std::string& object_name, std::string& content) = 0; + virtual Status + ListObjects(std::vector& object_list, const std::string& marker = "") = 0; + virtual Status + DeleteObject(const std::string& object_name) = 0; + virtual Status + DeleteObjects(const std::string& marker) = 0; +}; + +} // namespace storage +} // namespace milvus diff --git a/core/src/storage/file/FileIOReader.cpp b/core/src/storage/file/FileIOReader.cpp new file mode 100644 index 0000000000..0cb1b272ed --- /dev/null +++ b/core/src/storage/file/FileIOReader.cpp @@ -0,0 +1,47 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/file/FileIOReader.h" + +namespace milvus { +namespace storage { + +FileIOReader::FileIOReader(const std::string& name) : IOReader(name) { + fs_ = std::fstream(name_, std::ios::in | std::ios::binary); +} + +FileIOReader::~FileIOReader() { + fs_.close(); +} + +void +FileIOReader::read(void* ptr, size_t size) { + fs_.read(reinterpret_cast(ptr), size); +} + +void +FileIOReader::seekg(size_t pos) { + fs_.seekg(pos); +} + +size_t +FileIOReader::length() { + fs_.seekg(0, fs_.end); + return fs_.tellg(); +} +} // namespace storage +} // namespace milvus diff --git a/core/src/storage/file/FileIOReader.h b/core/src/storage/file/FileIOReader.h new file mode 100644 index 0000000000..ceb705a1c3 --- /dev/null +++ b/core/src/storage/file/FileIOReader.h @@ -0,0 +1,46 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include "storage/IOReader.h" + +namespace milvus { +namespace storage { + +class FileIOReader : public IOReader { + public: + explicit FileIOReader(const std::string& name); + ~FileIOReader(); + + void + read(void* ptr, size_t size) override; + + void + seekg(size_t pos) override; + + size_t + length() override; + + public: + std::fstream fs_; +}; + +} // namespace storage +} // namespace milvus diff --git a/core/src/storage/file/FileIOWriter.cpp b/core/src/storage/file/FileIOWriter.cpp new file mode 100644 index 0000000000..2ca3cfa4c2 --- /dev/null +++ b/core/src/storage/file/FileIOWriter.cpp @@ -0,0 +1,43 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/file/FileIOWriter.h" + +namespace milvus { +namespace storage { + +FileIOWriter::FileIOWriter(const std::string& name) : IOWriter(name) { + fs_ = std::fstream(name_, std::ios::out | std::ios::binary); +} + +FileIOWriter::~FileIOWriter() { + fs_.close(); +} + +void +FileIOWriter::write(void* ptr, size_t size) { + fs_.write(reinterpret_cast(ptr), size); + len_ += size; +} + +size_t +FileIOWriter::length() { + return len_; +} + +} // namespace storage +} // namespace milvus diff --git a/core/src/storage/file/FileIOWriter.h b/core/src/storage/file/FileIOWriter.h new file mode 100644 index 0000000000..691cd2e120 --- /dev/null +++ b/core/src/storage/file/FileIOWriter.h @@ -0,0 +1,43 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include "storage/IOWriter.h" + +namespace milvus { +namespace storage { + +class FileIOWriter : public IOWriter { + public: + explicit FileIOWriter(const std::string& name); + ~FileIOWriter(); + + void + write(void* ptr, size_t size) override; + + size_t + length() override; + + public: + std::fstream fs_; +}; + +} // namespace storage +} // namespace milvus diff --git a/core/src/storage/s3/S3ClientWrapper.cpp b/core/src/storage/s3/S3ClientWrapper.cpp new file mode 100644 index 0000000000..620b97ae95 --- /dev/null +++ b/core/src/storage/s3/S3ClientWrapper.cpp @@ -0,0 +1,277 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "server/Config.h" +#include "storage/s3/S3ClientWrapper.h" +#include "utils/Error.h" +#include "utils/Log.h" + +namespace milvus { +namespace storage { + +Status +S3ClientWrapper::StartService() { + server::Config& config = server::Config::GetInstance(); + bool minio_enable = false; + CONFIG_CHECK(config.GetStorageConfigMinioEnable(minio_enable)); + if (!minio_enable) { + STORAGE_LOG_INFO << "MinIO not enabled!"; + return Status::OK(); + } + + CONFIG_CHECK(config.GetStorageConfigMinioAddress(minio_address_)); + CONFIG_CHECK(config.GetStorageConfigMinioPort(minio_port_)); + CONFIG_CHECK(config.GetStorageConfigMinioAccessKey(minio_access_key_)); + CONFIG_CHECK(config.GetStorageConfigMinioSecretKey(minio_secret_key_)); + CONFIG_CHECK(config.GetStorageConfigMinioBucket(minio_bucket_)); + + Aws::InitAPI(options_); + Aws::Client::ClientConfiguration cfg; + + cfg.endpointOverride = minio_address_ + ":" + minio_port_; + cfg.scheme = Aws::Http::Scheme::HTTP; + cfg.verifySSL = false; + client_ptr_ = new Aws::S3::S3Client(Aws::Auth::AWSCredentials(minio_access_key_, minio_secret_key_), cfg, + Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Always, false); + if (client_ptr_ == nullptr) { + std::string str = "Cannot connect S3 server."; + return milvus::Status(SERVER_UNEXPECTED_ERROR, str); + } + + return CreateBucket(); +} + +Status +S3ClientWrapper::StopService() { + if (client_ptr_ != nullptr) { + delete client_ptr_; + client_ptr_ = nullptr; + } + Aws::ShutdownAPI(options_); + return Status::OK(); +} + +Status +S3ClientWrapper::CreateBucket() { + Aws::S3::Model::CreateBucketRequest request; + request.WithBucket(minio_bucket_); + + auto outcome = client_ptr_->CreateBucket(request); + + if (!outcome.IsSuccess()) { + auto err = outcome.GetError(); + if (err.GetErrorType() != Aws::S3::S3Errors::BUCKET_ALREADY_OWNED_BY_YOU) { + STORAGE_LOG_ERROR << "ERROR: CreateBucket: " << err.GetExceptionName() << ": " << err.GetMessage(); + return Status(SERVER_UNEXPECTED_ERROR, err.GetMessage()); + } + } + + STORAGE_LOG_DEBUG << "CreateBucket '" << minio_bucket_ << "' successfully!"; + return Status::OK(); +} + +Status +S3ClientWrapper::DeleteBucket() { + Aws::S3::Model::DeleteBucketRequest request; + request.WithBucket(minio_bucket_); + + auto outcome = client_ptr_->DeleteBucket(request); + + if (!outcome.IsSuccess()) { + auto err = outcome.GetError(); + STORAGE_LOG_ERROR << "ERROR: DeleteBucket: " << err.GetExceptionName() << ": " << err.GetMessage(); + return Status(SERVER_UNEXPECTED_ERROR, err.GetMessage()); + } + + STORAGE_LOG_DEBUG << "DeleteBucket '" << minio_bucket_ << "' successfully!"; + return Status::OK(); +} + +Status +S3ClientWrapper::PutObjectFile(const std::string& object_name, const std::string& file_path) { + struct stat buffer; + if (stat(file_path.c_str(), &buffer) != 0) { + std::string str = "File '" + file_path + "' not exist!"; + STORAGE_LOG_ERROR << "ERROR: " << str; + return Status(SERVER_UNEXPECTED_ERROR, str); + } + + Aws::S3::Model::PutObjectRequest request; + request.WithBucket(minio_bucket_).WithKey(object_name); + + auto input_data = + Aws::MakeShared("PutObjectFile", file_path.c_str(), std::ios_base::in | std::ios_base::binary); + request.SetBody(input_data); + + auto outcome = client_ptr_->PutObject(request); + + if (!outcome.IsSuccess()) { + auto err = outcome.GetError(); + STORAGE_LOG_ERROR << "ERROR: PutObject: " << err.GetExceptionName() << ": " << err.GetMessage(); + return Status(SERVER_UNEXPECTED_ERROR, err.GetMessage()); + } + + STORAGE_LOG_DEBUG << "PutObjectFile '" << file_path << "' successfully!"; + return Status::OK(); +} + +Status +S3ClientWrapper::PutObjectStr(const std::string& object_name, const std::string& content) { + Aws::S3::Model::PutObjectRequest request; + request.WithBucket(minio_bucket_).WithKey(object_name); + + const std::shared_ptr input_data = Aws::MakeShared(""); + input_data->write(content.data(), content.length()); + request.SetBody(input_data); + + auto outcome = client_ptr_->PutObject(request); + + if (!outcome.IsSuccess()) { + auto err = outcome.GetError(); + STORAGE_LOG_ERROR << "ERROR: PutObject: " << err.GetExceptionName() << ": " << err.GetMessage(); + return Status(SERVER_UNEXPECTED_ERROR, err.GetMessage()); + } + + STORAGE_LOG_DEBUG << "PutObjectStr successfully!"; + return Status::OK(); +} + +Status +S3ClientWrapper::GetObjectFile(const std::string& object_name, const std::string& file_path) { + Aws::S3::Model::GetObjectRequest request; + request.WithBucket(minio_bucket_).WithKey(object_name); + + auto outcome = client_ptr_->GetObject(request); + + if (!outcome.IsSuccess()) { + auto err = outcome.GetError(); + STORAGE_LOG_ERROR << "ERROR: GetObject: " << err.GetExceptionName() << ": " << err.GetMessage(); + return Status(SERVER_UNEXPECTED_ERROR, err.GetMessage()); + } + + auto& retrieved_file = outcome.GetResultWithOwnership().GetBody(); + std::ofstream output_file(file_path, std::ios::binary); + output_file << retrieved_file.rdbuf(); + output_file.close(); + + STORAGE_LOG_DEBUG << "GetObjectFile '" << file_path << "' successfully!"; + return Status::OK(); +} + +Status +S3ClientWrapper::GetObjectStr(const std::string& object_name, std::string& content) { + Aws::S3::Model::GetObjectRequest request; + request.WithBucket(minio_bucket_).WithKey(object_name); + + auto outcome = client_ptr_->GetObject(request); + + if (!outcome.IsSuccess()) { + auto err = outcome.GetError(); + STORAGE_LOG_ERROR << "ERROR: GetObject: " << err.GetExceptionName() << ": " << err.GetMessage(); + return Status(SERVER_UNEXPECTED_ERROR, err.GetMessage()); + } + + auto& retrieved_file = outcome.GetResultWithOwnership().GetBody(); + std::stringstream ss; + ss << retrieved_file.rdbuf(); + content = std::move(ss.str()); + + STORAGE_LOG_DEBUG << "GetObjectStr successfully!"; + return Status::OK(); +} + +Status +S3ClientWrapper::ListObjects(std::vector& object_list, const std::string& marker) { + Aws::S3::Model::ListObjectsRequest request; + request.WithBucket(minio_bucket_); + + if (!marker.empty()) { + request.WithMarker(marker); + } + + auto outcome = client_ptr_->ListObjects(request); + + if (!outcome.IsSuccess()) { + auto err = outcome.GetError(); + STORAGE_LOG_ERROR << "ERROR: ListObjects: " << err.GetExceptionName() << ": " << err.GetMessage(); + return Status(SERVER_UNEXPECTED_ERROR, err.GetMessage()); + } + + Aws::Vector result_list = outcome.GetResult().GetContents(); + + for (auto const& s3_object : result_list) { + object_list.push_back(s3_object.GetKey()); + } + + if (marker.empty()) { + STORAGE_LOG_DEBUG << "ListObjects '" << minio_bucket_ << "' successfully!"; + } else { + STORAGE_LOG_DEBUG << "ListObjects '" << minio_bucket_ << ":" << marker << "' successfully!"; + } + return Status::OK(); +} + +Status +S3ClientWrapper::DeleteObject(const std::string& object_name) { + Aws::S3::Model::DeleteObjectRequest request; + request.WithBucket(minio_bucket_).WithKey(object_name); + + auto outcome = client_ptr_->DeleteObject(request); + + if (!outcome.IsSuccess()) { + auto err = outcome.GetError(); + STORAGE_LOG_ERROR << "ERROR: DeleteObject: " << err.GetExceptionName() << ": " << err.GetMessage(); + return Status(SERVER_UNEXPECTED_ERROR, err.GetMessage()); + } + + STORAGE_LOG_DEBUG << "DeleteObject '" << object_name << "' successfully!"; + return Status::OK(); +} + +Status +S3ClientWrapper::DeleteObjects(const std::string& marker) { + std::vector object_list; + + Status stat = ListObjects(object_list, marker); + if (!stat.ok()) { + return stat; + } + + for (std::string& obj_name : object_list) { + stat = DeleteObject(obj_name); + if (!stat.ok()) { + return stat; + } + } + + return Status::OK(); +} + +} // namespace storage +} // namespace milvus diff --git a/core/src/storage/s3/S3ClientWrapper.h b/core/src/storage/s3/S3ClientWrapper.h new file mode 100644 index 0000000000..6f2dc1e0be --- /dev/null +++ b/core/src/storage/s3/S3ClientWrapper.h @@ -0,0 +1,76 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include +#include "storage/IStorage.h" + +namespace milvus { +namespace storage { + +class S3ClientWrapper : public IStorage { + public: + S3ClientWrapper() = default; + ~S3ClientWrapper() = default; + + static S3ClientWrapper& + GetInstance() { + static S3ClientWrapper wrapper; + return wrapper; + } + + Status + StartService(); + Status + StopService(); + + Status + CreateBucket() override; + Status + DeleteBucket() override; + Status + PutObjectFile(const std::string& object_key, const std::string& file_path) override; + Status + PutObjectStr(const std::string& object_key, const std::string& content) override; + Status + GetObjectFile(const std::string& object_key, const std::string& file_path) override; + Status + GetObjectStr(const std::string& object_key, std::string& content) override; + Status + ListObjects(std::vector& object_list, const std::string& marker = "") override; + Status + DeleteObject(const std::string& object_key) override; + Status + DeleteObjects(const std::string& marker) override; + + private: + Aws::S3::S3Client* client_ptr_ = nullptr; + Aws::SDKOptions options_; + + std::string minio_address_; + std::string minio_port_; + std::string minio_access_key_; + std::string minio_secret_key_; + std::string minio_bucket_; +}; + +} // namespace storage +} // namespace milvus diff --git a/core/src/storage/s3/S3IOReader.cpp b/core/src/storage/s3/S3IOReader.cpp new file mode 100644 index 0000000000..16b8baaaa2 --- /dev/null +++ b/core/src/storage/s3/S3IOReader.cpp @@ -0,0 +1,47 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/s3/S3IOReader.h" +#include "storage/s3/S3ClientWrapper.h" + +namespace milvus { +namespace storage { + +S3IOReader::S3IOReader(const std::string& name) : IOReader(name), pos_(0) { + S3ClientWrapper::GetInstance().GetObjectStr(name_, buffer_); +} + +S3IOReader::~S3IOReader() { +} + +void +S3IOReader::read(void* ptr, size_t size) { + memcpy(ptr, buffer_.data() + pos_, size); +} + +void +S3IOReader::seekg(size_t pos) { + pos_ = pos; +} + +size_t +S3IOReader::length() { + return buffer_.length(); +} + +} // namespace storage +} // namespace milvus diff --git a/core/src/storage/s3/S3IOReader.h b/core/src/storage/s3/S3IOReader.h new file mode 100644 index 0000000000..1c50fa0b72 --- /dev/null +++ b/core/src/storage/s3/S3IOReader.h @@ -0,0 +1,46 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include "storage/IOReader.h" + +namespace milvus { +namespace storage { + +class S3IOReader : public IOReader { + public: + explicit S3IOReader(const std::string& name); + ~S3IOReader(); + + void + read(void* ptr, size_t size) override; + + void + seekg(size_t pos) override; + + size_t + length() override; + + public: + std::string buffer_; + size_t pos_; +}; + +} // namespace storage +} // namespace milvus diff --git a/core/src/storage/s3/S3IOWriter.cpp b/core/src/storage/s3/S3IOWriter.cpp new file mode 100644 index 0000000000..fa73533c04 --- /dev/null +++ b/core/src/storage/s3/S3IOWriter.cpp @@ -0,0 +1,44 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/s3/S3IOWriter.h" +#include "storage/s3/S3ClientWrapper.h" + +namespace milvus { +namespace storage { + +S3IOWriter::S3IOWriter(const std::string& name) : IOWriter(name) { + buffer_ = ""; +} + +S3IOWriter::~S3IOWriter() { + S3ClientWrapper::GetInstance().PutObjectStr(name_, buffer_); +} + +void +S3IOWriter::write(void* ptr, size_t size) { + buffer_ += std::string(reinterpret_cast(ptr), size); + len_ += size; +} + +size_t +S3IOWriter::length() { + return len_; +} + +} // namespace storage +} // namespace milvus diff --git a/core/src/storage/s3/S3IOWriter.h b/core/src/storage/s3/S3IOWriter.h new file mode 100644 index 0000000000..0464f1ecd5 --- /dev/null +++ b/core/src/storage/s3/S3IOWriter.h @@ -0,0 +1,42 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include "storage/IOWriter.h" + +namespace milvus { +namespace storage { + +class S3IOWriter : public IOWriter { + public: + explicit S3IOWriter(const std::string& name); + ~S3IOWriter(); + + void + write(void* ptr, size_t size) override; + + size_t + length() override; + + public: + std::string buffer_; +}; + +} // namespace storage +} // namespace milvus diff --git a/core/src/utils/Log.h b/core/src/utils/Log.h index cef72a866d..3c404086f4 100644 --- a/core/src/utils/Log.h +++ b/core/src/utils/Log.h @@ -51,4 +51,14 @@ namespace milvus { #define WRAPPER_LOG_ERROR LOG(ERROR) << WRAPPER_DOMAIN_NAME #define WRAPPER_LOG_FATAL LOG(FATAL) << WRAPPER_DOMAIN_NAME +///////////////////////////////////////////////////////////////////////////////////////////////// +#define STORAGE_DOMAIN_NAME "[STORAGE] " + +#define STORAGE_LOG_TRACE LOG(TRACE) << STORAGE_DOMAIN_NAME +#define STORAGE_LOG_DEBUG LOG(DEBUG) << STORAGE_DOMAIN_NAME +#define STORAGE_LOG_INFO LOG(INFO) << STORAGE_DOMAIN_NAME +#define STORAGE_LOG_WARNING LOG(WARNING) << STORAGE_DOMAIN_NAME +#define STORAGE_LOG_ERROR LOG(ERROR) << STORAGE_DOMAIN_NAME +#define STORAGE_LOG_FATAL LOG(FATAL) << STORAGE_DOMAIN_NAME + } // namespace milvus diff --git a/core/src/wrapper/VecIndex.cpp b/core/src/wrapper/VecIndex.cpp index e6eb06d75b..aeeadb15c9 100644 --- a/core/src/wrapper/VecIndex.cpp +++ b/core/src/wrapper/VecIndex.cpp @@ -25,8 +25,13 @@ #include "knowhere/index/vector_index/IndexNSG.h" #include "knowhere/index/vector_index/IndexSPTAG.h" #include "server/Config.h" +#include "storage/file/FileIOReader.h" +#include "storage/file/FileIOWriter.h" +#include "storage/s3/S3IOReader.h" +#include "storage/s3/S3IOWriter.h" #include "utils/Exception.h" #include "utils/Log.h" +#include "utils/TimeRecorder.h" #ifdef MILVUS_GPU_VERSION #include @@ -54,64 +59,6 @@ VecIndex::set_size(int64_t size) { size_ = size; } -struct FileIOReader { - std::fstream fs; - std::string name; - - explicit FileIOReader(const std::string& fname); - - ~FileIOReader(); - - size_t - operator()(void* ptr, size_t size); - - size_t - operator()(void* ptr, size_t size, size_t pos); -}; - -FileIOReader::FileIOReader(const std::string& fname) { - name = fname; - fs = std::fstream(name, std::ios::in | std::ios::binary); -} - -FileIOReader::~FileIOReader() { - fs.close(); -} - -size_t -FileIOReader::operator()(void* ptr, size_t size) { - fs.read(reinterpret_cast(ptr), size); -} - -size_t -FileIOReader::operator()(void* ptr, size_t size, size_t pos) { - return 0; -} - -struct FileIOWriter { - std::fstream fs; - std::string name; - - explicit FileIOWriter(const std::string& fname); - ~FileIOWriter(); - size_t - operator()(void* ptr, size_t size); -}; - -FileIOWriter::FileIOWriter(const std::string& fname) { - name = fname; - fs = std::fstream(name, std::ios::out | std::ios::binary); -} - -FileIOWriter::~FileIOWriter() { - fs.close(); -} - -size_t -FileIOWriter::operator()(void* ptr, size_t size) { - fs.write(reinterpret_cast(ptr), size); -} - VecIndexPtr GetVecIndexFactory(const IndexType& type, const Config& cfg) { std::shared_ptr index; @@ -206,39 +153,55 @@ LoadVecIndex(const IndexType& index_type, const knowhere::BinarySet& index_binar VecIndexPtr read_index(const std::string& location) { + TimeRecorder recorder("read_index"); knowhere::BinarySet load_data_list; - FileIOReader reader(location); - reader.fs.seekg(0, reader.fs.end); - int64_t length = reader.fs.tellg(); + + bool minio_enable = false; + server::Config& config = server::Config::GetInstance(); + config.GetStorageConfigMinioEnable(minio_enable); + + std::shared_ptr reader_ptr; + if (minio_enable) { + reader_ptr = std::make_shared(location); + } else { + reader_ptr = std::make_shared(location); + } + + recorder.RecordSection("Start"); + + size_t length = reader_ptr->length(); if (length <= 0) { return nullptr; } - reader.fs.seekg(0); - size_t rp = 0; + reader_ptr->seekg(0); + auto current_type = IndexType::INVALID; - reader(¤t_type, sizeof(current_type)); + reader_ptr->read(¤t_type, sizeof(current_type)); rp += sizeof(current_type); + reader_ptr->seekg(rp); + while (rp < length) { size_t meta_length; - reader(&meta_length, sizeof(meta_length)); + reader_ptr->read(&meta_length, sizeof(meta_length)); rp += sizeof(meta_length); - reader.fs.seekg(rp); + reader_ptr->seekg(rp); auto meta = new char[meta_length]; - reader(meta, meta_length); + reader_ptr->read(meta, meta_length); rp += meta_length; - reader.fs.seekg(rp); + reader_ptr->seekg(rp); size_t bin_length; - reader(&bin_length, sizeof(bin_length)); + reader_ptr->read(&bin_length, sizeof(bin_length)); rp += sizeof(bin_length); - reader.fs.seekg(rp); + reader_ptr->seekg(rp); auto bin = new uint8_t[bin_length]; - reader(bin, bin_length); + reader_ptr->read(bin, bin_length); rp += bin_length; + reader_ptr->seekg(rp); auto binptr = std::make_shared(); binptr.reset(bin); @@ -246,28 +209,51 @@ read_index(const std::string& location) { delete[] meta; } + double span = recorder.RecordSection("End"); + double rate = length * 1000000.0 / span / 1024 / 1024; + STORAGE_LOG_DEBUG << "read_index(" << location << ") rate " << rate << "MB/s"; + return LoadVecIndex(current_type, load_data_list, length); } Status write_index(VecIndexPtr index, const std::string& location) { try { + TimeRecorder recorder("write_index"); + auto binaryset = index->Serialize(); auto index_type = index->GetType(); - FileIOWriter writer(location); - writer(&index_type, sizeof(IndexType)); + bool minio_enable = false; + server::Config& config = server::Config::GetInstance(); + config.GetStorageConfigMinioEnable(minio_enable); + + std::shared_ptr writer_ptr; + if (minio_enable) { + writer_ptr = std::make_shared(location); + } else { + writer_ptr = std::make_shared(location); + } + + recorder.RecordSection("Start"); + + writer_ptr->write(&index_type, sizeof(IndexType)); + for (auto& iter : binaryset.binary_map_) { auto meta = iter.first.c_str(); size_t meta_length = iter.first.length(); - writer(&meta_length, sizeof(meta_length)); - writer((void*)meta, meta_length); + writer_ptr->write(&meta_length, sizeof(meta_length)); + writer_ptr->write((void*)meta, meta_length); auto binary = iter.second; int64_t binary_length = binary->size; - writer(&binary_length, sizeof(binary_length)); - writer((void*)binary->data.get(), binary_length); + writer_ptr->write(&binary_length, sizeof(binary_length)); + writer_ptr->write((void*)binary->data.get(), binary_length); } + + double span = recorder.RecordSection("End"); + double rate = writer_ptr->length() * 1000000.0 / span / 1024 / 1024; + STORAGE_LOG_DEBUG << "write_index(" << location << ") rate " << rate << "MB/s"; } catch (knowhere::KnowhereException& e) { WRAPPER_LOG_ERROR << e.what(); return Status(KNOWHERE_UNEXPECTED_ERROR, e.what()); diff --git a/core/thirdparty/versions.txt b/core/thirdparty/versions.txt index 9c531c7d39..551a68bb80 100644 --- a/core/thirdparty/versions.txt +++ b/core/thirdparty/versions.txt @@ -11,5 +11,6 @@ GRPC_VERSION=master ZLIB_VERSION=v1.2.11 OPENTRACING_VERSION=v1.5.1 FIU_VERSION=1.00 +AWS_VERSION=1.7.250 # vim: set filetype=sh: diff --git a/core/ubuntu_build_deps.sh b/core/ubuntu_build_deps.sh index 109eea7dbd..e09692711a 100755 --- a/core/ubuntu_build_deps.sh +++ b/core/ubuntu_build_deps.sh @@ -7,7 +7,7 @@ sudo sh -c 'echo deb https://apt.repos.intel.com/mkl all main > /etc/apt/sources sudo apt-get -y update && sudo apt-get -y install intel-mkl-gnu-2019.5-281 intel-mkl-core-2019.5-281 sudo apt-get install -y gfortran libmysqlclient-dev mysql-client libcurl4-openssl-dev libboost-system-dev \ -libboost-filesystem-dev libboost-serialization-dev libboost-regex-dev libopenblas-dev liblapack3 +libboost-filesystem-dev libboost-serialization-dev libboost-regex-dev libopenblas-dev liblapack3 libssl-dev if [ ! -f "/usr/lib/x86_64-linux-gnu/libmysqlclient_r.so" ]; then sudo ln -s /usr/lib/x86_64-linux-gnu/libmysqlclient.so /usr/lib/x86_64-linux-gnu/libmysqlclient_r.so diff --git a/core/unittest/CMakeLists.txt b/core/unittest/CMakeLists.txt index 820559c064..ac5f51217f 100644 --- a/core/unittest/CMakeLists.txt +++ b/core/unittest/CMakeLists.txt @@ -84,6 +84,15 @@ aux_source_directory(${MILVUS_ENGINE_SRC}/utils utils_files) aux_source_directory(${MILVUS_ENGINE_SRC}/wrapper wrapper_files) aux_source_directory(${MILVUS_ENGINE_SRC}/tracing tracing_files) +aux_source_directory(${MILVUS_ENGINE_SRC}/storage storage_main_files) +aux_source_directory(${MILVUS_ENGINE_SRC}/storage/file storage_file_files) +aux_source_directory(${MILVUS_ENGINE_SRC}/storage/s3 storage_s3_files) +set(storage_files + ${storage_main_files} + ${storage_file_files} + ${storage_s3_files} + ) + set(entry_file ${CMAKE_CURRENT_SOURCE_DIR}/main.cpp) @@ -108,6 +117,7 @@ set(common_files ${thirdparty_files} ${scheduler_files} ${wrapper_files} + ${storage_files} ${helper_files} ${server_context_files} ${tracing_files} @@ -129,6 +139,10 @@ set(unittest_libs opentracing opentracing_mocktracer fiu + aws-cpp-sdk-s3 + aws-cpp-sdk-core + curl + crypto ) if (MILVUS_WITH_PROMETHEUS) set(unittest_libs ${unittest_libs} @@ -161,3 +175,4 @@ add_subdirectory(wrapper) add_subdirectory(metrics) add_subdirectory(scheduler) add_subdirectory(server) +add_subdirectory(storage) diff --git a/core/unittest/db/test_db.cpp b/core/unittest/db/test_db.cpp index 6262bda7d6..6e708cc161 100644 --- a/core/unittest/db/test_db.cpp +++ b/core/unittest/db/test_db.cpp @@ -40,6 +40,7 @@ static constexpr int64_t INSERT_LOOP = 1000; static constexpr int64_t SECONDS_EACH_HOUR = 3600; static constexpr int64_t DAY_SECONDS = 24 * 60 * 60; + milvus::engine::meta::TableSchema BuildTableSchema() { milvus::engine::meta::TableSchema table_info; diff --git a/core/unittest/db/utils.cpp b/core/unittest/db/utils.cpp index 346fba35d0..db0fc87ff0 100644 --- a/core/unittest/db/utils.cpp +++ b/core/unittest/db/utils.cpp @@ -49,15 +49,16 @@ static const char* CONFIG_STR = " time_zone: UTC+8\n" "\n" "db_config:\n" - " primary_path: /tmp/milvus # path used to store data and meta\n" - " secondary_path: # path used to store data only, split by semicolon\n" - "\n" " backend_url: sqlite://:@:/ \n" " \n" " # Replace 'dialect' with 'mysql' or 'sqlite'\n" "\n" " insert_buffer_size: 4 # GB, maximum insert buffer size allowed\n" "\n" + "storage_config:\n" + " primary_path: /tmp/milvus # path used to store data and meta\n" + " secondary_path: # path used to store data only, split by semicolon\n" + "\n" "metric_config:\n" " enable_monitor: false # enable monitoring or not\n" " collector: prometheus # prometheus\n" diff --git a/core/unittest/server/test_config.cpp b/core/unittest/server/test_config.cpp index 463d318a2a..c669ffa7e1 100644 --- a/core/unittest/server/test_config.cpp +++ b/core/unittest/server/test_config.cpp @@ -119,7 +119,6 @@ TEST_F(ConfigTest, CONFIG_TEST) { TEST_F(ConfigTest, SERVER_CONFIG_VALID_TEST) { std::string config_path(CONFIG_PATH); milvus::server::Config& config = milvus::server::Config::GetInstance(); - milvus::Status s; std::string str_val; int64_t int64_val; float float_val; @@ -127,159 +126,152 @@ TEST_F(ConfigTest, SERVER_CONFIG_VALID_TEST) { /* server config */ std::string server_addr = "192.168.1.155"; - s = config.SetServerConfigAddress(server_addr); - ASSERT_TRUE(s.ok()); - s = config.GetServerConfigAddress(str_val); - ASSERT_TRUE(s.ok()); + ASSERT_TRUE(config.SetServerConfigAddress(server_addr).ok()); + ASSERT_TRUE(config.GetServerConfigAddress(str_val).ok()); ASSERT_TRUE(str_val == server_addr); std::string server_port = "12345"; - s = config.SetServerConfigPort(server_port); - ASSERT_TRUE(s.ok()); - s = config.GetServerConfigPort(str_val); - ASSERT_TRUE(s.ok()); + ASSERT_TRUE(config.SetServerConfigPort(server_port).ok()); + ASSERT_TRUE(config.GetServerConfigPort(str_val).ok()); ASSERT_TRUE(str_val == server_port); std::string server_mode = "cluster_readonly"; - s = config.SetServerConfigDeployMode(server_mode); - ASSERT_TRUE(s.ok()); - s = config.GetServerConfigDeployMode(str_val); - ASSERT_TRUE(s.ok()); + ASSERT_TRUE(config.SetServerConfigDeployMode(server_mode).ok()); + ASSERT_TRUE(config.GetServerConfigDeployMode(str_val).ok()); ASSERT_TRUE(str_val == server_mode); std::string server_time_zone = "UTC+6"; - s = config.SetServerConfigTimeZone(server_time_zone); - ASSERT_TRUE(s.ok()); - s = config.GetServerConfigTimeZone(str_val); - ASSERT_TRUE(s.ok()); + ASSERT_TRUE(config.SetServerConfigTimeZone(server_time_zone).ok()); + ASSERT_TRUE(config.GetServerConfigTimeZone(str_val).ok()); ASSERT_TRUE(str_val == server_time_zone); /* db config */ - std::string db_primary_path = "/home/zilliz"; - s = config.SetDBConfigPrimaryPath(db_primary_path); - ASSERT_TRUE(s.ok()); - s = config.GetDBConfigPrimaryPath(str_val); - ASSERT_TRUE(s.ok()); - ASSERT_TRUE(str_val == db_primary_path); - - std::string db_secondary_path = "/home/zilliz"; - s = config.SetDBConfigSecondaryPath(db_secondary_path); - ASSERT_TRUE(s.ok()); - s = config.GetDBConfigSecondaryPath(str_val); - ASSERT_TRUE(s.ok()); - ASSERT_TRUE(str_val == db_secondary_path); - std::string db_backend_url = "mysql://root:123456@127.0.0.1:19530/milvus"; - s = config.SetDBConfigBackendUrl(db_backend_url); - ASSERT_TRUE(s.ok()); - s = config.GetDBConfigBackendUrl(str_val); - ASSERT_TRUE(s.ok()); + ASSERT_TRUE(config.SetDBConfigBackendUrl(db_backend_url).ok()); + ASSERT_TRUE(config.GetDBConfigBackendUrl(str_val).ok()); ASSERT_TRUE(str_val == db_backend_url); int64_t db_archive_disk_threshold = 100; - s = config.SetDBConfigArchiveDiskThreshold(std::to_string(db_archive_disk_threshold)); - ASSERT_TRUE(s.ok()); - s = config.GetDBConfigArchiveDiskThreshold(int64_val); - ASSERT_TRUE(s.ok()); + ASSERT_TRUE(config.SetDBConfigArchiveDiskThreshold(std::to_string(db_archive_disk_threshold)).ok()); + ASSERT_TRUE(config.GetDBConfigArchiveDiskThreshold(int64_val).ok()); ASSERT_TRUE(int64_val == db_archive_disk_threshold); int64_t db_archive_days_threshold = 365; - s = config.SetDBConfigArchiveDaysThreshold(std::to_string(db_archive_days_threshold)); - ASSERT_TRUE(s.ok()); - s = config.GetDBConfigArchiveDaysThreshold(int64_val); - ASSERT_TRUE(s.ok()); + ASSERT_TRUE(config.SetDBConfigArchiveDaysThreshold(std::to_string(db_archive_days_threshold)).ok()); + ASSERT_TRUE(config.GetDBConfigArchiveDaysThreshold(int64_val).ok()); ASSERT_TRUE(int64_val == db_archive_days_threshold); int64_t db_insert_buffer_size = 2; - s = config.SetDBConfigInsertBufferSize(std::to_string(db_insert_buffer_size)); - ASSERT_TRUE(s.ok()); - s = config.GetDBConfigInsertBufferSize(int64_val); - ASSERT_TRUE(s.ok()); + ASSERT_TRUE(config.SetDBConfigInsertBufferSize(std::to_string(db_insert_buffer_size)).ok()); + ASSERT_TRUE(config.GetDBConfigInsertBufferSize(int64_val).ok()); ASSERT_TRUE(int64_val == db_insert_buffer_size); + /* storage config */ + std::string storage_primary_path = "/home/zilliz"; + ASSERT_TRUE(config.SetStorageConfigPrimaryPath(storage_primary_path).ok()); + ASSERT_TRUE(config.GetStorageConfigPrimaryPath(str_val).ok()); + ASSERT_TRUE(str_val == storage_primary_path); + + std::string storage_secondary_path = "/home/zilliz"; + ASSERT_TRUE(config.SetStorageConfigSecondaryPath(storage_secondary_path).ok()); + ASSERT_TRUE(config.GetStorageConfigSecondaryPath(str_val).ok()); + ASSERT_TRUE(str_val == storage_secondary_path); + + bool storage_minio_enable = false; + ASSERT_TRUE(config.SetStorageConfigMinioEnable(std::to_string(storage_minio_enable)).ok()); + ASSERT_TRUE(config.GetStorageConfigMinioEnable(bool_val).ok()); + ASSERT_TRUE(bool_val == storage_minio_enable); + + std::string storage_minio_addr = "192.168.1.100"; + ASSERT_TRUE(config.SetStorageConfigMinioAddress(storage_minio_addr).ok()); + ASSERT_TRUE(config.GetStorageConfigMinioAddress(str_val).ok()); + ASSERT_TRUE(str_val == storage_minio_addr); + + std::string storage_minio_port = "12345"; + ASSERT_TRUE(config.SetStorageConfigMinioPort(storage_minio_port).ok()); + ASSERT_TRUE(config.GetStorageConfigMinioPort(str_val).ok()); + ASSERT_TRUE(str_val == storage_minio_port); + + std::string storage_minio_access_key = "minioadmin"; + ASSERT_TRUE(config.SetStorageConfigMinioAccessKey(storage_minio_access_key).ok()); + ASSERT_TRUE(config.GetStorageConfigMinioAccessKey(str_val).ok()); + ASSERT_TRUE(str_val == storage_minio_access_key); + + std::string storage_minio_secret_key = "minioadmin"; + ASSERT_TRUE(config.SetStorageConfigMinioSecretKey(storage_minio_secret_key).ok()); + ASSERT_TRUE(config.GetStorageConfigMinioSecretKey(str_val).ok()); + ASSERT_TRUE(str_val == storage_minio_secret_key); + + std::string storage_minio_bucket = "miniobucket"; + ASSERT_TRUE(config.SetStorageConfigMinioBucket(storage_minio_bucket).ok()); + ASSERT_TRUE(config.GetStorageConfigMinioBucket(str_val).ok()); + ASSERT_TRUE(str_val == storage_minio_bucket); + /* metric config */ bool metric_enable_monitor = false; - s = config.SetMetricConfigEnableMonitor(std::to_string(metric_enable_monitor)); - ASSERT_TRUE(s.ok()); - s = config.GetMetricConfigEnableMonitor(bool_val); - ASSERT_TRUE(s.ok()); + ASSERT_TRUE(config.SetMetricConfigEnableMonitor(std::to_string(metric_enable_monitor)).ok()); + ASSERT_TRUE(config.GetMetricConfigEnableMonitor(bool_val).ok()); ASSERT_TRUE(bool_val == metric_enable_monitor); std::string metric_collector = "prometheus"; - s = config.SetMetricConfigCollector(metric_collector); - ASSERT_TRUE(s.ok()); - s = config.GetMetricConfigCollector(str_val); + ASSERT_TRUE(config.SetMetricConfigCollector(metric_collector).ok()); + ASSERT_TRUE(config.GetMetricConfigCollector(str_val).ok()); ASSERT_TRUE(str_val == metric_collector); std::string metric_prometheus_port = "2222"; - s = config.SetMetricConfigPrometheusPort(metric_prometheus_port); - ASSERT_TRUE(s.ok()); - s = config.GetMetricConfigPrometheusPort(str_val); - ASSERT_TRUE(s.ok()); + ASSERT_TRUE(config.SetMetricConfigPrometheusPort(metric_prometheus_port).ok()); + ASSERT_TRUE(config.GetMetricConfigPrometheusPort(str_val).ok()); ASSERT_TRUE(str_val == metric_prometheus_port); /* cache config */ int64_t cache_cpu_cache_capacity = 1; - s = config.SetCacheConfigCpuCacheCapacity(std::to_string(cache_cpu_cache_capacity)); - ASSERT_TRUE(s.ok()); - s = config.GetCacheConfigCpuCacheCapacity(int64_val); - ASSERT_TRUE(s.ok()); + ASSERT_TRUE(config.SetCacheConfigCpuCacheCapacity(std::to_string(cache_cpu_cache_capacity)).ok()); + ASSERT_TRUE(config.GetCacheConfigCpuCacheCapacity(int64_val).ok()); ASSERT_TRUE(int64_val == cache_cpu_cache_capacity); float cache_cpu_cache_threshold = 0.1; - s = config.SetCacheConfigCpuCacheThreshold(std::to_string(cache_cpu_cache_threshold)); - ASSERT_TRUE(s.ok()); - s = config.GetCacheConfigCpuCacheThreshold(float_val); + ASSERT_TRUE(config.SetCacheConfigCpuCacheThreshold(std::to_string(cache_cpu_cache_threshold)).ok()); + ASSERT_TRUE(config.GetCacheConfigCpuCacheThreshold(float_val).ok()); ASSERT_TRUE(float_val == cache_cpu_cache_threshold); bool cache_insert_data = true; - s = config.SetCacheConfigCacheInsertData(std::to_string(cache_insert_data)); - ASSERT_TRUE(s.ok()); - s = config.GetCacheConfigCacheInsertData(bool_val); + ASSERT_TRUE(config.SetCacheConfigCacheInsertData(std::to_string(cache_insert_data)).ok()); + ASSERT_TRUE(config.GetCacheConfigCacheInsertData(bool_val).ok()); ASSERT_TRUE(bool_val == cache_insert_data); /* engine config */ int64_t engine_use_blas_threshold = 50; - s = config.SetEngineConfigUseBlasThreshold(std::to_string(engine_use_blas_threshold)); - ASSERT_TRUE(s.ok()); - s = config.GetEngineConfigUseBlasThreshold(int64_val); - ASSERT_TRUE(s.ok()); + ASSERT_TRUE(config.SetEngineConfigUseBlasThreshold(std::to_string(engine_use_blas_threshold)).ok()); + ASSERT_TRUE(config.GetEngineConfigUseBlasThreshold(int64_val).ok()); ASSERT_TRUE(int64_val == engine_use_blas_threshold); int64_t engine_omp_thread_num = 1; - s = config.SetEngineConfigOmpThreadNum(std::to_string(engine_omp_thread_num)); - ASSERT_TRUE(s.ok()); - s = config.GetEngineConfigOmpThreadNum(int64_val); - ASSERT_TRUE(s.ok()); + ASSERT_TRUE(config.SetEngineConfigOmpThreadNum(std::to_string(engine_omp_thread_num)).ok()); + ASSERT_TRUE(config.GetEngineConfigOmpThreadNum(int64_val).ok()); ASSERT_TRUE(int64_val == engine_omp_thread_num); #ifdef MILVUS_GPU_VERSION int64_t engine_gpu_search_threshold = 800; - s = config.SetEngineConfigGpuSearchThreshold(std::to_string(engine_gpu_search_threshold)); - ASSERT_TRUE(s.ok()); - s = config.GetEngineConfigGpuSearchThreshold(int64_val); - ASSERT_TRUE(s.ok()); + ASSERT_TRUE(config.SetEngineConfigGpuSearchThreshold(std::to_string(engine_gpu_search_threshold)).ok()); + ASSERT_TRUE(config.GetEngineConfigGpuSearchThreshold(int64_val).ok()); ASSERT_TRUE(int64_val == engine_gpu_search_threshold); +#endif /* gpu resource config */ +#ifdef MILVUS_GPU_VERSION bool resource_enable_gpu = true; - s = config.SetGpuResourceConfigEnable(std::to_string(resource_enable_gpu)); - ASSERT_TRUE(s.ok()); - s = config.GetGpuResourceConfigEnable(bool_val); - ASSERT_TRUE(s.ok()); + ASSERT_TRUE(config.SetGpuResourceConfigEnable(std::to_string(resource_enable_gpu)).ok()); + ASSERT_TRUE(config.GetGpuResourceConfigEnable(bool_val).ok()); ASSERT_TRUE(bool_val == resource_enable_gpu); int64_t gpu_cache_capacity = 1; - s = config.SetGpuResourceConfigCacheCapacity(std::to_string(gpu_cache_capacity)); - ASSERT_TRUE(s.ok()); - s = config.GetGpuResourceConfigCacheCapacity(int64_val); - ASSERT_TRUE(s.ok()); + ASSERT_TRUE(config.SetGpuResourceConfigCacheCapacity(std::to_string(gpu_cache_capacity)).ok()); + ASSERT_TRUE(config.GetGpuResourceConfigCacheCapacity(int64_val).ok()); ASSERT_TRUE(int64_val == gpu_cache_capacity); float gpu_cache_threshold = 0.2; - s = config.SetGpuResourceConfigCacheThreshold(std::to_string(gpu_cache_threshold)); - ASSERT_TRUE(s.ok()); - s = config.GetGpuResourceConfigCacheThreshold(float_val); + ASSERT_TRUE(config.SetGpuResourceConfigCacheThreshold(std::to_string(gpu_cache_threshold)).ok()); + ASSERT_TRUE(config.GetGpuResourceConfigCacheThreshold(float_val).ok()); ASSERT_TRUE(float_val == gpu_cache_threshold); std::vector search_resources = {"gpu0"}; @@ -287,10 +279,8 @@ TEST_F(ConfigTest, SERVER_CONFIG_VALID_TEST) { std::string search_res_str; milvus::server::StringHelpFunctions::MergeStringWithDelimeter( search_resources, milvus::server::CONFIG_GPU_RESOURCE_DELIMITER, search_res_str); - s = config.SetGpuResourceConfigSearchResources(search_res_str); - ASSERT_TRUE(s.ok()); - s = config.GetGpuResourceConfigSearchResources(search_res_vec); - ASSERT_TRUE(s.ok()); + ASSERT_TRUE(config.SetGpuResourceConfigSearchResources(search_res_str).ok()); + ASSERT_TRUE(config.GetGpuResourceConfigSearchResources(search_res_vec).ok()); for (size_t i = 0; i < search_resources.size(); i++) { ASSERT_TRUE(std::stoll(search_resources[i].substr(3)) == search_res_vec[i]); } @@ -300,10 +290,8 @@ TEST_F(ConfigTest, SERVER_CONFIG_VALID_TEST) { std::string build_index_res_str; milvus::server::StringHelpFunctions::MergeStringWithDelimeter( build_index_resources, milvus::server::CONFIG_GPU_RESOURCE_DELIMITER, build_index_res_str); - s = config.SetGpuResourceConfigBuildIndexResources(build_index_res_str); - ASSERT_TRUE(s.ok()); - s = config.GetGpuResourceConfigBuildIndexResources(build_index_res_vec); - ASSERT_TRUE(s.ok()); + ASSERT_TRUE(config.SetGpuResourceConfigBuildIndexResources(build_index_res_str).ok()); + ASSERT_TRUE(config.GetGpuResourceConfigBuildIndexResources(build_index_res_vec).ok()); for (size_t i = 0; i < build_index_resources.size(); i++) { ASSERT_TRUE(std::stoll(build_index_resources[i].substr(3)) == build_index_res_vec[i]); } @@ -341,9 +329,9 @@ TEST_F(ConfigTest, SERVER_CONFIG_CLI_TEST) { ASSERT_TRUE(s.ok()); /* db config */ - std::string db_primary_path = "/home/zilliz"; - get_cmd = gen_get_command(ms::CONFIG_DB, ms::CONFIG_DB_PRIMARY_PATH); - set_cmd = gen_set_command(ms::CONFIG_DB, ms::CONFIG_DB_PRIMARY_PATH, db_primary_path); + std::string db_insert_buffer_size = "4"; + get_cmd = gen_get_command(ms::CONFIG_DB, ms::CONFIG_DB_INSERT_BUFFER_SIZE); + set_cmd = gen_set_command(ms::CONFIG_DB, ms::CONFIG_DB_INSERT_BUFFER_SIZE, db_insert_buffer_size); s = config.ProcessConfigCli(dummy, set_cmd); ASSERT_FALSE(s.ok()); s = config.ProcessConfigCli(result, get_cmd); @@ -358,6 +346,15 @@ TEST_F(ConfigTest, SERVER_CONFIG_CLI_TEST) { s = config.ProcessConfigCli(result, get_cmd); ASSERT_TRUE(s.ok()); + /* storage config */ + std::string storage_minio_enable = "true"; + get_cmd = gen_get_command(ms::CONFIG_STORAGE, ms::CONFIG_STORAGE_MINIO_ENABLE); + set_cmd = gen_set_command(ms::CONFIG_STORAGE, ms::CONFIG_STORAGE_MINIO_ENABLE, storage_minio_enable); + s = config.ProcessConfigCli(dummy, set_cmd); + ASSERT_FALSE(s.ok()); + s = config.ProcessConfigCli(result, get_cmd); + ASSERT_TRUE(s.ok()); + /* cache config */ std::string cache_cpu_cache_capacity = "1"; get_cmd = gen_get_command(ms::CONFIG_CACHE, ms::CONFIG_CACHE_CPU_CACHE_CAPACITY); @@ -412,8 +409,10 @@ TEST_F(ConfigTest, SERVER_CONFIG_CLI_TEST) { s = config.ProcessConfigCli(result, get_cmd); ASSERT_TRUE(s.ok()); ASSERT_TRUE(result == engine_gpu_search_threshold); +#endif /* gpu resource config */ +#ifdef MILVUS_GPU_VERSION std::string resource_enable_gpu = "true"; get_cmd = gen_get_command(ms::CONFIG_GPU_RESOURCE, ms::CONFIG_GPU_RESOURCE_ENABLE); set_cmd = gen_set_command(ms::CONFIG_GPU_RESOURCE, ms::CONFIG_GPU_RESOURCE_ENABLE, resource_enable_gpu); @@ -466,138 +465,110 @@ TEST_F(ConfigTest, SERVER_CONFIG_INVALID_TEST) { milvus::server::Config& config = milvus::server::Config::GetInstance(); milvus::Status s; - s = config.LoadConfigFile(""); - ASSERT_FALSE(s.ok()); + ASSERT_FALSE(config.LoadConfigFile("").ok()); - s = config.LoadConfigFile(config_path + INVALID_CONFIG_FILE); - ASSERT_FALSE(s.ok()); - s = config.LoadConfigFile(config_path + "dummy.yaml"); - ASSERT_FALSE(s.ok()); + ASSERT_FALSE(config.LoadConfigFile(config_path + INVALID_CONFIG_FILE).ok()); + ASSERT_FALSE(config.LoadConfigFile(config_path + "dummy.yaml").ok()); /* server config */ - s = config.SetServerConfigAddress("0.0.0"); - ASSERT_FALSE(s.ok()); - s = config.SetServerConfigAddress("0.0.0.256"); - ASSERT_FALSE(s.ok()); + ASSERT_FALSE(config.SetServerConfigAddress("0.0.0").ok()); + ASSERT_FALSE(config.SetServerConfigAddress("0.0.0.256").ok()); - s = config.SetServerConfigPort("a"); - ASSERT_FALSE(s.ok()); - s = config.SetServerConfigPort("99999"); - ASSERT_FALSE(s.ok()); + ASSERT_FALSE(config.SetServerConfigPort("a").ok()); + ASSERT_FALSE(config.SetServerConfigPort("99999").ok()); - s = config.SetServerConfigDeployMode("cluster"); - ASSERT_FALSE(s.ok()); + ASSERT_FALSE(config.SetServerConfigDeployMode("cluster").ok()); - s = config.SetServerConfigTimeZone("GM"); - ASSERT_FALSE(s.ok()); - s = config.SetServerConfigTimeZone("GMT8"); - ASSERT_FALSE(s.ok()); - s = config.SetServerConfigTimeZone("UTCA"); - ASSERT_FALSE(s.ok()); + ASSERT_FALSE(config.SetServerConfigTimeZone("GM").ok()); + ASSERT_FALSE(config.SetServerConfigTimeZone("GMT8").ok()); + ASSERT_FALSE(config.SetServerConfigTimeZone("UTCA").ok()); /* db config */ - s = config.SetDBConfigPrimaryPath(""); - ASSERT_FALSE(s.ok()); + ASSERT_FALSE(config.SetDBConfigBackendUrl("http://www.google.com").ok()); + ASSERT_FALSE(config.SetDBConfigBackendUrl("sqlite://:@:").ok()); + ASSERT_FALSE(config.SetDBConfigBackendUrl("mysql://root:123456@127.0.0.1/milvus").ok()); - // s = config.SetDBConfigSecondaryPath(""); - // ASSERT_FALSE(s.ok()); + ASSERT_FALSE(config.SetDBConfigArchiveDiskThreshold("0x10").ok()); - s = config.SetDBConfigBackendUrl("http://www.google.com"); - ASSERT_FALSE(s.ok()); - s = config.SetDBConfigBackendUrl("sqlite://:@:"); - ASSERT_FALSE(s.ok()); - s = config.SetDBConfigBackendUrl("mysql://root:123456@127.0.0.1/milvus"); - ASSERT_FALSE(s.ok()); + ASSERT_FALSE(config.SetDBConfigArchiveDaysThreshold("0x10").ok()); - s = config.SetDBConfigArchiveDiskThreshold("0x10"); - ASSERT_FALSE(s.ok()); + ASSERT_FALSE(config.SetDBConfigInsertBufferSize("a").ok()); + ASSERT_FALSE(config.SetDBConfigInsertBufferSize("0").ok()); + ASSERT_FALSE(config.SetDBConfigInsertBufferSize("2048").ok()); - s = config.SetDBConfigArchiveDaysThreshold("0x10"); - ASSERT_FALSE(s.ok()); + /* storage config */ + ASSERT_FALSE(config.SetStorageConfigPrimaryPath("").ok()); - s = config.SetDBConfigInsertBufferSize("a"); - ASSERT_FALSE(s.ok()); - s = config.SetDBConfigInsertBufferSize("0"); - ASSERT_FALSE(s.ok()); - s = config.SetDBConfigInsertBufferSize("2048"); - ASSERT_FALSE(s.ok()); + // ASSERT_FALSE(config.SetStorageConfigSecondaryPath("").ok()); + + ASSERT_FALSE(config.SetStorageConfigMinioEnable("10").ok()); + + ASSERT_FALSE(config.SetStorageConfigMinioAddress("127.0.0").ok()); + + ASSERT_FALSE(config.SetStorageConfigMinioPort("100").ok()); + ASSERT_FALSE(config.SetStorageConfigMinioPort("100000").ok()); + + ASSERT_FALSE(config.SetStorageConfigMinioAccessKey("").ok()); + + ASSERT_FALSE(config.SetStorageConfigMinioSecretKey("").ok()); + + ASSERT_FALSE(config.SetStorageConfigMinioBucket("").ok()); /* metric config */ - s = config.SetMetricConfigEnableMonitor("Y"); - ASSERT_FALSE(s.ok()); + ASSERT_FALSE(config.SetMetricConfigEnableMonitor("Y").ok()); - s = config.SetMetricConfigCollector("zilliz"); - ASSERT_FALSE(s.ok()); + ASSERT_FALSE(config.SetMetricConfigCollector("zilliz").ok()); - s = config.SetMetricConfigPrometheusPort("0xff"); - ASSERT_FALSE(s.ok()); + ASSERT_FALSE(config.SetMetricConfigPrometheusPort("0xff").ok()); /* cache config */ - s = config.SetCacheConfigCpuCacheCapacity("a"); - ASSERT_FALSE(s.ok()); - s = config.SetCacheConfigCpuCacheCapacity("0"); - ASSERT_FALSE(s.ok()); - s = config.SetCacheConfigCpuCacheCapacity("2048"); - ASSERT_FALSE(s.ok()); + ASSERT_FALSE(config.SetCacheConfigCpuCacheCapacity("a").ok()); + ASSERT_FALSE(config.SetCacheConfigCpuCacheCapacity("0").ok()); + ASSERT_FALSE(config.SetCacheConfigCpuCacheCapacity("2048").ok()); - s = config.SetCacheConfigCpuCacheThreshold("a"); - ASSERT_FALSE(s.ok()); - s = config.SetCacheConfigCpuCacheThreshold("1.0"); - ASSERT_FALSE(s.ok()); + ASSERT_FALSE(config.SetCacheConfigCpuCacheThreshold("a").ok()); + ASSERT_FALSE(config.SetCacheConfigCpuCacheThreshold("1.0").ok()); - s = config.SetCacheConfigCacheInsertData("N"); - ASSERT_FALSE(s.ok()); + ASSERT_FALSE(config.SetCacheConfigCacheInsertData("N").ok()); /* engine config */ - s = config.SetEngineConfigUseBlasThreshold("0xff"); - ASSERT_FALSE(s.ok()); + ASSERT_FALSE(config.SetEngineConfigUseBlasThreshold("0xff").ok()); - s = config.SetEngineConfigOmpThreadNum("a"); - ASSERT_FALSE(s.ok()); - s = config.SetEngineConfigOmpThreadNum("10000"); - ASSERT_FALSE(s.ok()); + ASSERT_FALSE(config.SetEngineConfigOmpThreadNum("a").ok()); + ASSERT_FALSE(config.SetEngineConfigOmpThreadNum("10000").ok()); #ifdef MILVUS_GPU_VERSION - s = config.SetEngineConfigGpuSearchThreshold("-1"); - ASSERT_FALSE(s.ok()); + ASSERT_FALSE(config.SetEngineConfigGpuSearchThreshold("-1").ok()); +#endif /* gpu resource config */ - s = config.SetGpuResourceConfigEnable("ok"); - ASSERT_FALSE(s.ok()); +#ifdef MILVUS_GPU_VERSION + ASSERT_FALSE(config.SetGpuResourceConfigEnable("ok").ok()); - s = config.SetGpuResourceConfigCacheCapacity("a"); - ASSERT_FALSE(s.ok()); - s = config.SetGpuResourceConfigCacheCapacity("128"); - ASSERT_FALSE(s.ok()); + ASSERT_FALSE(config.SetGpuResourceConfigCacheCapacity("a").ok()); + ASSERT_FALSE(config.SetGpuResourceConfigCacheCapacity("128").ok()); - s = config.SetGpuResourceConfigCacheThreshold("a"); - ASSERT_FALSE(s.ok()); - s = config.SetGpuResourceConfigCacheThreshold("1.0"); - ASSERT_FALSE(s.ok()); + ASSERT_FALSE(config.SetGpuResourceConfigCacheThreshold("a").ok()); + ASSERT_FALSE(config.SetGpuResourceConfigCacheThreshold("1.0").ok()); - s = config.SetGpuResourceConfigSearchResources("gpu10"); - ASSERT_FALSE(s.ok()); + ASSERT_FALSE(config.SetGpuResourceConfigSearchResources("gpu10").ok()); - s = config.SetGpuResourceConfigBuildIndexResources("gup2"); - ASSERT_FALSE(s.ok()); - s = config.SetGpuResourceConfigBuildIndexResources("gpu16"); - ASSERT_FALSE(s.ok()); + ASSERT_FALSE(config.SetGpuResourceConfigBuildIndexResources("gup2").ok()); + ASSERT_FALSE(config.SetGpuResourceConfigBuildIndexResources("gpu16").ok()); #endif } TEST_F(ConfigTest, SERVER_CONFIG_TEST) { std::string config_path(CONFIG_PATH); milvus::server::Config& config = milvus::server::Config::GetInstance(); - milvus::Status s = config.LoadConfigFile(config_path + VALID_CONFIG_FILE); - ASSERT_TRUE(s.ok()); - s = config.ValidateConfig(); - ASSERT_TRUE(s.ok()); + ASSERT_TRUE(config.LoadConfigFile(config_path + VALID_CONFIG_FILE).ok()); + + ASSERT_TRUE(config.ValidateConfig().ok()); std::string config_json_str; config.GetConfigJsonStr(config_json_str); std::cout << config_json_str << std::endl; - s = config.ResetDefaultConfig(); - ASSERT_TRUE(s.ok()); + ASSERT_TRUE(config.ResetDefaultConfig().ok()); } diff --git a/core/unittest/server/test_rpc.cpp b/core/unittest/server/test_rpc.cpp index 46ef866e61..531956e00a 100644 --- a/core/unittest/server/test_rpc.cpp +++ b/core/unittest/server/test_rpc.cpp @@ -73,10 +73,10 @@ class RpcHandlerTest : public testing::Test { milvus::engine::DBOptions opt; milvus::server::Config::GetInstance().SetDBConfigBackendUrl("sqlite://:@:/"); - milvus::server::Config::GetInstance().SetDBConfigPrimaryPath("/tmp/milvus_test"); - milvus::server::Config::GetInstance().SetDBConfigSecondaryPath(""); milvus::server::Config::GetInstance().SetDBConfigArchiveDiskThreshold(""); milvus::server::Config::GetInstance().SetDBConfigArchiveDaysThreshold(""); + milvus::server::Config::GetInstance().SetStorageConfigPrimaryPath("/tmp/milvus_test"); + milvus::server::Config::GetInstance().SetStorageConfigSecondaryPath(""); milvus::server::Config::GetInstance().SetCacheConfigCacheInsertData(""); milvus::server::Config::GetInstance().SetEngineConfigOmpThreadNum(""); diff --git a/core/unittest/server/test_util.cpp b/core/unittest/server/test_util.cpp index 01cf713bcd..5e0f12f8b3 100644 --- a/core/unittest/server/test_util.cpp +++ b/core/unittest/server/test_util.cpp @@ -92,9 +92,9 @@ TEST(UtilTest, COMMON_TEST) { tm time_struct; memset(&time_struct, 0, sizeof(tm)); milvus::server::CommonUtil::ConvertTime(tt, time_struct); - ASSERT_GT(time_struct.tm_year, 0); - ASSERT_GT(time_struct.tm_mon, 0); - ASSERT_GT(time_struct.tm_mday, 0); + ASSERT_GE(time_struct.tm_year, 0); + ASSERT_GE(time_struct.tm_mon, 0); + ASSERT_GE(time_struct.tm_mday, 0); milvus::server::CommonUtil::ConvertTime(time_struct, tt); ASSERT_GT(tt, 0); diff --git a/core/unittest/server/utils.cpp b/core/unittest/server/utils.cpp index c0f32bb716..301397e047 100644 --- a/core/unittest/server/utils.cpp +++ b/core/unittest/server/utils.cpp @@ -37,14 +37,15 @@ static const char* VALID_CONFIG_STR = " time_zone: UTC+8\n" "\n" "db_config:\n" - " primary_path: /tmp/milvus # path used to store data and meta\n" - " secondary_path: # path used to store data only, split by semicolon\n" - "\n" " backend_url: sqlite://:@:/ \n" "\n" " insert_buffer_size: 4 # GB, maximum insert buffer size allowed\n" " preload_table: \n" "\n" + "storage_config:\n" + " primary_path: /tmp/milvus # path used to store data and meta\n" + " secondary_path: # path used to store data only, split by semicolon\n" + "\n" "metric_config:\n" " enable_monitor: false # enable monitoring or not\n" " collector: prometheus # prometheus\n" diff --git a/core/unittest/storage/CMakeLists.txt b/core/unittest/storage/CMakeLists.txt new file mode 100644 index 0000000000..10071facb4 --- /dev/null +++ b/core/unittest/storage/CMakeLists.txt @@ -0,0 +1,44 @@ +#------------------------------------------------------------------------------- +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http:#www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +#------------------------------------------------------------------------------- + +set(test_files + ${CMAKE_CURRENT_SOURCE_DIR}/test_s3_client.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/utils.cpp + ) + +include_directories("${CUDA_TOOLKIT_ROOT_DIR}/include") +link_directories("${CUDA_TOOLKIT_ROOT_DIR}/lib64") + +set(util_files + ${MILVUS_THIRDPARTY_SRC}/easyloggingpp/easylogging++.cc + ${MILVUS_ENGINE_SRC}/utils/Status.cpp) + +add_executable(test_storage + ${test_files} + ${util_files} + ${common_files} + ) + +target_link_libraries(test_storage + stdc++ + knowhere + ${unittest_libs} + ) + +install(TARGETS test_storage DESTINATION bin) \ No newline at end of file diff --git a/core/unittest/storage/test_s3_client.cpp b/core/unittest/storage/test_s3_client.cpp new file mode 100644 index 0000000000..0dda5fd7e6 --- /dev/null +++ b/core/unittest/storage/test_s3_client.cpp @@ -0,0 +1,81 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + +#include +#include +#include + +#include "easyloggingpp/easylogging++.h" +#include "server/Config.h" +#include "storage/IStorage.h" +#include "storage/s3/S3ClientWrapper.h" +#include "storage/utils.h" + +INITIALIZE_EASYLOGGINGPP + +TEST_F(StorageTest, S3_CLIENT_TEST) { + const std::string filename = "/tmp/test_file_in"; + const std::string filename_out = "/tmp/test_file_out"; + const std::string object_name = "/tmp/test_obj"; + const std::string content = "abcdefghijklmnopqrstuvwxyz"; + + std::string config_path(CONFIG_PATH); + config_path += CONFIG_FILE; + milvus::server::Config& config = milvus::server::Config::GetInstance(); + ASSERT_TRUE(config.LoadConfigFile(config_path).ok()); + + auto storage_inst = milvus::storage::S3ClientWrapper::GetInstance(); + ASSERT_TRUE(storage_inst.StartService().ok()); + + /////////////////////////////////////////////////////////////////////////// + /* check PutObjectFile() and GetObjectFile() */ + { + std::ofstream fs_in(filename); + std::stringstream ss_in; + for (int i = 0; i < 1024; ++i) { + ss_in << i; + } + fs_in << ss_in.str() << std::endl; + fs_in.close(); + ASSERT_TRUE(storage_inst.PutObjectFile(filename, filename).ok()); + + ASSERT_TRUE(storage_inst.GetObjectFile(filename, filename_out).ok()); + std::ifstream fs_out(filename_out); + std::string str_out; + fs_out >> str_out; + ASSERT_TRUE(str_out == ss_in.str()); + } + + /////////////////////////////////////////////////////////////////////////// + /* check PutObjectStr() and GetObjectStr() */ + { + ASSERT_TRUE(storage_inst.PutObjectStr(object_name, content).ok()); + + std::string content_out; + ASSERT_TRUE(storage_inst.GetObjectStr(object_name, content_out).ok()); + ASSERT_TRUE(content_out == content); + } + + /////////////////////////////////////////////////////////////////////////// + ASSERT_TRUE(storage_inst.DeleteObjects("/tmp").ok()); + + ASSERT_TRUE(storage_inst.DeleteBucket().ok()); + + ASSERT_TRUE(storage_inst.StopService().ok()); +} + diff --git a/core/unittest/storage/utils.cpp b/core/unittest/storage/utils.cpp new file mode 100644 index 0000000000..bcc76c0bed --- /dev/null +++ b/core/unittest/storage/utils.cpp @@ -0,0 +1,60 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include +#include + +#include "storage/utils.h" +#include "utils/CommonUtil.h" + +namespace { + +static const char* CONFIG_STR = + "storage_config:\n" + " primary_path: /tmp/milvus\n" + " secondary_path:\n" + " minio_enable: true\n" + " minio_address: 127.0.0.1\n" + " minio_port: 9000\n" + " minio_access_key: minioadmin\n" + " minio_secret_key: minioadmin\n" + " minio_bucket: milvus-bucket\n" + "\n"; + +void +WriteToFile(const std::string& file_path, const char* content) { + std::fstream fs(file_path.c_str(), std::ios_base::out); + + // write data to file + fs << content; + fs.close(); +} + +} // namespace + +void +StorageTest::SetUp() { + std::string config_path(CONFIG_PATH); + milvus::server::CommonUtil::CreateDirectory(config_path); + WriteToFile(config_path + CONFIG_FILE, CONFIG_STR); +} + +void +StorageTest::TearDown() { + std::string config_path(CONFIG_PATH); + milvus::server::CommonUtil::DeleteDirectory(config_path); +} diff --git a/core/unittest/storage/utils.h b/core/unittest/storage/utils.h new file mode 100644 index 0000000000..e194dafddb --- /dev/null +++ b/core/unittest/storage/utils.h @@ -0,0 +1,31 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +static const char* CONFIG_PATH = "/tmp/milvus_test/"; +static const char* CONFIG_FILE = "server_config.yaml"; + +class StorageTest : public ::testing::Test { + protected: + void + SetUp() override; + void + TearDown() override; +}; diff --git a/core/unittest/wrapper/CMakeLists.txt b/core/unittest/wrapper/CMakeLists.txt index a320ef723d..ce08a4f9df 100644 --- a/core/unittest/wrapper/CMakeLists.txt +++ b/core/unittest/wrapper/CMakeLists.txt @@ -28,7 +28,13 @@ endif () set(wrapper_files ${MILVUS_ENGINE_SRC}/wrapper/DataTransfer.cpp ${MILVUS_ENGINE_SRC}/wrapper/VecImpl.cpp - ${MILVUS_ENGINE_SRC}/wrapper/VecIndex.cpp) + ${MILVUS_ENGINE_SRC}/wrapper/VecIndex.cpp + ) + +set(storage_files + ${MILVUS_ENGINE_SRC}/storage/file/FileIOReader.cpp + ${MILVUS_ENGINE_SRC}/storage/file/FileIOWriter.cpp + ) set(util_files utils.cpp @@ -39,6 +45,7 @@ set(util_files add_executable(test_wrapper ${test_files} ${wrapper_files} + ${storage_files} ${util_files} ${common_files}) diff --git a/core/unittest/wrapper/utils.cpp b/core/unittest/wrapper/utils.cpp index a5f8e1b6b2..f4329a4a5f 100644 --- a/core/unittest/wrapper/utils.cpp +++ b/core/unittest/wrapper/utils.cpp @@ -33,15 +33,16 @@ static const char* CONFIG_STR = " time_zone: UTC+8\n" "\n" "db_config:\n" - " primary_path: /tmp/milvus # path used to store data and meta\n" - " secondary_path: # path used to store data only, split by semicolon\n" - "\n" " backend_url: sqlite://:@:/ # URI format: dialect://username:password@host:port/database\n" " \n" " # Replace 'dialect' with 'mysql' or 'sqlite'\n" "\n" " insert_buffer_size: 4 # GB, maximum insert buffer size allowed\n" "\n" + "storage_config:\n" + " primary_path: /tmp/milvus # path used to store data and meta\n" + " secondary_path: # path used to store data only, split by semicolon\n" + "\n" "metric_config:\n" " enable_monitor: false # enable monitoring or not\n" " collector: prometheus # prometheus\n" diff --git a/docs/README.md b/docs/README.md index f5f964de0d..51f9f556f0 100644 --- a/docs/README.md +++ b/docs/README.md @@ -16,8 +16,10 @@ The following is a list of existing test reports: - [IVF_SQ8](test_report/milvus_ivfsq8_test_report_detailed_version.md) - [IVF_SQ8H](test_report/milvus_ivfsq8h_test_report_detailed_version.md) +- [IVFLAT](test-report/ivfflat_test_report_en.md) To read the CN version of these reports: - [IVF_SQ8_cn](test_report/milvus_ivfsq8_test_report_detailed_version_cn.md) - [IVF_SQ8H_cn](test_report/milvus_ivfsq8h_test_report_detailed_version_cn.md) +- [IVFLAT_cn](test-report/ivfflat_test_report_cn.md)