diff --git a/CHANGELOG.md b/CHANGELOG.md index d24663b896..016fd33a5a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,7 +20,11 @@ Please mark all change in change log and use the ticket from JIRA. - \#440 - Query API in customization still uses old version - \#440 - Server cannot startup with gpu_resource_config.enable=false in GPU version - \#458 - Index data is not compatible between 0.5 and 0.6 +- \#465 - Server hang caused by searching with nsg index - \#486 - gpu no usage during index building +- \#509 - IVF_PQ index build trapped into dead loop caused by invalid params +- \#513 - Unittest DELETE_BY_RANGE sometimes failed +- \#527 - faiss benchmark not compatible with faiss 1.6.0 - \#530 - BuildIndex stop when do build index and search simultaneously ## Feature @@ -31,6 +35,7 @@ Please mark all change in change log and use the ticket from JIRA. - \#227 - Support new index types SPTAG-KDT and SPTAG-BKT - \#346 - Support build index with multiple gpu - \#488 - Add log in scheduler/optimizer +- \#502 - C++ SDK support IVFPQ and SPTAG ## Improvement - \#255 - Add ivfsq8 test report detailed version diff --git a/README.md b/README.md index 110f0b6c1e..c06074277c 100644 --- a/README.md +++ b/README.md @@ -7,6 +7,7 @@ [![codebeat badge](https://codebeat.co/badges/e030a4f6-b126-4475-a938-4723d54ec3a7?style=plastic)](https://codebeat.co/projects/github-com-jinhai-cn-milvus-master) ![Release](https://img.shields.io/badge/release-v0.5.3-yellowgreen) ![Release_date](https://img.shields.io/badge/release%20date-November-yellowgreen) +[![codecov](https://codecov.io/gh/milvus-io/milvus/branch/master/graph/badge.svg)](https://codecov.io/gh/milvus-io/milvus) [中文版](README_CN.md) | [日本語版](README_JP.md) diff --git a/README_CN.md b/README_CN.md index 374cefa9bd..1c554ac0d8 100644 --- a/README_CN.md +++ b/README_CN.md @@ -6,6 +6,8 @@ [![codebeat badge](https://codebeat.co/badges/e030a4f6-b126-4475-a938-4723d54ec3a7?style=plastic)](https://codebeat.co/projects/github-com-jinhai-cn-milvus-master) ![Release](https://img.shields.io/badge/release-v0.5.3-yellowgreen) ![Release_date](https://img.shields.io/badge/release_date-October-yellowgreen) +[![codecov](https://codecov.io/gh/milvus-io/milvus/branch/master/graph/badge.svg)](https://codecov.io/gh/milvus-io/milvus) + # 欢迎来到 Milvus @@ -15,7 +17,7 @@ Milvus 是一款开源的、针对海量特征向量的相似性搜索引擎。 若要了解 Milvus 详细介绍和整体架构，请访问 [Milvus 简介](https://www.milvus.io/docs/zh-CN/aboutmilvus/overview/)。 -Milvus 提供稳定的 [Python](https://github.com/milvus-io/pymilvus)、[Java](https://github.com/milvus-io/milvus-sdk-java) 以及 C++ 的 API 接口。 +Milvus 提供稳定的 [Python](https://github.com/milvus-io/pymilvus)、[Java](https://github.com/milvus-io/milvus-sdk-java) 以及[C++](https://github.com/milvus-io/milvus/tree/master/core/src/sdk) 的 API 接口。 通过 [版本发布说明](https://milvus.io/docs/zh-CN/release/v0.5.3/) 获取最新版本的功能和更新。 diff --git a/ci/jenkins/Jenkinsfile b/ci/jenkins/Jenkinsfile index 01048bd953..0eed167fb4 100644 --- a/ci/jenkins/Jenkinsfile +++ b/ci/jenkins/Jenkinsfile @@ -114,7 +114,8 @@ pipeline { stage("Deploy to Development") { environment { - HELM_RELEASE_NAME = "${env.PIPELINE_NAME}-${env.SEMVER}-${env.BUILD_NUMBER}-single-${env.BINRARY_VERSION}".toLowerCase() + FROMAT_SEMVER = "${env.SEMVER}".replaceAll("\\.", "-") + HELM_RELEASE_NAME = "${env.PIPELINE_NAME}-${env.FROMAT_SEMVER}-${env.BUILD_NUMBER}-single-${env.BINRARY_VERSION}".toLowerCase() } agent { @@ -248,7 +249,8 @@ pipeline { stage("Deploy to Development") { environment { - HELM_RELEASE_NAME = "${env.PIPELINE_NAME}-${env.SEMVER}-${env.BUILD_NUMBER}-single-${env.BINRARY_VERSION}".toLowerCase() + FROMAT_SEMVER = "${env.SEMVER}".replaceAll("\\.", "-") + HELM_RELEASE_NAME = "${env.PIPELINE_NAME}-${env.FROMAT_SEMVER}-${env.BUILD_NUMBER}-single-${env.BINRARY_VERSION}".toLowerCase() } agent { diff --git a/ci/jenkins/step/package.groovy b/ci/jenkins/step/package.groovy index 80128b3b74..35d2d8cfdf 100644 --- a/ci/jenkins/step/package.groovy +++ b/ci/jenkins/step/package.groovy @@ -2,8 +2,9 @@ timeout(time: 5, unit: 'MINUTES') { dir ("ci/jenkins/scripts") { sh "pip3 install -r requirements.txt" sh "./yaml_processor.py merge -f /opt/milvus/conf/server_config.yaml -m ../yaml/update_server_config.yaml -i && rm /opt/milvus/conf/server_config.yaml.bak" - sh "sed -i 's/\/tmp\/milvus/\/opt\/milvus/g' /opt/milvus/conf/log_config.conf" + sh "sed -i 's/\\/tmp\\/milvus/\\/opt\\/milvus/g' /opt/milvus/conf/log_config.conf" } + sh "rm -rf /opt/milvus/unittest" sh "tar -zcvf ./${PROJECT_NAME}-${PACKAGE_VERSION}.tar.gz -C /opt/ milvus" withCredentials([usernamePassword(credentialsId: "${params.JFROG_CREDENTIALS_ID}", usernameVariable: 'JFROG_USERNAME', passwordVariable: 'JFROG_PASSWORD')]) { def uploadStatus = sh(returnStatus: true, script: "curl -u${JFROG_USERNAME}:${JFROG_PASSWORD} -T ./${PROJECT_NAME}-${PACKAGE_VERSION}.tar.gz ${params.JFROG_ARTFACTORY_URL}/milvus/package/${PROJECT_NAME}-${PACKAGE_VERSION}.tar.gz") diff --git a/core/CMakeLists.txt b/core/CMakeLists.txt index 671330c192..ba391c2d6b 100644 --- a/core/CMakeLists.txt +++ b/core/CMakeLists.txt @@ -31,9 +31,21 @@ GET_CURRENT_TIME(BUILD_TIME) string(REGEX REPLACE "\n" "" BUILD_TIME ${BUILD_TIME}) message(STATUS "Build time = ${BUILD_TIME}") +if (NOT DEFINED CMAKE_BUILD_TYPE) + set(CMAKE_BUILD_TYPE Release CACHE STRING "Choose the type of build.") +endif () + +set (GIT_BRANCH_NAME_REGEX "[0-9]+\\.[0-9]+\\.[0-9]") + MACRO(GET_GIT_BRANCH_NAME GIT_BRANCH_NAME) execute_process(COMMAND sh "-c" "git log --decorate | head -n 1 | sed 's/.*(\\(.*\\))/\\1/' | sed 's/.* \\(.*\\),.*/\\1/' | sed 's=[a-zA-Z]*\/==g'" OUTPUT_VARIABLE ${GIT_BRANCH_NAME}) + if(NOT GIT_BRANCH_NAME MATCHES "${GIT_BRANCH_NAME_REGEX}") + execute_process(COMMAND "git" rev-parse --abbrev-ref HEAD OUTPUT_VARIABLE ${GIT_BRANCH_NAME}) + endif () + if(NOT GIT_BRANCH_NAME MATCHES "${GIT_BRANCH_NAME_REGEX}") + execute_process(COMMAND "git" symbolic-ref --short -q HEAD HEAD OUTPUT_VARIABLE ${GIT_BRANCH_NAME}) + endif () ENDMACRO(GET_GIT_BRANCH_NAME) GET_GIT_BRANCH_NAME(GIT_BRANCH_NAME) @@ -43,7 +55,7 @@ if (NOT GIT_BRANCH_NAME STREQUAL "") endif () set(MILVUS_VERSION "${GIT_BRANCH_NAME}") -string(REGEX MATCH "[0-9]+\\.[0-9]+\\.[0-9]" MILVUS_VERSION "${MILVUS_VERSION}") +string(REGEX MATCH "${GIT_BRANCH_NAME_REGEX}" MILVUS_VERSION "${MILVUS_VERSION}") if (CMAKE_BUILD_TYPE STREQUAL "Release") set(BUILD_TYPE "Release") diff --git a/core/build.sh b/core/build.sh index 3afb5d1b37..badcc5a032 100755 --- a/core/build.sh +++ b/core/build.sh @@ -56,7 +56,7 @@ while getopts "p:d:t:f:ulrcgjhxzme" arg; do USE_JFROG_CACHE="ON" ;; x) - CUSTOMIZATION="OFF" # force use ori faiss + CUSTOMIZATION="ON" ;; g) GPU_VERSION="ON" diff --git a/core/src/cache/GpuCacheMgr.cpp b/core/src/cache/GpuCacheMgr.cpp index 72229527fa..1802fb3935 100644 --- a/core/src/cache/GpuCacheMgr.cpp +++ b/core/src/cache/GpuCacheMgr.cpp @@ -25,6 +25,7 @@ namespace milvus { namespace cache { +#ifdef MILVUS_GPU_VERSION std::mutex GpuCacheMgr::mutex_; std::unordered_map GpuCacheMgr::instance_; @@ -76,6 +77,7 @@ GpuCacheMgr::GetIndex(const std::string& key) { DataObjPtr obj = GetItem(key); return obj; } +#endif } // namespace cache } // namespace milvus diff --git a/core/src/cache/GpuCacheMgr.h b/core/src/cache/GpuCacheMgr.h index 4d434b2cfb..06dd44cca2 100644 --- a/core/src/cache/GpuCacheMgr.h +++ b/core/src/cache/GpuCacheMgr.h @@ -25,6 +25,7 @@ namespace milvus { namespace cache { +#ifdef MILVUS_GPU_VERSION class GpuCacheMgr; using GpuCacheMgrPtr = std::shared_ptr; @@ -42,6 +43,7 @@ class GpuCacheMgr : public CacheMgr { static std::mutex mutex_; static std::unordered_map instance_; }; +#endif } // namespace cache } // namespace milvus diff --git a/core/src/db/DBImpl.cpp b/core/src/db/DBImpl.cpp index 51ea665064..1c18c22409 100644 --- a/core/src/db/DBImpl.cpp +++ b/core/src/db/DBImpl.cpp @@ -105,7 +105,8 @@ DBImpl::Stop() { shutting_down_.store(true, std::memory_order_release); // makesure all memory data serialized - MemSerialize(); + std::set sync_table_ids; + SyncMemData(sync_table_ids); // wait compaction/buildindex finish bg_timer_thread_.join(); @@ -329,7 +330,10 @@ DBImpl::CreateIndex(const std::string& table_id, const TableIndex& index) { return SHUTDOWN_ERROR; } - Status status; + // serialize memory data + std::set sync_table_ids; + auto status = SyncMemData(sync_table_ids); + { std::unique_lock lock(build_index_mutex_); @@ -588,12 +592,12 @@ DBImpl::StartMetricTask() { } Status -DBImpl::MemSerialize() { +DBImpl::SyncMemData(std::set& sync_table_ids) { std::lock_guard lck(mem_serialize_mutex_); std::set temp_table_ids; mem_mgr_->Serialize(temp_table_ids); for (auto& id : temp_table_ids) { - compact_table_ids_.insert(id); + sync_table_ids.insert(id); } if (!temp_table_ids.empty()) { @@ -612,7 +616,7 @@ DBImpl::StartCompactionTask() { } // serialize memory data - MemSerialize(); + SyncMemData(compact_table_ids_); // compactiong has been finished? { diff --git a/core/src/db/DBImpl.h b/core/src/db/DBImpl.h index bff56efded..82a5d3096b 100644 --- a/core/src/db/DBImpl.h +++ b/core/src/db/DBImpl.h @@ -150,7 +150,7 @@ class DBImpl : public DB { BackgroundBuildIndex(); Status - MemSerialize(); + SyncMemData(std::set& sync_table_ids); Status GetFilesToBuildIndex(const std::string& table_id, const std::vector& file_types, diff --git a/core/src/db/engine/ExecutionEngineImpl.cpp b/core/src/db/engine/ExecutionEngineImpl.cpp index ca307b90fc..c0ab4e829e 100644 --- a/core/src/db/engine/ExecutionEngineImpl.cpp +++ b/core/src/db/engine/ExecutionEngineImpl.cpp @@ -151,6 +151,7 @@ ExecutionEngineImpl::HybridLoad() const { return; } +#ifdef MILVUS_GPU_VERSION const std::string key = location_ + ".quantizer"; server::Config& config = server::Config::GetInstance(); @@ -205,6 +206,7 @@ ExecutionEngineImpl::HybridLoad() const { auto cache_quantizer = std::make_shared(quantizer); cache::GpuCacheMgr::GetInstance(best_device_id)->InsertItem(key, cache_quantizer); } +#endif } void @@ -342,6 +344,7 @@ ExecutionEngineImpl::CopyToGpu(uint64_t device_id, bool hybrid) { } #endif +#ifdef MILVUS_GPU_VERSION auto index = std::static_pointer_cast(cache::GpuCacheMgr::GetInstance(device_id)->GetIndex(location_)); bool already_in_cache = (index != nullptr); if (already_in_cache) { @@ -364,16 +367,19 @@ ExecutionEngineImpl::CopyToGpu(uint64_t device_id, bool hybrid) { if (!already_in_cache) { GpuCache(device_id); } +#endif return Status::OK(); } Status ExecutionEngineImpl::CopyToIndexFileToGpu(uint64_t device_id) { +#ifdef MILVUS_GPU_VERSION gpu_num_ = device_id; auto to_index_data = std::make_shared(PhysicalSize()); cache::DataObjPtr obj = std::static_pointer_cast(to_index_data); milvus::cache::GpuCacheMgr::GetInstance(device_id)->InsertItem(location_, obj); +#endif return Status::OK(); } @@ -584,15 +590,17 @@ ExecutionEngineImpl::Cache() { Status ExecutionEngineImpl::GpuCache(uint64_t gpu_id) { +#ifdef MILVUS_GPU_VERSION cache::DataObjPtr obj = std::static_pointer_cast(index_); milvus::cache::GpuCacheMgr::GetInstance(gpu_id)->InsertItem(location_, obj); - +#endif return Status::OK(); } // TODO(linxj): remove. Status ExecutionEngineImpl::Init() { +#ifdef MILVUS_GPU_VERSION server::Config& config = server::Config::GetInstance(); std::vector gpu_ids; Status s = config.GetGpuResourceConfigBuildIndexResources(gpu_ids); @@ -604,6 +612,9 @@ ExecutionEngineImpl::Init() { std::string msg = "Invalid gpu_num"; return Status(SERVER_INVALID_ARGUMENT, msg); +#else + return Status::OK(); +#endif } } // namespace engine diff --git a/core/src/index/cmake/BuildUtilsCore.cmake b/core/src/index/cmake/BuildUtilsCore.cmake index 1bf6ba2af4..2261ce3e29 100644 --- a/core/src/index/cmake/BuildUtilsCore.cmake +++ b/core/src/index/cmake/BuildUtilsCore.cmake @@ -74,7 +74,7 @@ function(ExternalProject_Use_Cache project_name package_file install_path) ${CMAKE_COMMAND} -E echo "Extracting ${package_file} to ${install_path}" COMMAND - ${CMAKE_COMMAND} -E tar xzvf ${package_file} ${install_path} + ${CMAKE_COMMAND} -E tar xzf ${package_file} ${install_path} WORKING_DIRECTORY ${INDEX_BINARY_DIR} ) diff --git a/core/src/index/knowhere/knowhere/common/Timer.cpp b/core/src/index/knowhere/knowhere/common/Timer.cpp index fefe1e1705..141e1ceed6 100644 --- a/core/src/index/knowhere/knowhere/common/Timer.cpp +++ b/core/src/index/knowhere/knowhere/common/Timer.cpp @@ -17,6 +17,7 @@ #include // TODO(linxj): using Log instead +#include "knowhere/common/Log.h" #include "knowhere/common/Timer.h" namespace knowhere { @@ -51,30 +52,18 @@ TimeRecorder::PrintTimeRecord(const std::string& msg, double span) { std::cout << str_log << std::endl; break; } - // case 1: { - // SERVER_LOG_DEBUG << str_log; - // break; - //} + case 1: { + KNOWHERE_LOG_DEBUG << str_log; + break; + } // case 2: { - // SERVER_LOG_INFO << str_log; - // break; - //} + // KNOWHERE_LOG_TRACE << str_log; + // break; + // } // case 3: { - // SERVER_LOG_WARNING << str_log; - // break; - //} - // case 4: { - // SERVER_LOG_ERROR << str_log; - // break; - //} - // case 5: { - // SERVER_LOG_FATAL << str_log; - // break; - //} - // default: { - // SERVER_LOG_INFO << str_log; - // break; - //} + // KNOWHERE_LOG_WARNING << str_log; + // break; + // } } } diff --git a/core/src/index/knowhere/knowhere/index/vector_index/nsg/NSG.cpp b/core/src/index/knowhere/knowhere/index/vector_index/nsg/NSG.cpp index bdf538c204..b4e00e57b7 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/nsg/NSG.cpp +++ b/core/src/index/knowhere/knowhere/index/vector_index/nsg/NSG.cpp @@ -718,29 +718,38 @@ NsgIndex::Search(const float* query, const unsigned& nq, const unsigned& dim, co int64_t* ids, SearchParams& params) { std::vector> resset(nq); - TimeRecorder rc("search"); - if (nq == 1) { + params.search_length = k; + TimeRecorder rc("NsgIndex::search", 1); + // TODO(linxj): when to use openmp + if (nq <= 4) { GetNeighbors(query, resset[0], nsg, ¶ms); } else { -//#pragma omp parallel for schedule(dynamic, 50) #pragma omp parallel for for (unsigned int i = 0; i < nq; ++i) { - // TODO(linxj): when to use openmp auto single_query = query + i * dim; GetNeighbors(single_query, resset[i], nsg, ¶ms); } } - rc.ElapseFromBegin("cost"); - + rc.RecordSection("search"); for (unsigned int i = 0; i < nq; ++i) { - for (unsigned int j = 0; j < k; ++j) { - // ids[i * k + j] = resset[i][j].id; - - // Fix(linxj): bug, reset[i][j] out of range - ids[i * k + j] = ids_[resset[i][j].id]; - dist[i * k + j] = resset[i][j].distance; + int64_t var = resset[i].size() - k; + if (var >= 0) { + for (unsigned int j = 0; j < k; ++j) { + ids[i * k + j] = ids_[resset[i][j].id]; + dist[i * k + j] = resset[i][j].distance; + } + } else { + for (unsigned int j = 0; j < resset[i].size(); ++j) { + ids[i * k + j] = ids_[resset[i][j].id]; + dist[i * k + j] = resset[i][j].distance; + } + for (unsigned int j = resset[i].size(); j < k; ++j) { + ids[i * k + j] = -1; + dist[i * k + j] = -1; + } } } + rc.RecordSection("merge"); //>> Debug: test single insert // int x_0 = resset[0].size(); diff --git a/core/src/index/unittest/faiss_benchmark/faiss_benchmark_test.cpp b/core/src/index/unittest/faiss_benchmark/faiss_benchmark_test.cpp index ce5aa4bafa..0d49004621 100644 --- a/core/src/index/unittest/faiss_benchmark/faiss_benchmark_test.cpp +++ b/core/src/index/unittest/faiss_benchmark/faiss_benchmark_test.cpp @@ -15,6 +15,8 @@ // specific language governing permissions and limitations // under the License. +#define USE_FAISS_V0_2_1 0 + #include #include @@ -26,21 +28,6 @@ #include #include -#define USE_FAISS_V1_5_3 0 - -#if USE_FAISS_V1_5_3 -#include -#include -#include -#include -#include - -#else -#include -#include -#include -#endif - #include #include #include @@ -48,8 +35,22 @@ #include #include +#if USE_FAISS_V0_2_1 +#include +#include +#include +#include +#include +#else +#include +#include +#include +#endif + #ifdef CUSTOMIZATION #include +#else +#include #endif /***************************************************** @@ -295,10 +296,12 @@ load_base_data(faiss::Index*& index, const std::string& ann_test_name, const std cpu_index = faiss::gpu::index_gpu_to_cpu(gpu_index); delete gpu_index; +#ifdef CUSTOMIZATION faiss::IndexIVF* cpu_ivf_index = dynamic_cast(cpu_index); if (cpu_ivf_index != nullptr) { cpu_ivf_index->to_readonly(); } +#endif printf("[%.3f s] Writing index file: %s\n", elapsed() - t0, index_file_name.c_str()); faiss::write_index(cpu_index, index_file_name.c_str()); @@ -374,13 +377,15 @@ test_with_nprobes(const std::string& ann_test_name, const std::string& index_key faiss::Index *gpu_index, *index; if (query_mode != MODE_CPU) { faiss::gpu::GpuClonerOptions option; +#ifdef CUSTOMIZATION option.allInGpu = true; faiss::IndexComposition index_composition; index_composition.index = cpu_index; index_composition.quantizer = nullptr; - +#endif switch (query_mode) { +#ifdef CUSTOMIZATION case MODE_MIX: { index_composition.mode = 1; // 0: all data, 1: copy quantizer, 2: copy data @@ -403,7 +408,9 @@ test_with_nprobes(const std::string& ann_test_name, const std::string& index_key index = cpu_index; break; } +#endif case MODE_GPU: +#ifdef CUSTOMIZATION index_composition.mode = 0; // 0: all data, 1: copy quantizer, 2: copy data // warm up the transmission @@ -412,6 +419,14 @@ test_with_nprobes(const std::string& ann_test_name, const std::string& index_key copy_time = elapsed(); gpu_index = faiss::gpu::index_cpu_to_gpu(&res, GPU_DEVICE_IDX, &index_composition, &option); +#else + // warm up the transmission + gpu_index = faiss::gpu::index_cpu_to_gpu(&res, GPU_DEVICE_IDX, cpu_index, &option); + delete gpu_index; + + copy_time = elapsed(); + gpu_index = faiss::gpu::index_cpu_to_gpu(&res, GPU_DEVICE_IDX, cpu_index, &option); +#endif copy_time = elapsed() - copy_time; printf("[%.3f s] Copy data completed, cost %f s\n", elapsed() - t0, copy_time); diff --git a/core/src/main.cpp b/core/src/main.cpp index b39ba87997..5c97a061d2 100644 --- a/core/src/main.cpp +++ b/core/src/main.cpp @@ -59,9 +59,9 @@ print_banner() { #endif << " library." << std::endl; #ifdef MILVUS_CPU_VERSION - std::cout << "You are using Milvus CPU version" << std::endl; + std::cout << "You are using Milvus CPU edition" << std::endl; #else - std::cout << "You are using Milvus GPU version" << std::endl; + std::cout << "You are using Milvus GPU edition" << std::endl; #endif std::cout << std::endl; } diff --git a/core/src/scheduler/optimizer/BuildIndexPass.cpp b/core/src/scheduler/optimizer/BuildIndexPass.cpp index 5e5719a1bd..67036f9d7b 100644 --- a/core/src/scheduler/optimizer/BuildIndexPass.cpp +++ b/core/src/scheduler/optimizer/BuildIndexPass.cpp @@ -25,11 +25,13 @@ namespace scheduler { void BuildIndexPass::Init() { +#ifdef MILVUS_GPU_VERSION server::Config& config = server::Config::GetInstance(); Status s = config.GetGpuResourceConfigBuildIndexResources(build_gpu_ids_); if (!s.ok()) { throw; } +#endif } bool diff --git a/core/src/scheduler/optimizer/FaissFlatPass.cpp b/core/src/scheduler/optimizer/FaissFlatPass.cpp index c78f7d57e1..53256f7790 100644 --- a/core/src/scheduler/optimizer/FaissFlatPass.cpp +++ b/core/src/scheduler/optimizer/FaissFlatPass.cpp @@ -29,6 +29,7 @@ namespace scheduler { void FaissFlatPass::Init() { +#ifdef MILVUS_GPU_VERSION server::Config& config = server::Config::GetInstance(); Status s = config.GetEngineConfigGpuSearchThreshold(threshold_); if (!s.ok()) { @@ -38,6 +39,7 @@ FaissFlatPass::Init() { if (!s.ok()) { throw; } +#endif } bool diff --git a/core/src/scheduler/optimizer/FaissIVFFlatPass.cpp b/core/src/scheduler/optimizer/FaissIVFFlatPass.cpp index 17067cf24e..8ad51fb14a 100644 --- a/core/src/scheduler/optimizer/FaissIVFFlatPass.cpp +++ b/core/src/scheduler/optimizer/FaissIVFFlatPass.cpp @@ -29,6 +29,7 @@ namespace scheduler { void FaissIVFFlatPass::Init() { +#ifdef MILVUS_GPU_VERSION server::Config& config = server::Config::GetInstance(); Status s = config.GetEngineConfigGpuSearchThreshold(threshold_); if (!s.ok()) { @@ -38,6 +39,7 @@ FaissIVFFlatPass::Init() { if (!s.ok()) { throw; } +#endif } bool diff --git a/core/src/scheduler/optimizer/FaissIVFSQ8HPass.cpp b/core/src/scheduler/optimizer/FaissIVFSQ8HPass.cpp index 676ed1720e..cc04944fda 100644 --- a/core/src/scheduler/optimizer/FaissIVFSQ8HPass.cpp +++ b/core/src/scheduler/optimizer/FaissIVFSQ8HPass.cpp @@ -29,12 +29,14 @@ namespace scheduler { void FaissIVFSQ8HPass::Init() { +#ifdef MILVUS_GPU_VERSION server::Config& config = server::Config::GetInstance(); Status s = config.GetEngineConfigGpuSearchThreshold(threshold_); if (!s.ok()) { threshold_ = std::numeric_limits::max(); } s = config.GetGpuResourceConfigSearchResources(gpus); +#endif } bool diff --git a/core/src/scheduler/optimizer/FaissIVFSQ8Pass.cpp b/core/src/scheduler/optimizer/FaissIVFSQ8Pass.cpp index 4f06c1e1fc..c2a1858966 100644 --- a/core/src/scheduler/optimizer/FaissIVFSQ8Pass.cpp +++ b/core/src/scheduler/optimizer/FaissIVFSQ8Pass.cpp @@ -29,6 +29,7 @@ namespace scheduler { void FaissIVFSQ8Pass::Init() { +#ifdef MILVUS_GPU_VERSION server::Config& config = server::Config::GetInstance(); Status s = config.GetEngineConfigGpuSearchThreshold(threshold_); if (!s.ok()) { @@ -38,6 +39,7 @@ FaissIVFSQ8Pass::Init() { if (!s.ok()) { throw; } +#endif } bool diff --git a/core/src/sdk/examples/utils/Utils.cpp b/core/src/sdk/examples/utils/Utils.cpp index fa373cd498..d3bf9eec25 100644 --- a/core/src/sdk/examples/utils/Utils.cpp +++ b/core/src/sdk/examples/utils/Utils.cpp @@ -99,6 +99,12 @@ Utils::IndexTypeName(const milvus::IndexType& index_type) { return "NSG"; case milvus::IndexType::IVFSQ8H: return "IVFSQ8H"; + case milvus::IndexType::IVFPQ: + return "IVFPQ"; + case milvus::IndexType::SPTAGKDT: + return "SPTAGKDT"; + case milvus::IndexType::SPTAGBKT: + return "SPTAGBKT"; default: return "Unknown index type"; } diff --git a/core/src/sdk/include/MilvusApi.h b/core/src/sdk/include/MilvusApi.h index 9fa98deb40..5c7736d4e2 100644 --- a/core/src/sdk/include/MilvusApi.h +++ b/core/src/sdk/include/MilvusApi.h @@ -37,6 +37,9 @@ enum class IndexType { IVFSQ8 = 3, NSG = 4, IVFSQ8H = 5, + IVFPQ = 6, + SPTAGKDT = 7, + SPTAGBKT = 8, }; enum class MetricType { diff --git a/core/src/server/Config.cpp b/core/src/server/Config.cpp index 5465c6c505..95bab84391 100644 --- a/core/src/server/Config.cpp +++ b/core/src/server/Config.cpp @@ -182,6 +182,7 @@ Config::ValidateConfig() { return s; } +#ifdef MILVUS_GPU_VERSION int64_t engine_gpu_search_threshold; s = GetEngineConfigGpuSearchThreshold(engine_gpu_search_threshold); if (!s.ok()) { @@ -195,7 +196,6 @@ Config::ValidateConfig() { return s; } -#ifdef MILVUS_GPU_VERSION if (gpu_resource_enable) { int64_t resource_cache_capacity; s = GetGpuResourceConfigCacheCapacity(resource_cache_capacity); @@ -325,13 +325,13 @@ Config::ResetDefaultConfig() { return s; } +#ifdef MILVUS_GPU_VERSION + /* gpu resource config */ s = SetEngineConfigGpuSearchThreshold(CONFIG_ENGINE_GPU_SEARCH_THRESHOLD_DEFAULT); if (!s.ok()) { return s; } - /* gpu resource config */ -#ifdef MILVUS_GPU_VERSION s = SetGpuResourceConfigEnable(CONFIG_GPU_RESOURCE_ENABLE_DEFAULT); if (!s.ok()) { return s; @@ -632,6 +632,7 @@ Config::CheckEngineConfigOmpThreadNum(const std::string& value) { return Status::OK(); } +#ifdef MILVUS_GPU_VERSION Status Config::CheckEngineConfigGpuSearchThreshold(const std::string& value) { if (!ValidationUtil::ValidateStringIsNumber(value).ok()) { @@ -761,6 +762,7 @@ Config::CheckGpuResourceConfigBuildIndexResources(const std::vector return Status::OK(); } +#endif //////////////////////////////////////////////////////////////////////////////// ConfigNode& @@ -981,6 +983,7 @@ Config::GetEngineConfigOmpThreadNum(int64_t& value) { return Status::OK(); } +#ifdef MILVUS_GPU_VERSION Status Config::GetEngineConfigGpuSearchThreshold(int64_t& value) { std::string str = @@ -1097,6 +1100,7 @@ Config::GetGpuResourceConfigBuildIndexResources(std::vector& value) { } return Status::OK(); } +#endif /////////////////////////////////////////////////////////////////////////////// /* server config */ @@ -1284,6 +1288,8 @@ Config::SetEngineConfigOmpThreadNum(const std::string& value) { return Status::OK(); } +#ifdef MILVUS_GPU_VERSION +/* gpu resource config */ Status Config::SetEngineConfigGpuSearchThreshold(const std::string& value) { Status s = CheckEngineConfigGpuSearchThreshold(value); @@ -1294,7 +1300,6 @@ Config::SetEngineConfigGpuSearchThreshold(const std::string& value) { return Status::OK(); } -/* gpu resource config */ Status Config::SetGpuResourceConfigEnable(const std::string& value) { Status s = CheckGpuResourceConfigEnable(value); @@ -1348,6 +1353,7 @@ Config::SetGpuResourceConfigBuildIndexResources(const std::string& value) { SetConfigValueInMem(CONFIG_GPU_RESOURCE, CONFIG_GPU_RESOURCE_BUILD_INDEX_RESOURCES, value); return Status::OK(); } // namespace server +#endif } // namespace server } // namespace milvus diff --git a/core/src/server/Config.h b/core/src/server/Config.h index 0907080a6f..281a832d57 100644 --- a/core/src/server/Config.h +++ b/core/src/server/Config.h @@ -170,6 +170,8 @@ class Config { CheckEngineConfigUseBlasThreshold(const std::string& value); Status CheckEngineConfigOmpThreadNum(const std::string& value); + +#ifdef MILVUS_GPU_VERSION Status CheckEngineConfigGpuSearchThreshold(const std::string& value); @@ -184,6 +186,7 @@ class Config { CheckGpuResourceConfigSearchResources(const std::vector& value); Status CheckGpuResourceConfigBuildIndexResources(const std::vector& value); +#endif std::string GetConfigStr(const std::string& parent_key, const std::string& child_key, const std::string& default_value = ""); @@ -239,6 +242,8 @@ class Config { GetEngineConfigUseBlasThreshold(int64_t& value); Status GetEngineConfigOmpThreadNum(int64_t& value); + +#ifdef MILVUS_GPU_VERSION Status GetEngineConfigGpuSearchThreshold(int64_t& value); @@ -253,6 +258,7 @@ class Config { GetGpuResourceConfigSearchResources(std::vector& value); Status GetGpuResourceConfigBuildIndexResources(std::vector& value); +#endif public: /* server config */ @@ -300,6 +306,8 @@ class Config { SetEngineConfigUseBlasThreshold(const std::string& value); Status SetEngineConfigOmpThreadNum(const std::string& value); + +#ifdef MILVUS_GPU_VERSION Status SetEngineConfigGpuSearchThreshold(const std::string& value); @@ -314,6 +322,7 @@ class Config { SetGpuResourceConfigSearchResources(const std::string& value); Status SetGpuResourceConfigBuildIndexResources(const std::string& value); +#endif private: std::unordered_map> config_map_; diff --git a/core/src/server/Server.cpp b/core/src/server/Server.cpp index 5676504722..169463080e 100644 --- a/core/src/server/Server.cpp +++ b/core/src/server/Server.cpp @@ -183,7 +183,11 @@ Server::Start() { // print version information SERVER_LOG_INFO << "Milvus " << BUILD_TYPE << " version: v" << MILVUS_VERSION << ", built at " << BUILD_TIME; - +#ifdef MILVUS_CPU_VERSION + SERVER_LOG_INFO << "CPU edition"; +#else + SERVER_LOG_INFO << "GPU edition"; +#endif server::Metrics::GetInstance().Init(); server::SystemInfo::GetInstance().Init(); diff --git a/core/src/utils/StringHelpFunctions.h b/core/src/utils/StringHelpFunctions.h index 3a41e53f4b..51812fae40 100644 --- a/core/src/utils/StringHelpFunctions.h +++ b/core/src/utils/StringHelpFunctions.h @@ -30,9 +30,13 @@ class StringHelpFunctions { StringHelpFunctions() = default; public: + // trim blanks from begin and end + // " a b c " => "a b c" static void TrimStringBlank(std::string& string); + // trim quotes from begin and end + // "'abc'" => "abc" static void TrimStringQuote(std::string& string, const std::string& qoute); @@ -46,6 +50,8 @@ class StringHelpFunctions { static void SplitStringByDelimeter(const std::string& str, const std::string& delimeter, std::vector& result); + // merge strings with delimeter + // "a", "b", "c" => "a,b,c" static void MergeStringWithDelimeter(const std::vector& strs, const std::string& delimeter, std::string& result); diff --git a/core/src/utils/ValidationUtil.cpp b/core/src/utils/ValidationUtil.cpp index 12b2372fc5..2d1a0e257e 100644 --- a/core/src/utils/ValidationUtil.cpp +++ b/core/src/utils/ValidationUtil.cpp @@ -218,10 +218,9 @@ ValidationUtil::ValidateGpuIndex(int32_t gpu_index) { return Status::OK(); } +#ifdef MILVUS_GPU_VERSION Status ValidationUtil::GetGpuMemory(int32_t gpu_index, size_t& memory) { -#ifdef MILVUS_GPU_VERSION - cudaDeviceProp deviceProp; auto cuda_err = cudaGetDeviceProperties(&deviceProp, gpu_index); if (cuda_err) { @@ -232,10 +231,9 @@ ValidationUtil::GetGpuMemory(int32_t gpu_index, size_t& memory) { } memory = deviceProp.totalGlobalMem; -#endif - return Status::OK(); } +#endif Status ValidationUtil::ValidateIpAddress(const std::string& ip_address) { diff --git a/core/src/utils/ValidationUtil.h b/core/src/utils/ValidationUtil.h index ab32c35c40..bc523654e5 100644 --- a/core/src/utils/ValidationUtil.h +++ b/core/src/utils/ValidationUtil.h @@ -64,8 +64,10 @@ class ValidationUtil { static Status ValidateGpuIndex(int32_t gpu_index); +#ifdef MILVUS_GPU_VERSION static Status GetGpuMemory(int32_t gpu_index, size_t& memory); +#endif static Status ValidateIpAddress(const std::string& ip_address); diff --git a/core/src/wrapper/ConfAdapter.cpp b/core/src/wrapper/ConfAdapter.cpp index aa4b3c12b8..d49747d8f4 100644 --- a/core/src/wrapper/ConfAdapter.cpp +++ b/core/src/wrapper/ConfAdapter.cpp @@ -22,6 +22,7 @@ #include #include +#include // TODO(lxj): add conf checker @@ -129,17 +130,35 @@ IVFPQConfAdapter::Match(const TempMetaConf& metaconf) { conf->metric_type = metaconf.metric_type; conf->gpu_id = metaconf.gpu_id; conf->nbits = 8; - - if (!(conf->d % 4)) - conf->m = conf->d / 4; // compression radio = 16 - else if (!(conf->d % 2)) - conf->m = conf->d / 2; // compression radio = 8 - else if (!(conf->d % 3)) - conf->m = conf->d / 3; // compression radio = 12 - else - conf->m = conf->d; // same as SQ8, compression radio = 4 - MatchBase(conf); + + /* + * Faiss 1.6 + * Only 1, 2, 3, 4, 6, 8, 10, 12, 16, 20, 24, 28, 32 dims per sub-quantizer are currently supporte with + * no precomputed codes. Precomputed codes supports any number of dimensions, but will involve memory overheads. + */ + static std::vector support_dim_per_subquantizer{32, 28, 24, 20, 16, 12, 10, 8, 6, 4, 3, 2, 1}; + static std::vector support_subquantizer{96, 64, 56, 48, 40, 32, 28, 24, 20, 16, 12, 8, 4, 3, 2, 1}; + std::vector resset; + for (const auto& dimperquantizer : support_dim_per_subquantizer) { + if (!(conf->d % dimperquantizer)) { + auto subquantzier_num = conf->d / dimperquantizer; + auto finder = std::find(support_subquantizer.begin(), support_subquantizer.end(), subquantzier_num); + if (finder != support_subquantizer.end()) { + resset.push_back(subquantzier_num); + } + } + } + + if (resset.empty()) { + // todo(linxj): throw exception here. + return nullptr; + } + static int64_t compression_level = 1; // 1:low, 2:high + if (compression_level == 1) { + conf->m = resset[int(resset.size() / 2)]; + WRAPPER_LOG_DEBUG << "PQ m = " << conf->m << ", compression radio = " << conf->d / conf->m * 4; + } return conf; } diff --git a/core/unittest/db/utils.cpp b/core/unittest/db/utils.cpp index afa1d39006..293eeccc69 100644 --- a/core/unittest/db/utils.cpp +++ b/core/unittest/db/utils.cpp @@ -132,8 +132,8 @@ BaseTest::SetUp() { void BaseTest::TearDown() { milvus::cache::CpuCacheMgr::GetInstance()->ClearCache(); - milvus::cache::GpuCacheMgr::GetInstance(0)->ClearCache(); #ifdef MILVUS_GPU_VERSION + milvus::cache::GpuCacheMgr::GetInstance(0)->ClearCache(); knowhere::FaissGpuResourceMgr::GetInstance().Free(); #endif } diff --git a/core/unittest/scheduler/test_scheduler.cpp b/core/unittest/scheduler/test_scheduler.cpp index 72538113c3..c839307958 100644 --- a/core/unittest/scheduler/test_scheduler.cpp +++ b/core/unittest/scheduler/test_scheduler.cpp @@ -98,24 +98,25 @@ class SchedulerTest : public testing::Test { protected: void SetUp() override { + res_mgr_ = std::make_shared(); + ResourcePtr disk = ResourceFactory::Create("disk", "DISK", 0, true, false); + ResourcePtr cpu = ResourceFactory::Create("cpu", "CPU", 0, true, false); + disk_resource_ = res_mgr_->Add(std::move(disk)); + cpu_resource_ = res_mgr_->Add(std::move(cpu)); + +#ifdef MILVUS_GPU_VERSION constexpr int64_t cache_cap = 1024 * 1024 * 1024; cache::GpuCacheMgr::GetInstance(0)->SetCapacity(cache_cap); cache::GpuCacheMgr::GetInstance(1)->SetCapacity(cache_cap); - - ResourcePtr disk = ResourceFactory::Create("disk", "DISK", 0, true, false); - ResourcePtr cpu = ResourceFactory::Create("cpu", "CPU", 0, true, false); ResourcePtr gpu_0 = ResourceFactory::Create("gpu0", "GPU", 0); ResourcePtr gpu_1 = ResourceFactory::Create("gpu1", "GPU", 1); - - res_mgr_ = std::make_shared(); - disk_resource_ = res_mgr_->Add(std::move(disk)); - cpu_resource_ = res_mgr_->Add(std::move(cpu)); gpu_resource_0_ = res_mgr_->Add(std::move(gpu_0)); gpu_resource_1_ = res_mgr_->Add(std::move(gpu_1)); auto PCIE = Connection("IO", 11000.0); res_mgr_->Connect("cpu", "gpu0", PCIE); res_mgr_->Connect("cpu", "gpu1", PCIE); +#endif scheduler_ = std::make_shared(res_mgr_); @@ -138,17 +139,6 @@ class SchedulerTest : public testing::Test { std::shared_ptr scheduler_; }; -void -insert_dummy_index_into_gpu_cache(uint64_t device_id) { - MockVecIndex* mock_index = new MockVecIndex(); - mock_index->ntotal_ = 1000; - engine::VecIndexPtr index(mock_index); - - cache::DataObjPtr obj = std::static_pointer_cast(index); - - cache::GpuCacheMgr::GetInstance(device_id)->InsertItem("location", obj); -} - class SchedulerTest2 : public testing::Test { protected: void @@ -157,16 +147,13 @@ class SchedulerTest2 : public testing::Test { ResourcePtr cpu0 = ResourceFactory::Create("cpu0", "CPU", 0, true, false); ResourcePtr cpu1 = ResourceFactory::Create("cpu1", "CPU", 1, true, false); ResourcePtr cpu2 = ResourceFactory::Create("cpu2", "CPU", 2, true, false); - ResourcePtr gpu0 = ResourceFactory::Create("gpu0", "GPU", 0, true, true); - ResourcePtr gpu1 = ResourceFactory::Create("gpu1", "GPU", 1, true, true); res_mgr_ = std::make_shared(); disk_ = res_mgr_->Add(std::move(disk)); cpu_0_ = res_mgr_->Add(std::move(cpu0)); cpu_1_ = res_mgr_->Add(std::move(cpu1)); cpu_2_ = res_mgr_->Add(std::move(cpu2)); - gpu_0_ = res_mgr_->Add(std::move(gpu0)); - gpu_1_ = res_mgr_->Add(std::move(gpu1)); + auto IO = Connection("IO", 5.0); auto PCIE1 = Connection("PCIE", 11.0); auto PCIE2 = Connection("PCIE", 20.0); @@ -174,8 +161,15 @@ class SchedulerTest2 : public testing::Test { res_mgr_->Connect("cpu0", "cpu1", IO); res_mgr_->Connect("cpu1", "cpu2", IO); res_mgr_->Connect("cpu0", "cpu2", IO); + +#ifdef MILVUS_GPU_VERSION + ResourcePtr gpu0 = ResourceFactory::Create("gpu0", "GPU", 0, true, true); + ResourcePtr gpu1 = ResourceFactory::Create("gpu1", "GPU", 1, true, true); + gpu_0_ = res_mgr_->Add(std::move(gpu0)); + gpu_1_ = res_mgr_->Add(std::move(gpu1)); res_mgr_->Connect("cpu1", "gpu0", PCIE1); res_mgr_->Connect("cpu2", "gpu1", PCIE2); +#endif scheduler_ = std::make_shared(res_mgr_); diff --git a/core/unittest/server/test_cache.cpp b/core/unittest/server/test_cache.cpp index 67e9664d2c..92e09d4a26 100644 --- a/core/unittest/server/test_cache.cpp +++ b/core/unittest/server/test_cache.cpp @@ -175,6 +175,7 @@ TEST(CacheTest, CPU_CACHE_TEST) { cpu_mgr->PrintInfo(); } +#ifdef MILVUS_GPU_VERSION TEST(CacheTest, GPU_CACHE_TEST) { auto gpu_mgr = milvus::cache::GpuCacheMgr::GetInstance(0); @@ -202,6 +203,7 @@ TEST(CacheTest, GPU_CACHE_TEST) { gpu_mgr->ClearCache(); ASSERT_EQ(gpu_mgr->ItemCount(), 0); } +#endif TEST(CacheTest, INVALID_TEST) { { diff --git a/core/unittest/server/test_config.cpp b/core/unittest/server/test_config.cpp index 664a08d631..791876ee8b 100644 --- a/core/unittest/server/test_config.cpp +++ b/core/unittest/server/test_config.cpp @@ -25,6 +25,8 @@ #include "utils/StringHelpFunctions.h" #include "utils/ValidationUtil.h" +#include + namespace { static constexpr uint64_t KB = 1024; @@ -63,9 +65,21 @@ TEST_F(ConfigTest, CONFIG_TEST) { int64_t port = server_config.GetInt64Value("port"); ASSERT_NE(port, 0); - server_config.SetValue("test", "2.5"); - double test = server_config.GetDoubleValue("test"); - ASSERT_EQ(test, 2.5); + server_config.SetValue("float_test", "2.5"); + double dbl = server_config.GetDoubleValue("float_test"); + ASSERT_LE(abs(dbl - 2.5), std::numeric_limits::epsilon()); + float flt = server_config.GetFloatValue("float_test"); + ASSERT_LE(abs(flt - 2.5), std::numeric_limits::epsilon()); + + server_config.SetValue("bool_test", "true"); + bool blt = server_config.GetBoolValue("bool_test"); + ASSERT_TRUE(blt); + + server_config.SetValue("int_test", "34"); + int32_t it32 = server_config.GetInt32Value("int_test"); + ASSERT_EQ(it32, 34); + int64_t it64 = server_config.GetInt64Value("int_test"); + ASSERT_EQ(it64, 34); milvus::server::ConfigNode fake; server_config.AddChild("fake", fake); @@ -236,6 +250,7 @@ TEST_F(ConfigTest, SERVER_CONFIG_VALID_TEST) { ASSERT_TRUE(s.ok()); ASSERT_TRUE(int64_val == engine_omp_thread_num); +#ifdef MILVUS_GPU_VERSION int64_t engine_gpu_search_threshold = 800; s = config.SetEngineConfigGpuSearchThreshold(std::to_string(engine_gpu_search_threshold)); ASSERT_TRUE(s.ok()); @@ -251,7 +266,6 @@ TEST_F(ConfigTest, SERVER_CONFIG_VALID_TEST) { ASSERT_TRUE(s.ok()); ASSERT_TRUE(bool_val == resource_enable_gpu); -#ifdef MILVUS_GPU_VERSION int64_t gpu_cache_capacity = 1; s = config.SetGpuResourceConfigCacheCapacity(std::to_string(gpu_cache_capacity)); ASSERT_TRUE(s.ok()); @@ -389,6 +403,7 @@ TEST_F(ConfigTest, SERVER_CONFIG_INVALID_TEST) { s = config.SetEngineConfigOmpThreadNum("10000"); ASSERT_FALSE(s.ok()); +#ifdef MILVUS_GPU_VERSION s = config.SetEngineConfigGpuSearchThreshold("-1"); ASSERT_FALSE(s.ok()); @@ -396,7 +411,6 @@ TEST_F(ConfigTest, SERVER_CONFIG_INVALID_TEST) { s = config.SetGpuResourceConfigEnable("ok"); ASSERT_FALSE(s.ok()); -#ifdef MILVUS_GPU_VERSION s = config.SetGpuResourceConfigCacheCapacity("a"); ASSERT_FALSE(s.ok()); s = config.SetGpuResourceConfigCacheCapacity("128"); diff --git a/core/unittest/server/test_rpc.cpp b/core/unittest/server/test_rpc.cpp index 5753c68422..e04411fad3 100644 --- a/core/unittest/server/test_rpc.cpp +++ b/core/unittest/server/test_rpc.cpp @@ -313,6 +313,9 @@ TEST_F(RpcHandlerTest, TABLES_TEST) { std::vector> record_array; BuildVectors(0, VECTOR_COUNT, record_array); ::milvus::grpc::VectorIds vector_ids; + for (int64_t i = 0; i < VECTOR_COUNT; i++) { + vector_ids.add_vector_id_array(i); + } // Insert vectors // test invalid table name handler->Insert(&context, &request, &vector_ids); diff --git a/core/unittest/server/test_util.cpp b/core/unittest/server/test_util.cpp index e5884cac65..01cf713bcd 100644 --- a/core/unittest/server/test_util.cpp +++ b/core/unittest/server/test_util.cpp @@ -120,7 +120,13 @@ TEST(UtilTest, STRINGFUNCTIONS_TEST) { milvus::server::StringHelpFunctions::SplitStringByDelimeter(str, ",", result); ASSERT_EQ(result.size(), 3UL); + std::string merge_str; + milvus::server::StringHelpFunctions::MergeStringWithDelimeter(result, ",", merge_str); + ASSERT_EQ(merge_str, "a,b,c"); result.clear(); + milvus::server::StringHelpFunctions::MergeStringWithDelimeter(result, ",", merge_str); + ASSERT_TRUE(merge_str.empty()); + auto status = milvus::server::StringHelpFunctions::SplitStringByQuote(str, ",", "\"", result); ASSERT_TRUE(status.ok()); ASSERT_EQ(result.size(), 3UL); @@ -211,6 +217,11 @@ TEST(UtilTest, STATUS_TEST) { str = status.ToString(); ASSERT_FALSE(str.empty()); + status = milvus::Status(milvus::DB_INVALID_PATH, "mistake"); + ASSERT_EQ(status.code(), milvus::DB_INVALID_PATH); + str = status.ToString(); + ASSERT_FALSE(str.empty()); + status = milvus::Status(milvus::DB_META_TRANSACTION_FAILED, "mistake"); ASSERT_EQ(status.code(), milvus::DB_META_TRANSACTION_FAILED); str = status.ToString(); @@ -261,6 +272,10 @@ TEST(ValidationUtilTest, VALIDATE_TABLENAME_TEST) { table_name = std::string(10000, 'a'); status = milvus::server::ValidationUtil::ValidateTableName(table_name); ASSERT_EQ(status.code(), milvus::SERVER_INVALID_TABLE_NAME); + + table_name = ""; + status = milvus::server::ValidationUtil::ValidatePartitionName(table_name); + ASSERT_EQ(status.code(), milvus::SERVER_INVALID_TABLE_NAME); } TEST(ValidationUtilTest, VALIDATE_DIMENSION_TEST) { diff --git a/docs/test_report/milvus_ivfsq8_test_report_detailed_version_cn.md b/docs/test_report/milvus_ivfsq8_test_report_detailed_version_cn.md index a6e5e75ea4..098f9e69a4 100644 --- a/docs/test_report/milvus_ivfsq8_test_report_detailed_version_cn.md +++ b/docs/test_report/milvus_ivfsq8_test_report_detailed_version_cn.md @@ -16,25 +16,25 @@ ### 软硬件环境 -操作系统: CentOS Linux release 7.6.1810 (Core) +操作系统：CentOS Linux release 7.6.1810 (Core) -CPU: Intel(R) Xeon(R) CPU E5-2678 v3 @ 2.50GHz +CPU：Intel(R) Xeon(R) CPU E5-2678 v3 @ 2.50GHz -GPU0: GeForce GTX 1080 +GPU0：GeForce GTX 1080 -GPU1: GeForce GTX 1080 +GPU1：GeForce GTX 1080 -内存: 503GB +内存：503GB -Docker版本: 18.09 +Docker版本：18.09 -NVIDIA Driver版本: 430.34 +NVIDIA Driver版本：430.34 -Milvus版本: 0.5.3 +Milvus版本：0.5.3 -SDK接口: Python 3.6.8 +SDK接口：Python 3.6.8 -pymilvus版本: 0.2.5 +pymilvus版本：0.2.5 @@ -51,7 +51,7 @@ pymilvus版本: 0.2.5 ### 测试指标 -- Query Elapsed Time: 数据库查询所有向量的时间（以秒计）。影响Query Elapsed Time的变量: +- Query Elapsed Time：数据库查询所有向量的时间（以秒计）。影响Query Elapsed Time的变量： - nq (被查询向量的数量) @@ -59,7 +59,7 @@ pymilvus版本: 0.2.5 > > 被查询向量的数量nq将按照 [1, 5, 10, 200, 400, 600, 800, 1000]的数量分组。 -- Recall: 实际返回的正确结果占总数之比 . 影响Recall的变量: +- Recall：实际返回的正确结果占总数之比。影响Recall的变量： - nq (被查询向量的数量) - topk (单条查询中最相似的K个结果) @@ -76,7 +76,7 @@ pymilvus版本: 0.2.5 ### 测试环境 -数据集: sift1b-1,000,000,000向量, 128维 +数据集：sift1b-1,000,000,000向量，128维 表格属性： @@ -143,7 +143,7 @@ search_resources: cpu, gpu0 | nq=800 | 23.24 | | nq=1000 | 27.41 | -当nq为1000时，在GPU模式下查询一条128维向量需要耗时约27毫秒。 +当nq为1000时，在CPU模式下查询一条128维向量需要耗时约27毫秒。 diff --git a/docs/test_report/milvus_ivfsq8h_test_report_detailed_version_cn.md b/docs/test_report/milvus_ivfsq8h_test_report_detailed_version_cn.md index b50d00f9bd..daac2af545 100644 --- a/docs/test_report/milvus_ivfsq8h_test_report_detailed_version_cn.md +++ b/docs/test_report/milvus_ivfsq8h_test_report_detailed_version_cn.md @@ -139,7 +139,7 @@ topk = 100 **总结** -当nq小于1200时，查询耗时随nq的增长快速增大；当nq大于1200时，查询耗时的增大则缓慢许多。这是因为gpu_search_threshold这一参数的值被设为1200，当nq<1200时，选择CPU进行操作，否则选择GPU进行操作。与CPU。 +当nq小于1200时，查询耗时随nq的增长快速增大；当nq大于1200时，查询耗时的增大则缓慢许多。这是因为gpu_search_threshold这一参数的值被设为1200，当nq小于1200时，选择CPU进行操作，否则选择GPU进行操作。 在GPU模式下的查询耗时由两部分组成：（1）索引从CPU到GPU的拷贝时间；（2）所有分桶的查询时间。当nq小于500时，索引从CPU到GPU 的拷贝时间无法被有效均摊，此时CPU模式时一个更优的选择；当nq大于500时，选择GPU模式更合理。和CPU相比，GPU具有更多的核数和更强的算力。当nq较大时，GPU在计算上的优势能被更好地被体现。 diff --git a/tests/milvus_python_test/test_index.py b/tests/milvus_python_test/test_index.py index 39aadb9d33..b253cf02a3 100644 --- a/tests/milvus_python_test/test_index.py +++ b/tests/milvus_python_test/test_index.py @@ -1208,9 +1208,9 @@ class TestCreateIndexParamsInvalid(object): nlist = index_params["nlist"] logging.getLogger().info(index_params) status, ids = connect.add_vectors(table, vectors) - # if not isinstance(index_type, int) or not isinstance(nlist, int): - with pytest.raises(Exception) as e: + if (not index_type) or (not nlist) or (not isinstance(index_type, IndexType)) or (not isinstance(nlist, int)): + with pytest.raises(Exception) as e: + status = connect.create_index(table, index_params) + else: status = connect.create_index(table, index_params) - # else: - # status = connect.create_index(table, index_params) - # assert not status.OK() + assert not status.OK() diff --git a/tests/milvus_python_test/utils.py b/tests/milvus_python_test/utils.py index 159c8407c6..1686ad7129 100644 --- a/tests/milvus_python_test/utils.py +++ b/tests/milvus_python_test/utils.py @@ -54,12 +54,12 @@ def gen_long_str(num): def gen_invalid_ips(): ips = [ - "255.0.0.0", - "255.255.0.0", - "255.255.255.0", - "255.255.255.255", + # "255.0.0.0", + # "255.255.0.0", + # "255.255.255.0", + # "255.255.255.255", "127.0.0", - "123.0.0.2", + # "123.0.0.2", "12-s", " ", "12 s", @@ -114,12 +114,12 @@ def gen_invalid_uris(): # invalid ip "tcp:// :%s" % port, - "tcp://123.0.0.1:%s" % port, + # "tcp://123.0.0.1:%s" % port, "tcp://127.0.0:%s" % port, "tcp://255.0.0.0:%s" % port, - "tcp://255.255.0.0:%s" % port, - "tcp://255.255.255.0:%s" % port, - "tcp://255.255.255.255:%s" % port, + # "tcp://255.255.0.0:%s" % port, + # "tcp://255.255.255.0:%s" % port, + # "tcp://255.255.255.255:%s" % port, "tcp://\n:%s" % port, ] @@ -263,9 +263,7 @@ def gen_invalid_index_types(): def gen_invalid_nlists(): nlists = [ - 0, -1, - 1000000000000001, # None, [1,2,3], (1,2), @@ -549,4 +547,4 @@ if __name__ == "__main__": p.join() time.sleep(3) status, count = milvus.get_table_row_count(table) - assert count == process_num * loop_num \ No newline at end of file + assert count == process_num * loop_num