From fbffc78b9e9a36432011bb7b69bf5be221840c24 Mon Sep 17 00:00:00 2001 From: JinHai-CN Date: Fri, 18 Oct 2019 17:06:48 +0800 Subject: [PATCH 01/31] Update FAISS package to 0.3.0 Former-commit-id: d8baa122f0a9e2e02709d50bc7d4a5105cab7f2f --- core/src/index/cmake/ThirdPartyPackagesCore.cmake | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/core/src/index/cmake/ThirdPartyPackagesCore.cmake b/core/src/index/cmake/ThirdPartyPackagesCore.cmake index ee1d88ee32..2dd619f7bc 100644 --- a/core/src/index/cmake/ThirdPartyPackagesCore.cmake +++ b/core/src/index/cmake/ThirdPartyPackagesCore.cmake @@ -243,7 +243,8 @@ if(CUSTOMIZATION) # set(FAISS_MD5 "57da9c4f599cc8fa4260488b1c96e1cc") # commit-id 6dbdf75987c34a2c853bd172ea0d384feea8358c branch-0.2.0 # set(FAISS_MD5 "21deb1c708490ca40ecb899122c01403") # commit-id 643e48f479637fd947e7b93fa4ca72b38ecc9a39 branch-0.2.0 # set(FAISS_MD5 "072db398351cca6e88f52d743bbb9fa0") # commit-id 3a2344d04744166af41ef1a74449d68a315bfe17 branch-0.2.1 - set(FAISS_MD5 "c89ea8e655f5cdf58f42486f13614714") # commit-id 9c28a1cbb88f41fa03b03d7204106201ad33276b branch-0.2.1 + # set(FAISS_MD5 "c89ea8e655f5cdf58f42486f13614714") # commit-id 9c28a1cbb88f41fa03b03d7204106201ad33276b branch-0.2.1 + set(FAISS_MD5 "f3b2ce3364c3fa7febd3aa7fdd0fe380") # commit-id 694e03458e6b69ce8a62502f71f69a614af5af8f branch-0.3.0 endif() else() set(FAISS_SOURCE_URL "https://github.com/facebookresearch/faiss/archive/v1.5.3.tar.gz") From 6572e6bf25cf3908b4c27c5812dbd15290380c4b Mon Sep 17 00:00:00 2001 From: "xiaojun.lin" Date: Sat, 19 Oct 2019 17:49:28 +0800 Subject: [PATCH 02/31] MS-671 Fix HybridIndex Crash MS-670 Adapt to faiss-1.6 Former-commit-id: 12300b390b6d300dc460f0729a368a240c972a80 --- core/src/db/engine/ExecutionEngineImpl.cpp | 53 +++++++- .../index/vector_index/FaissBaseIndex.h | 2 +- .../index/vector_index/IndexGPUIVF.cpp | 11 +- .../index/vector_index/IndexGPUIVFPQ.cpp | 4 +- .../index/vector_index/IndexGPUIVFSQ.cpp | 14 +- .../index/vector_index/IndexGPUIVFSQ.h | 4 - .../index/vector_index/IndexIDMAP.cpp | 5 +- .../knowhere/index/vector_index/IndexIVF.cpp | 6 +- .../knowhere/index/vector_index/IndexIVF.h | 2 +- .../index/vector_index/IndexIVFSQ.cpp | 10 +- .../index/vector_index/IndexIVFSQHybrid.cpp | 97 ++++++++++---- .../index/vector_index/IndexIVFSQHybrid.h | 7 + .../knowhere/index/vector_index/VectorIndex.h | 1 + .../index/vector_index/helpers/FaissIO.h | 2 +- core/src/index/unittest/Helper.h | 2 +- .../index/unittest/test_customized_index.cpp | 120 +++++++++++++++++- core/src/index/unittest/test_ivf.cpp | 41 ++---- core/src/scheduler/SchedInst.h | 1 + .../src/scheduler/optimizer/LargeSQ8HPass.cpp | 84 ++++++------ core/src/scheduler/optimizer/LargeSQ8HPass.h | 4 +- core/src/server/DBWrapper.cpp | 2 +- core/src/wrapper/VecIndex.h | 1 + core/unittest/wrapper/test_wrapper.cpp | 2 +- 23 files changed, 325 insertions(+), 150 deletions(-) diff --git a/core/src/db/engine/ExecutionEngineImpl.cpp b/core/src/db/engine/ExecutionEngineImpl.cpp index 862c1026d2..9c411deba1 100644 --- a/core/src/db/engine/ExecutionEngineImpl.cpp +++ b/core/src/db/engine/ExecutionEngineImpl.cpp @@ -22,10 +22,7 @@ #include "utils/CommonUtil.h" #include "utils/Exception.h" #include "utils/Log.h" - #include "knowhere/common/Config.h" -#include "knowhere/common/Exception.h" -#include "knowhere/index/vector_index/IndexIVFSQHybrid.h" #include "scheduler/Utils.h" #include "server/Config.h" #include "wrapper/ConfAdapter.h" @@ -249,6 +246,56 @@ ExecutionEngineImpl::Load(bool to_cache) { Status ExecutionEngineImpl::CopyToGpu(uint64_t device_id, bool hybrid) { if (hybrid) { +#if 1 + const std::string key = location_ + ".quantizer"; + std::vector gpus = scheduler::get_gpu_pool(); + + const int64_t NOT_FOUND = -1; + int64_t device_id = NOT_FOUND; + + // cache hit + { + knowhere::QuantizerPtr quantizer = nullptr; + + for (auto& gpu : gpus) { + auto cache = cache::GpuCacheMgr::GetInstance(gpu); + if (auto cached_quantizer = cache->GetIndex(key)) { + device_id = gpu; + quantizer = std::static_pointer_cast(cached_quantizer)->Data(); + } + } + + if (device_id != NOT_FOUND) { + // cache hit + auto config = std::make_shared(); + config->gpu_id = device_id; + config->mode = 2; + auto new_index = index_->LoadData(quantizer, config); + index_ = new_index; + } + } + + if (device_id == NOT_FOUND) { + // cache miss + std::vector all_free_mem; + for (auto& gpu : gpus) { + auto cache = cache::GpuCacheMgr::GetInstance(gpu); + auto free_mem = cache->CacheCapacity() - cache->CacheUsage(); + all_free_mem.push_back(free_mem); + } + + auto max_e = std::max_element(all_free_mem.begin(), all_free_mem.end()); + auto best_index = std::distance(all_free_mem.begin(), max_e); + device_id = gpus[best_index]; + + auto pair = index_->CopyToGpuWithQuantizer(device_id); + index_ = pair.first; + + // cache + auto cached_quantizer = std::make_shared(pair.second); + cache::GpuCacheMgr::GetInstance(device_id)->InsertItem(key, cached_quantizer); + } +#endif return Status::OK(); } diff --git a/core/src/index/knowhere/knowhere/index/vector_index/FaissBaseIndex.h b/core/src/index/knowhere/knowhere/index/vector_index/FaissBaseIndex.h index f3fceebb88..359af97d90 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/FaissBaseIndex.h +++ b/core/src/index/knowhere/knowhere/index/vector_index/FaissBaseIndex.h @@ -38,7 +38,7 @@ class FaissBaseIndex { virtual void SealImpl(); - protected: + public: std::shared_ptr index_ = nullptr; }; diff --git a/core/src/index/knowhere/knowhere/index/vector_index/IndexGPUIVF.cpp b/core/src/index/knowhere/knowhere/index/vector_index/IndexGPUIVF.cpp index a5e8f90f34..65938e1630 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/IndexGPUIVF.cpp +++ b/core/src/index/knowhere/knowhere/index/vector_index/IndexGPUIVF.cpp @@ -15,12 +15,12 @@ // specific language governing permissions and limitations // under the License. -#include -#include +#include + #include #include #include -#include +#include #include "knowhere/adapter/VectorAdapter.h" #include "knowhere/common/Exception.h" @@ -130,13 +130,12 @@ void GPUIVF::search_impl(int64_t n, const float* data, int64_t k, float* distances, int64_t* labels, const Config& cfg) { std::lock_guard lk(mutex_); - // TODO(linxj): gpu index support GenParams if (auto device_index = std::dynamic_pointer_cast(index_)) { auto search_cfg = std::dynamic_pointer_cast(cfg); - device_index->setNumProbes(search_cfg->nprobe); + device_index->nprobe = search_cfg->nprobe; +// assert(device_index->getNumProbes() == search_cfg->nprobe); { - // TODO(linxj): allocate gpu mem ResScope rs(res_, gpu_id_); device_index->search(n, (float*)data, k, distances, labels); } diff --git a/core/src/index/knowhere/knowhere/index/vector_index/IndexGPUIVFPQ.cpp b/core/src/index/knowhere/knowhere/index/vector_index/IndexGPUIVFPQ.cpp index 213141b3ac..9ba8dd0456 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/IndexGPUIVFPQ.cpp +++ b/core/src/index/knowhere/knowhere/index/vector_index/IndexGPUIVFPQ.cpp @@ -16,8 +16,10 @@ // under the License. #include -#include #include +#include +#include + #include #include "knowhere/adapter/VectorAdapter.h" diff --git a/core/src/index/knowhere/knowhere/index/vector_index/IndexGPUIVFSQ.cpp b/core/src/index/knowhere/knowhere/index/vector_index/IndexGPUIVFSQ.cpp index 5e1f5226f2..fff27cd7db 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/IndexGPUIVFSQ.cpp +++ b/core/src/index/knowhere/knowhere/index/vector_index/IndexGPUIVFSQ.cpp @@ -15,9 +15,10 @@ // specific language governing permissions and limitations // under the License. -#include +#include +#include + #include -#include #include "knowhere/adapter/VectorAdapter.h" #include "knowhere/common/Exception.h" @@ -71,13 +72,4 @@ GPUIVFSQ::CopyGpuToCpu(const Config& config) { return std::make_shared(new_index); } -void -GPUIVFSQ::search_impl(int64_t n, const float* data, int64_t k, float* distances, int64_t* labels, const Config& cfg) { -#ifdef CUSTOMIZATION - GPUIVF::search_impl(n, data, k, distances, labels, cfg); -#else - IVF::search_impl(n, data, k, distances, labels, cfg); -#endif -} - } // namespace knowhere diff --git a/core/src/index/knowhere/knowhere/index/vector_index/IndexGPUIVFSQ.h b/core/src/index/knowhere/knowhere/index/vector_index/IndexGPUIVFSQ.h index 7332bce691..ed8013d77f 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/IndexGPUIVFSQ.h +++ b/core/src/index/knowhere/knowhere/index/vector_index/IndexGPUIVFSQ.h @@ -38,10 +38,6 @@ class GPUIVFSQ : public GPUIVF { VectorIndexPtr CopyGpuToCpu(const Config& config) override; - - protected: - void - search_impl(int64_t n, const float* data, int64_t k, float* distances, int64_t* labels, const Config& cfg) override; }; } // namespace knowhere diff --git a/core/src/index/knowhere/knowhere/index/vector_index/IndexIDMAP.cpp b/core/src/index/knowhere/knowhere/index/vector_index/IndexIDMAP.cpp index 2371591b5c..643bb16076 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/IndexIDMAP.cpp +++ b/core/src/index/knowhere/knowhere/index/vector_index/IndexIDMAP.cpp @@ -15,11 +15,12 @@ // specific language governing permissions and limitations // under the License. -#include #include #include -#include #include +#include +#include + #include #include "knowhere/adapter/VectorAdapter.h" diff --git a/core/src/index/knowhere/knowhere/index/vector_index/IndexIVF.cpp b/core/src/index/knowhere/knowhere/index/vector_index/IndexIVF.cpp index 0c4856f2b6..02708ff5d7 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/IndexIVF.cpp +++ b/core/src/index/knowhere/knowhere/index/vector_index/IndexIVF.cpp @@ -15,15 +15,11 @@ // specific language governing permissions and limitations // under the License. -#include -#include +#include #include #include #include #include -#include -#include -#include #include #include #include diff --git a/core/src/index/knowhere/knowhere/index/vector_index/IndexIVF.h b/core/src/index/knowhere/knowhere/index/vector_index/IndexIVF.h index ef9982fa30..e064b6f08c 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/IndexIVF.h +++ b/core/src/index/knowhere/knowhere/index/vector_index/IndexIVF.h @@ -30,7 +30,7 @@ namespace knowhere { using Graph = std::vector>; -class IVF : public VectorIndex, protected FaissBaseIndex { +class IVF : public VectorIndex, public FaissBaseIndex { public: IVF() : FaissBaseIndex(nullptr) { } diff --git a/core/src/index/knowhere/knowhere/index/vector_index/IndexIVFSQ.cpp b/core/src/index/knowhere/knowhere/index/vector_index/IndexIVFSQ.cpp index 063dc63550..80b4c78883 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/IndexIVFSQ.cpp +++ b/core/src/index/knowhere/knowhere/index/vector_index/IndexIVFSQ.cpp @@ -15,7 +15,8 @@ // specific language governing permissions and limitations // under the License. -#include +#include +#include #include #include "knowhere/adapter/VectorAdapter.h" @@ -56,14 +57,7 @@ IVFSQ::CopyCpuToGpu(const int64_t& device_id, const Config& config) { if (auto res = FaissGpuResourceMgr::GetInstance().GetRes(device_id)) { ResScope rs(res, device_id, false); -#ifdef CUSTOMIZATION - faiss::gpu::GpuClonerOptions option; - option.allInGpu = true; - - auto gpu_index = faiss::gpu::index_cpu_to_gpu(res->faiss_res.get(), device_id, index_.get(), &option); -#else auto gpu_index = faiss::gpu::index_cpu_to_gpu(res->faiss_res.get(), device_id, index_.get()); -#endif std::shared_ptr device_index; device_index.reset(gpu_index); diff --git a/core/src/index/knowhere/knowhere/index/vector_index/IndexIVFSQHybrid.cpp b/core/src/index/knowhere/knowhere/index/vector_index/IndexIVFSQHybrid.cpp index fe5bf0990a..af67722266 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/IndexIVFSQHybrid.cpp +++ b/core/src/index/knowhere/knowhere/index/vector_index/IndexIVFSQHybrid.cpp @@ -17,19 +17,25 @@ // under the License. #include "knowhere/index/vector_index/IndexIVFSQHybrid.h" -#include -#include "faiss/AutoTune.h" -#include "faiss/gpu/GpuAutoTune.h" -#include "faiss/gpu/GpuIndexIVF.h" #include "knowhere/adapter/VectorAdapter.h" #include "knowhere/common/Exception.h" +#include + +#include +#include +#include + namespace knowhere { #ifdef CUSTOMIZATION +//std::mutex g_mutex; + IndexModelPtr IVFSQHybrid::Train(const DatasetPtr& dataset, const Config& config) { +// std::lock_guard lk(g_mutex); + auto build_cfg = std::dynamic_pointer_cast(config); if (build_cfg != nullptr) { build_cfg->CheckValid(); // throw exception @@ -63,23 +69,25 @@ IVFSQHybrid::Train(const DatasetPtr& dataset, const Config& config) { VectorIndexPtr IVFSQHybrid::CopyGpuToCpu(const Config& config) { + if (gpu_mode == 0) { + return std::make_shared(index_); + } std::lock_guard lk(mutex_); - if (auto device_idx = std::dynamic_pointer_cast(index_)) { faiss::Index* device_index = index_.get(); faiss::Index* host_index = faiss::gpu::index_gpu_to_cpu(device_index); std::shared_ptr new_index; new_index.reset(host_index); return std::make_shared(new_index); - } else { - // TODO(linxj): why? jinhai - return std::make_shared(index_); - } } VectorIndexPtr IVFSQHybrid::CopyCpuToGpu(const int64_t& device_id, const Config& config) { + if (gpu_mode != 0) { + KNOWHERE_THROW_MSG("Not a GpuIndex Type"); + } + if (auto res = FaissGpuResourceMgr::GetInstance().GetRes(device_id)) { ResScope rs(res, device_id, false); faiss::gpu::GpuClonerOptions option; @@ -105,16 +113,26 @@ IVFSQHybrid::LoadImpl(const BinarySet& index_binary) { FaissBaseIndex::LoadImpl(index_binary); // load on cpu auto* ivf_index = dynamic_cast(index_.get()); ivf_index->backup_quantizer(); + gpu_mode = 0; } void IVFSQHybrid::search_impl(int64_t n, const float* data, int64_t k, float* distances, int64_t* labels, const Config& cfg) { +// std::lock_guard lk(g_mutex); +// static int64_t search_count; +// ++search_count; + if (gpu_mode == 2) { GPUIVF::search_impl(n, data, k, distances, labels, cfg); - } else if (gpu_mode == 1) { - ResScope rs(res_, gpu_id_); - IVF::search_impl(n, data, k, distances, labels, cfg); +// index_->search(n, (float*)data, k, distances, labels); + } else if (gpu_mode == 1) { // hybrid + if (auto res = FaissGpuResourceMgr::GetInstance().GetRes(quantizer_gpu_id_)) { + ResScope rs(res, quantizer_gpu_id_, true); + IVF::search_impl(n, data, k, distances, labels, cfg); + } else { + KNOWHERE_THROW_MSG("Hybrid Search Error, can't get gpu: " + std::to_string(quantizer_gpu_id_) + "resource"); + } } else if (gpu_mode == 0) { IVF::search_impl(n, data, k, distances, labels, cfg); } @@ -122,16 +140,18 @@ IVFSQHybrid::search_impl(int64_t n, const float* data, int64_t k, float* distanc QuantizerPtr IVFSQHybrid::LoadQuantizer(const Config& conf) { +// std::lock_guard lk(g_mutex); + auto quantizer_conf = std::dynamic_pointer_cast(conf); if (quantizer_conf != nullptr) { if (quantizer_conf->mode != 1) { KNOWHERE_THROW_MSG("mode only support 1 in this func"); } } - gpu_id_ = quantizer_conf->gpu_id; + auto gpu_id = quantizer_conf->gpu_id; - if (auto res = FaissGpuResourceMgr::GetInstance().GetRes(gpu_id_)) { - ResScope rs(res, gpu_id_, false); + if (auto res = FaissGpuResourceMgr::GetInstance().GetRes(gpu_id)) { + ResScope rs(res, gpu_id, false); faiss::gpu::GpuClonerOptions option; option.allInGpu = true; @@ -148,16 +168,19 @@ IVFSQHybrid::LoadQuantizer(const Config& conf) { auto& q_ptr = index_composition->quantizer; q->size = q_ptr->d * q_ptr->getNumVecs() * sizeof(float); q->quantizer = q_ptr; + q->gpu_id = gpu_id; res_ = res; gpu_mode = 1; return q; } else { - KNOWHERE_THROW_MSG("CopyCpuToGpu Error, can't get gpu: " + std::to_string(gpu_id_) + "resource"); + KNOWHERE_THROW_MSG("CopyCpuToGpu Error, can't get gpu: " + std::to_string(gpu_id) + "resource"); } } void IVFSQHybrid::SetQuantizer(const QuantizerPtr& q) { +// std::lock_guard lk(g_mutex); + auto ivf_quantizer = std::dynamic_pointer_cast(q); if (ivf_quantizer == nullptr) { KNOWHERE_THROW_MSG("Quantizer type error"); @@ -170,20 +193,27 @@ IVFSQHybrid::SetQuantizer(const QuantizerPtr& q) { // delete ivf_index->quantizer; ivf_index->quantizer = ivf_quantizer->quantizer; } + quantizer_gpu_id_ = ivf_quantizer->gpu_id; + gpu_mode = 1; } void IVFSQHybrid::UnsetQuantizer() { +// std::lock_guard lk(g_mutex); + auto* ivf_index = dynamic_cast(index_.get()); if (ivf_index == nullptr) { KNOWHERE_THROW_MSG("Index type error"); } ivf_index->quantizer = nullptr; + quantizer_gpu_id_ = -1; } VectorIndexPtr IVFSQHybrid::LoadData(const knowhere::QuantizerPtr& q, const Config& conf) { +// std::lock_guard lk(g_mutex); + auto quantizer_conf = std::dynamic_pointer_cast(conf); if (quantizer_conf != nullptr) { if (quantizer_conf->mode != 2) { @@ -192,13 +222,11 @@ IVFSQHybrid::LoadData(const knowhere::QuantizerPtr& q, const Config& conf) { } else { KNOWHERE_THROW_MSG("conf error"); } - // if (quantizer_conf->gpu_id != gpu_id_) { - // KNOWHERE_THROW_MSG("quantizer and data must on the same gpu card"); - // } - gpu_id_ = quantizer_conf->gpu_id; - if (auto res = FaissGpuResourceMgr::GetInstance().GetRes(gpu_id_)) { - ResScope rs(res, gpu_id_, false); + auto gpu_id = quantizer_conf->gpu_id; + + if (auto res = FaissGpuResourceMgr::GetInstance().GetRes(gpu_id)) { + ResScope rs(res, gpu_id, false); faiss::gpu::GpuClonerOptions option; option.allInGpu = true; @@ -211,18 +239,20 @@ IVFSQHybrid::LoadData(const knowhere::QuantizerPtr& q, const Config& conf) { index_composition->quantizer = ivf_quantizer->quantizer; index_composition->mode = quantizer_conf->mode; // only 2 - auto gpu_index = faiss::gpu::index_cpu_to_gpu(res->faiss_res.get(), gpu_id_, index_composition, &option); + auto gpu_index = faiss::gpu::index_cpu_to_gpu(res->faiss_res.get(), gpu_id, index_composition, &option); std::shared_ptr new_idx; new_idx.reset(gpu_index); - auto sq_idx = std::make_shared(new_idx, gpu_id_, res); + auto sq_idx = std::make_shared(new_idx, gpu_id, res); return sq_idx; } else { - KNOWHERE_THROW_MSG("CopyCpuToGpu Error, can't get gpu: " + std::to_string(gpu_id_) + "resource"); + KNOWHERE_THROW_MSG("CopyCpuToGpu Error, can't get gpu: " + std::to_string(gpu_id) + "resource"); } } std::pair IVFSQHybrid::CopyCpuToGpuWithQuantizer(const int64_t& device_id, const Config& config) { +// std::lock_guard lk(g_mutex); + if (auto res = FaissGpuResourceMgr::GetInstance().GetRes(device_id)) { ResScope rs(res, device_id, false); faiss::gpu::GpuClonerOptions option; @@ -242,12 +272,29 @@ IVFSQHybrid::CopyCpuToGpuWithQuantizer(const int64_t& device_id, const Config& c auto q = std::make_shared(); q->quantizer = index_composition.quantizer; q->size = index_composition.quantizer->d * index_composition.quantizer->getNumVecs() * sizeof(float); + q->gpu_id = device_id; return std::make_pair(new_idx, q); } else { KNOWHERE_THROW_MSG("CopyCpuToGpu Error, can't get gpu: " + std::to_string(gpu_id_) + "resource"); } } +void +IVFSQHybrid::set_index_model(IndexModelPtr model) { + std::lock_guard lk(mutex_); + + auto host_index = std::static_pointer_cast(model); + if (auto gpures = FaissGpuResourceMgr::GetInstance().GetRes(gpu_id_)) { + ResScope rs(gpures, gpu_id_, false); + auto device_index = faiss::gpu::index_cpu_to_gpu(gpures->faiss_res.get(), gpu_id_, host_index->index_.get()); + index_.reset(device_index); + res_ = gpures; + gpu_mode = 2; + } else { + KNOWHERE_THROW_MSG("load index model error, can't get gpu_resource"); + } +} + FaissIVFQuantizer::~FaissIVFQuantizer() { if (quantizer != nullptr) { delete quantizer; diff --git a/core/src/index/knowhere/knowhere/index/vector_index/IndexIVFSQHybrid.h b/core/src/index/knowhere/knowhere/index/vector_index/IndexIVFSQHybrid.h index f54c61c20f..87cc22931f 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/IndexIVFSQHybrid.h +++ b/core/src/index/knowhere/knowhere/index/vector_index/IndexIVFSQHybrid.h @@ -18,6 +18,8 @@ #pragma once #include +#include + #include #include @@ -29,6 +31,7 @@ namespace knowhere { #ifdef CUSTOMIZATION struct FaissIVFQuantizer : public Quantizer { faiss::gpu::GpuIndexFlat* quantizer = nullptr; + int64_t gpu_id; ~FaissIVFQuantizer() override; }; @@ -52,6 +55,9 @@ class IVFSQHybrid : public GPUIVFSQ { } public: + void + set_index_model(IndexModelPtr model) override; + QuantizerPtr LoadQuantizer(const Config& conf); @@ -85,6 +91,7 @@ class IVFSQHybrid : public GPUIVFSQ { protected: int64_t gpu_mode = 0; // 0,1,2 + int64_t quantizer_gpu_id_ = -1; }; } // namespace knowhere diff --git a/core/src/index/knowhere/knowhere/index/vector_index/VectorIndex.h b/core/src/index/knowhere/knowhere/index/vector_index/VectorIndex.h index 810c4d2ea4..6509458b7b 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/VectorIndex.h +++ b/core/src/index/knowhere/knowhere/index/vector_index/VectorIndex.h @@ -48,6 +48,7 @@ class VectorIndex : public Index { virtual void Seal() = 0; + // TODO(linxj): Deprecated virtual VectorIndexPtr Clone() = 0; diff --git a/core/src/index/knowhere/knowhere/index/vector_index/helpers/FaissIO.h b/core/src/index/knowhere/knowhere/index/vector_index/helpers/FaissIO.h index 7cce5bbbac..a7f8f349e1 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/helpers/FaissIO.h +++ b/core/src/index/knowhere/knowhere/index/vector_index/helpers/FaissIO.h @@ -17,7 +17,7 @@ #pragma once -#include +#include namespace knowhere { diff --git a/core/src/index/unittest/Helper.h b/core/src/index/unittest/Helper.h index d11a484c03..8d4bb0f4ae 100644 --- a/core/src/index/unittest/Helper.h +++ b/core/src/index/unittest/Helper.h @@ -26,7 +26,7 @@ #include "knowhere/index/vector_index/IndexIVFSQ.h" #include "knowhere/index/vector_index/IndexIVFSQHybrid.h" -constexpr int DEVICEID = 0; +int DEVICEID = 0; constexpr int64_t DIM = 128; constexpr int64_t NB = 10000; constexpr int64_t NQ = 10; diff --git a/core/src/index/unittest/test_customized_index.cpp b/core/src/index/unittest/test_customized_index.cpp index 1e0b1d932d..f9b48b8b67 100644 --- a/core/src/index/unittest/test_customized_index.cpp +++ b/core/src/index/unittest/test_customized_index.cpp @@ -16,17 +16,23 @@ // under the License. #include +#include #include "unittest/Helper.h" #include "unittest/utils.h" +#include "knowhere/common/Timer.h" + class SingleIndexTest : public DataGen, public TestGpuIndexBase { protected: void SetUp() override { TestGpuIndexBase::SetUp(); - Generate(DIM, NB, NQ); - k = K; + nb = 1000000; + nq = 1000; + dim = DIM; + Generate(dim, nb, nq); + k = 1000; } void @@ -119,4 +125,114 @@ TEST_F(SingleIndexTest, IVFSQHybrid) { } } +//TEST_F(SingleIndexTest, thread_safe) { +// assert(!xb.empty()); +// +// index_type = "IVFSQHybrid"; +// index_ = IndexFactory(index_type); +// auto base = ParamGenerator::GetInstance().Gen(ParameterType::ivfsq); +// auto conf = std::dynamic_pointer_cast(base); +// conf->nlist = 16384; +// conf->k = k; +// conf->nprobe = 10; +// conf->d = dim; +// auto preprocessor = index_->BuildPreprocessor(base_dataset, conf); +// index_->set_preprocessor(preprocessor); +// +// auto model = index_->Train(base_dataset, conf); +// index_->set_index_model(model); +// index_->Add(base_dataset, conf); +// EXPECT_EQ(index_->Count(), nb); +// EXPECT_EQ(index_->Dimension(), dim); +// +// auto binaryset = index_->Serialize(); +// +// +// +// auto cpu_idx = std::make_shared(DEVICEID); +// cpu_idx->Load(binaryset); +// auto pair = cpu_idx->CopyCpuToGpuWithQuantizer(DEVICEID, conf); +// auto quantizer = pair.second; +// +// auto quantizer_conf = std::make_shared(); +// quantizer_conf->mode = 2; // only copy data +// quantizer_conf->gpu_id = DEVICEID; +// +// auto CopyAllToGpu = [&](int64_t search_count, bool do_search = false) { +// for (int i = 0; i < search_count; ++i) { +// auto gpu_idx = cpu_idx->CopyCpuToGpu(DEVICEID, conf); +// if (do_search) { +// auto result = gpu_idx->Search(query_dataset, conf); +// AssertAnns(result, nq, conf->k); +// } +// } +// }; +// +// auto hybrid_qt_idx = std::make_shared(DEVICEID); +// hybrid_qt_idx->Load(binaryset); +// auto SetQuantizerDoSearch = [&](int64_t search_count) { +// for (int i = 0; i < search_count; ++i) { +// hybrid_qt_idx->SetQuantizer(quantizer); +// auto result = hybrid_qt_idx->Search(query_dataset, conf); +// AssertAnns(result, nq, conf->k); +// // PrintResult(result, nq, k); +// hybrid_qt_idx->UnsetQuantizer(); +// } +// }; +// +// auto hybrid_data_idx = std::make_shared(DEVICEID); +// hybrid_data_idx->Load(binaryset); +// auto LoadDataDoSearch = [&](int64_t search_count, bool do_search = false) { +// for (int i = 0; i < search_count; ++i) { +// auto hybrid_idx = hybrid_data_idx->LoadData(quantizer, quantizer_conf); +// if (do_search) { +// auto result = hybrid_idx->Search(query_dataset, conf); +//// AssertAnns(result, nq, conf->k); +// } +// } +// }; +// +// knowhere::TimeRecorder tc(""); +// CopyAllToGpu(2000/2, false); +// tc.RecordSection("CopyAllToGpu witout search"); +// CopyAllToGpu(400/2, true); +// tc.RecordSection("CopyAllToGpu with search"); +// SetQuantizerDoSearch(6); +// tc.RecordSection("SetQuantizer with search"); +// LoadDataDoSearch(2000/2, false); +// tc.RecordSection("LoadData without search"); +// LoadDataDoSearch(400/2, true); +// tc.RecordSection("LoadData with search"); +// +// { +// std::thread t1(CopyAllToGpu, 2000, false); +// std::thread t2(CopyAllToGpu, 400, true); +// t1.join(); +// t2.join(); +// } +// +// { +// std::thread t1(SetQuantizerDoSearch, 12); +// std::thread t2(CopyAllToGpu, 400, true); +// t1.join(); +// t2.join(); +// } +// +// { +// std::thread t1(SetQuantizerDoSearch, 12); +// std::thread t2(LoadDataDoSearch, 400, true); +// t1.join(); +// t2.join(); +// } +// +// { +// std::thread t1(LoadDataDoSearch, 2000, false); +// std::thread t2(LoadDataDoSearch, 400, true); +// t1.join(); +// t2.join(); +// } +// +//} + + #endif diff --git a/core/src/index/unittest/test_ivf.cpp b/core/src/index/unittest/test_ivf.cpp index fae27b0dd3..3fd3e16d0e 100644 --- a/core/src/index/unittest/test_ivf.cpp +++ b/core/src/index/unittest/test_ivf.cpp @@ -20,19 +20,12 @@ #include #include -#include -#include #include #include "knowhere/common/Exception.h" #include "knowhere/common/Timer.h" #include "knowhere/index/vector_index/IndexGPUIVF.h" -#include "knowhere/index/vector_index/IndexGPUIVFPQ.h" -#include "knowhere/index/vector_index/IndexGPUIVFSQ.h" #include "knowhere/index/vector_index/IndexIVF.h" -#include "knowhere/index/vector_index/IndexIVFPQ.h" -#include "knowhere/index/vector_index/IndexIVFSQ.h" -#include "knowhere/index/vector_index/IndexIVFSQHybrid.h" #include "knowhere/index/vector_index/helpers/Cloner.h" #include "unittest/Helper.h" @@ -51,6 +44,9 @@ class IVFTest : public DataGen, public TestWithParam<::std::tuple gpu_idx{"GPUIVFSQ"}; - auto finder = std::find(gpu_idx.cbegin(), gpu_idx.cend(), index_type); - if (finder != gpu_idx.cend()) { - return knowhere::cloner::CopyCpuToGpu(index_, DEVICEID, knowhere::Config()); - } - return index_; - } - protected: std::string index_type; knowhere::Config conf; @@ -100,8 +86,7 @@ TEST_P(IVFTest, ivf_basic) { EXPECT_EQ(index_->Count(), nb); EXPECT_EQ(index_->Dimension(), dim); - auto new_idx = ChooseTodo(); - auto result = new_idx->Search(query_dataset, conf); + auto result = index_->Search(query_dataset, conf); AssertAnns(result, nq, conf->k); // PrintResult(result, nq, k); } @@ -134,8 +119,7 @@ TEST_P(IVFTest, ivf_serialize) { index_->set_index_model(model); index_->Add(base_dataset, conf); - auto new_idx = ChooseTodo(); - auto result = new_idx->Search(query_dataset, conf); + auto result = index_->Search(query_dataset, conf); AssertAnns(result, nq, conf->k); } @@ -159,8 +143,7 @@ TEST_P(IVFTest, ivf_serialize) { index_->Load(binaryset); EXPECT_EQ(index_->Count(), nb); EXPECT_EQ(index_->Dimension(), dim); - auto new_idx = ChooseTodo(); - auto result = new_idx->Search(query_dataset, conf); + auto result = index_->Search(query_dataset, conf); AssertAnns(result, nq, conf->k); } } @@ -176,8 +159,7 @@ TEST_P(IVFTest, clone_test) { index_->Add(base_dataset, conf); EXPECT_EQ(index_->Count(), nb); EXPECT_EQ(index_->Dimension(), dim); - auto new_idx = ChooseTodo(); - auto result = new_idx->Search(query_dataset, conf); + auto result = index_->Search(query_dataset, conf); AssertAnns(result, nq, conf->k); // PrintResult(result, nq, k); @@ -210,12 +192,6 @@ TEST_P(IVFTest, clone_test) { // } // } - { - if (index_type == "IVFSQHybrid") { - return; - } - } - { // copy from gpu to cpu std::vector support_idx_vec{"GPUIVF", "GPUIVFSQ", "IVFSQHybrid"}; @@ -277,8 +253,7 @@ TEST_P(IVFTest, gpu_seal_test) { index_->Add(base_dataset, conf); EXPECT_EQ(index_->Count(), nb); EXPECT_EQ(index_->Dimension(), dim); - auto new_idx = ChooseTodo(); - auto result = new_idx->Search(query_dataset, conf); + auto result = index_->Search(query_dataset, conf); AssertAnns(result, nq, conf->k); auto cpu_idx = knowhere::cloner::CopyGpuToCpu(index_, knowhere::Config()); diff --git a/core/src/scheduler/SchedInst.h b/core/src/scheduler/SchedInst.h index 60033731ae..b9153d3bc3 100644 --- a/core/src/scheduler/SchedInst.h +++ b/core/src/scheduler/SchedInst.h @@ -94,6 +94,7 @@ class OptimizerInst { std::lock_guard lock(mutex_); if (instance == nullptr) { std::vector pass_list; + pass_list.push_back(std::make_shared()); pass_list.push_back(std::make_shared()); instance = std::make_shared(pass_list); } diff --git a/core/src/scheduler/optimizer/LargeSQ8HPass.cpp b/core/src/scheduler/optimizer/LargeSQ8HPass.cpp index 8368a90000..0d5a81a7b6 100644 --- a/core/src/scheduler/optimizer/LargeSQ8HPass.cpp +++ b/core/src/scheduler/optimizer/LargeSQ8HPass.cpp @@ -26,48 +26,48 @@ namespace milvus { namespace scheduler { -// bool -// LargeSQ8HPass::Run(const TaskPtr& task) { -// if (task->Type() != TaskType::SearchTask) { -// return false; -// } -// -// auto search_task = std::static_pointer_cast(task); -// if (search_task->file_->engine_type_ != (int)engine::EngineType::FAISS_IVFSQ8H) { -// return false; -// } -// -// auto search_job = std::static_pointer_cast(search_task->job_.lock()); -// -// // TODO: future, Index::IVFSQ8H, if nq < threshold set cpu, else set gpu -// if (search_job->nq() < 100) { -// return false; -// } -// -// std::vector gpus = scheduler::get_gpu_pool(); -// std::vector all_free_mem; -// for (auto& gpu : gpus) { -// auto cache = cache::GpuCacheMgr::GetInstance(gpu); -// auto free_mem = cache->CacheCapacity() - cache->CacheUsage(); -// all_free_mem.push_back(free_mem); -// } -// -// auto max_e = std::max_element(all_free_mem.begin(), all_free_mem.end()); -// auto best_index = std::distance(all_free_mem.begin(), max_e); -// auto best_device_id = gpus[best_index]; -// -// ResourcePtr res_ptr = ResMgrInst::GetInstance()->GetResource(ResourceType::GPU, best_device_id); -// if (not res_ptr) { -// SERVER_LOG_ERROR << "GpuResource " << best_device_id << " invalid."; -// // TODO: throw critical error and exit -// return false; -// } -// -// auto label = std::make_shared(std::weak_ptr(res_ptr)); -// task->label() = label; -// -// return true; -// } + bool + LargeSQ8HPass::Run(const TaskPtr& task) { + if (task->Type() != TaskType::SearchTask) { + return false; + } + + auto search_task = std::static_pointer_cast(task); + if (search_task->file_->engine_type_ != (int)engine::EngineType::FAISS_IVFSQ8H) { + return false; + } + + auto search_job = std::static_pointer_cast(search_task->job_.lock()); + + // TODO: future, Index::IVFSQ8H, if nq < threshold set cpu, else set gpu + if (search_job->nq() < 100) { + return false; + } + + std::vector gpus = scheduler::get_gpu_pool(); + std::vector all_free_mem; + for (auto& gpu : gpus) { + auto cache = cache::GpuCacheMgr::GetInstance(gpu); + auto free_mem = cache->CacheCapacity() - cache->CacheUsage(); + all_free_mem.push_back(free_mem); + } + + auto max_e = std::max_element(all_free_mem.begin(), all_free_mem.end()); + auto best_index = std::distance(all_free_mem.begin(), max_e); + auto best_device_id = gpus[best_index]; + + ResourcePtr res_ptr = ResMgrInst::GetInstance()->GetResource(ResourceType::GPU, best_device_id); + if (not res_ptr) { + SERVER_LOG_ERROR << "GpuResource " << best_device_id << " invalid."; + // TODO: throw critical error and exit + return false; + } + + auto label = std::make_shared(std::weak_ptr(res_ptr)); + task->label() = label; + + return true; + } } // namespace scheduler } // namespace milvus diff --git a/core/src/scheduler/optimizer/LargeSQ8HPass.h b/core/src/scheduler/optimizer/LargeSQ8HPass.h index 3335a37cc7..49e658002f 100644 --- a/core/src/scheduler/optimizer/LargeSQ8HPass.h +++ b/core/src/scheduler/optimizer/LargeSQ8HPass.h @@ -37,8 +37,8 @@ class LargeSQ8HPass : public Pass { LargeSQ8HPass() = default; public: - // bool - // Run(const TaskPtr& task) override; + bool + Run(const TaskPtr& task) override; }; using LargeSQ8HPassPtr = std::shared_ptr; diff --git a/core/src/server/DBWrapper.cpp b/core/src/server/DBWrapper.cpp index a5b892ad47..34c8d38faf 100644 --- a/core/src/server/DBWrapper.cpp +++ b/core/src/server/DBWrapper.cpp @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -#include +#include #include #include #include diff --git a/core/src/wrapper/VecIndex.h b/core/src/wrapper/VecIndex.h index 05da9ccc03..1729d583ae 100644 --- a/core/src/wrapper/VecIndex.h +++ b/core/src/wrapper/VecIndex.h @@ -70,6 +70,7 @@ class VecIndex : public cache::DataObj { virtual VecIndexPtr CopyToCpu(const Config& cfg = Config()) = 0; + // TODO(linxj): Deprecated virtual VecIndexPtr Clone() = 0; diff --git a/core/unittest/wrapper/test_wrapper.cpp b/core/unittest/wrapper/test_wrapper.cpp index f112fc7e65..2f8fd6fafe 100644 --- a/core/unittest/wrapper/test_wrapper.cpp +++ b/core/unittest/wrapper/test_wrapper.cpp @@ -74,7 +74,7 @@ INSTANTIATE_TEST_CASE_P(WrapperParam, KnowhereWrapperTest, 10, 10), std::make_tuple(milvus::engine::IndexType::FAISS_IVFSQ8_CPU, "Default", DIM, NB, 10, 10), -// std::make_tuple(milvus::engine::IndexType::FAISS_IVFSQ8_GPU, "Default", DIM, NB, 10, 10), + std::make_tuple(milvus::engine::IndexType::FAISS_IVFSQ8_GPU, "Default", DIM, NB, 10, 10), std::make_tuple(milvus::engine::IndexType::FAISS_IVFSQ8_MIX, "Default", DIM, NB, 10, 10), // std::make_tuple(IndexType::NSG_MIX, "Default", 128, 250000, 10, 10), // std::make_tuple(IndexType::SPTAG_KDT_RNT_CPU, "Default", 128, 250000, 10, 10), From f23e8e2143caac8c5fbb4b1332f0124d159919ac Mon Sep 17 00:00:00 2001 From: "xiaojun.lin" Date: Sat, 19 Oct 2019 17:51:59 +0800 Subject: [PATCH 03/31] clang-format... Former-commit-id: 3bd5e246b34d8ca7800dbc6ce0e784dfa26ce18a --- core/src/db/engine/ExecutionEngineImpl.cpp | 6 ++-- .../index/vector_index/IndexGPUIVF.cpp | 4 +-- .../index/vector_index/IndexGPUIVFPQ.cpp | 2 +- .../index/vector_index/IndexGPUIVFSQ.cpp | 2 +- .../index/vector_index/IndexIDMAP.cpp | 4 +-- .../knowhere/index/vector_index/IndexIVF.cpp | 2 +- .../index/vector_index/IndexIVFSQ.cpp | 2 +- .../index/vector_index/IndexIVFSQHybrid.cpp | 36 +++++++++---------- .../index/vector_index/IndexIVFSQHybrid.h | 2 +- .../index/unittest/test_customized_index.cpp | 3 +- core/src/index/unittest/test_ivf.cpp | 6 ++-- .../src/scheduler/optimizer/LargeSQ8HPass.cpp | 6 ++-- 12 files changed, 37 insertions(+), 38 deletions(-) diff --git a/core/src/db/engine/ExecutionEngineImpl.cpp b/core/src/db/engine/ExecutionEngineImpl.cpp index 9c411deba1..aeb17203ef 100644 --- a/core/src/db/engine/ExecutionEngineImpl.cpp +++ b/core/src/db/engine/ExecutionEngineImpl.cpp @@ -18,13 +18,13 @@ #include "db/engine/ExecutionEngineImpl.h" #include "cache/CpuCacheMgr.h" #include "cache/GpuCacheMgr.h" +#include "knowhere/common/Config.h" #include "metrics/Metrics.h" +#include "scheduler/Utils.h" +#include "server/Config.h" #include "utils/CommonUtil.h" #include "utils/Exception.h" #include "utils/Log.h" -#include "knowhere/common/Config.h" -#include "scheduler/Utils.h" -#include "server/Config.h" #include "wrapper/ConfAdapter.h" #include "wrapper/ConfAdapterMgr.h" #include "wrapper/VecImpl.h" diff --git a/core/src/index/knowhere/knowhere/index/vector_index/IndexGPUIVF.cpp b/core/src/index/knowhere/knowhere/index/vector_index/IndexGPUIVF.cpp index 65938e1630..a26f947181 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/IndexGPUIVF.cpp +++ b/core/src/index/knowhere/knowhere/index/vector_index/IndexGPUIVF.cpp @@ -17,10 +17,10 @@ #include +#include #include #include #include -#include #include "knowhere/adapter/VectorAdapter.h" #include "knowhere/common/Exception.h" @@ -133,7 +133,7 @@ GPUIVF::search_impl(int64_t n, const float* data, int64_t k, float* distances, i if (auto device_index = std::dynamic_pointer_cast(index_)) { auto search_cfg = std::dynamic_pointer_cast(cfg); device_index->nprobe = search_cfg->nprobe; -// assert(device_index->getNumProbes() == search_cfg->nprobe); + // assert(device_index->getNumProbes() == search_cfg->nprobe); { ResScope rs(res_, gpu_id_); diff --git a/core/src/index/knowhere/knowhere/index/vector_index/IndexGPUIVFPQ.cpp b/core/src/index/knowhere/knowhere/index/vector_index/IndexGPUIVFPQ.cpp index 9ba8dd0456..b027539c37 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/IndexGPUIVFPQ.cpp +++ b/core/src/index/knowhere/knowhere/index/vector_index/IndexGPUIVFPQ.cpp @@ -16,9 +16,9 @@ // under the License. #include +#include #include #include -#include #include diff --git a/core/src/index/knowhere/knowhere/index/vector_index/IndexGPUIVFSQ.cpp b/core/src/index/knowhere/knowhere/index/vector_index/IndexGPUIVFSQ.cpp index fff27cd7db..941f9adc48 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/IndexGPUIVFSQ.cpp +++ b/core/src/index/knowhere/knowhere/index/vector_index/IndexGPUIVFSQ.cpp @@ -15,8 +15,8 @@ // specific language governing permissions and limitations // under the License. -#include #include +#include #include diff --git a/core/src/index/knowhere/knowhere/index/vector_index/IndexIDMAP.cpp b/core/src/index/knowhere/knowhere/index/vector_index/IndexIDMAP.cpp index 643bb16076..f926951736 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/IndexIDMAP.cpp +++ b/core/src/index/knowhere/knowhere/index/vector_index/IndexIDMAP.cpp @@ -17,9 +17,9 @@ #include #include -#include -#include #include +#include +#include #include diff --git a/core/src/index/knowhere/knowhere/index/vector_index/IndexIVF.cpp b/core/src/index/knowhere/knowhere/index/vector_index/IndexIVF.cpp index 02708ff5d7..73f5c4164b 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/IndexIVF.cpp +++ b/core/src/index/knowhere/knowhere/index/vector_index/IndexIVF.cpp @@ -15,11 +15,11 @@ // specific language governing permissions and limitations // under the License. -#include #include #include #include #include +#include #include #include #include diff --git a/core/src/index/knowhere/knowhere/index/vector_index/IndexIVFSQ.cpp b/core/src/index/knowhere/knowhere/index/vector_index/IndexIVFSQ.cpp index 80b4c78883..6e9a1d94da 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/IndexIVFSQ.cpp +++ b/core/src/index/knowhere/knowhere/index/vector_index/IndexIVFSQ.cpp @@ -15,8 +15,8 @@ // specific language governing permissions and limitations // under the License. -#include #include +#include #include #include "knowhere/adapter/VectorAdapter.h" diff --git a/core/src/index/knowhere/knowhere/index/vector_index/IndexIVFSQHybrid.cpp b/core/src/index/knowhere/knowhere/index/vector_index/IndexIVFSQHybrid.cpp index af67722266..7b229db21e 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/IndexIVFSQHybrid.cpp +++ b/core/src/index/knowhere/knowhere/index/vector_index/IndexIVFSQHybrid.cpp @@ -22,19 +22,19 @@ #include +#include #include #include -#include namespace knowhere { #ifdef CUSTOMIZATION -//std::mutex g_mutex; +// std::mutex g_mutex; IndexModelPtr IVFSQHybrid::Train(const DatasetPtr& dataset, const Config& config) { -// std::lock_guard lk(g_mutex); + // std::lock_guard lk(g_mutex); auto build_cfg = std::dynamic_pointer_cast(config); if (build_cfg != nullptr) { @@ -74,12 +74,12 @@ IVFSQHybrid::CopyGpuToCpu(const Config& config) { } std::lock_guard lk(mutex_); - faiss::Index* device_index = index_.get(); - faiss::Index* host_index = faiss::gpu::index_gpu_to_cpu(device_index); + faiss::Index* device_index = index_.get(); + faiss::Index* host_index = faiss::gpu::index_gpu_to_cpu(device_index); - std::shared_ptr new_index; - new_index.reset(host_index); - return std::make_shared(new_index); + std::shared_ptr new_index; + new_index.reset(host_index); + return std::make_shared(new_index); } VectorIndexPtr @@ -119,14 +119,14 @@ IVFSQHybrid::LoadImpl(const BinarySet& index_binary) { void IVFSQHybrid::search_impl(int64_t n, const float* data, int64_t k, float* distances, int64_t* labels, const Config& cfg) { -// std::lock_guard lk(g_mutex); -// static int64_t search_count; -// ++search_count; + // std::lock_guard lk(g_mutex); + // static int64_t search_count; + // ++search_count; if (gpu_mode == 2) { GPUIVF::search_impl(n, data, k, distances, labels, cfg); -// index_->search(n, (float*)data, k, distances, labels); - } else if (gpu_mode == 1) { // hybrid + // index_->search(n, (float*)data, k, distances, labels); + } else if (gpu_mode == 1) { // hybrid if (auto res = FaissGpuResourceMgr::GetInstance().GetRes(quantizer_gpu_id_)) { ResScope rs(res, quantizer_gpu_id_, true); IVF::search_impl(n, data, k, distances, labels, cfg); @@ -140,7 +140,7 @@ IVFSQHybrid::search_impl(int64_t n, const float* data, int64_t k, float* distanc QuantizerPtr IVFSQHybrid::LoadQuantizer(const Config& conf) { -// std::lock_guard lk(g_mutex); + // std::lock_guard lk(g_mutex); auto quantizer_conf = std::dynamic_pointer_cast(conf); if (quantizer_conf != nullptr) { @@ -179,7 +179,7 @@ IVFSQHybrid::LoadQuantizer(const Config& conf) { void IVFSQHybrid::SetQuantizer(const QuantizerPtr& q) { -// std::lock_guard lk(g_mutex); + // std::lock_guard lk(g_mutex); auto ivf_quantizer = std::dynamic_pointer_cast(q); if (ivf_quantizer == nullptr) { @@ -199,7 +199,7 @@ IVFSQHybrid::SetQuantizer(const QuantizerPtr& q) { void IVFSQHybrid::UnsetQuantizer() { -// std::lock_guard lk(g_mutex); + // std::lock_guard lk(g_mutex); auto* ivf_index = dynamic_cast(index_.get()); if (ivf_index == nullptr) { @@ -212,7 +212,7 @@ IVFSQHybrid::UnsetQuantizer() { VectorIndexPtr IVFSQHybrid::LoadData(const knowhere::QuantizerPtr& q, const Config& conf) { -// std::lock_guard lk(g_mutex); + // std::lock_guard lk(g_mutex); auto quantizer_conf = std::dynamic_pointer_cast(conf); if (quantizer_conf != nullptr) { @@ -251,7 +251,7 @@ IVFSQHybrid::LoadData(const knowhere::QuantizerPtr& q, const Config& conf) { std::pair IVFSQHybrid::CopyCpuToGpuWithQuantizer(const int64_t& device_id, const Config& config) { -// std::lock_guard lk(g_mutex); + // std::lock_guard lk(g_mutex); if (auto res = FaissGpuResourceMgr::GetInstance().GetRes(device_id)) { ResScope rs(res, device_id, false); diff --git a/core/src/index/knowhere/knowhere/index/vector_index/IndexIVFSQHybrid.h b/core/src/index/knowhere/knowhere/index/vector_index/IndexIVFSQHybrid.h index 87cc22931f..d2a3be6c39 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/IndexIVFSQHybrid.h +++ b/core/src/index/knowhere/knowhere/index/vector_index/IndexIVFSQHybrid.h @@ -17,8 +17,8 @@ #pragma once -#include #include +#include #include #include diff --git a/core/src/index/unittest/test_customized_index.cpp b/core/src/index/unittest/test_customized_index.cpp index f9b48b8b67..346e8e3d93 100644 --- a/core/src/index/unittest/test_customized_index.cpp +++ b/core/src/index/unittest/test_customized_index.cpp @@ -125,7 +125,7 @@ TEST_F(SingleIndexTest, IVFSQHybrid) { } } -//TEST_F(SingleIndexTest, thread_safe) { +// TEST_F(SingleIndexTest, thread_safe) { // assert(!xb.empty()); // // index_type = "IVFSQHybrid"; @@ -234,5 +234,4 @@ TEST_F(SingleIndexTest, IVFSQHybrid) { // //} - #endif diff --git a/core/src/index/unittest/test_ivf.cpp b/core/src/index/unittest/test_ivf.cpp index 3fd3e16d0e..20addc82bb 100644 --- a/core/src/index/unittest/test_ivf.cpp +++ b/core/src/index/unittest/test_ivf.cpp @@ -44,9 +44,9 @@ class IVFTest : public DataGen, public TestWithParam<::std::tupleType() != TaskType::SearchTask) { return false; } @@ -67,7 +67,7 @@ namespace scheduler { task->label() = label; return true; - } +} } // namespace scheduler } // namespace milvus From 6a005b9f714a71284a6fcfb5018948b4b34cab5a Mon Sep 17 00:00:00 2001 From: JinHai-CN Date: Sat, 19 Oct 2019 20:33:55 +0800 Subject: [PATCH 04/31] update ci files Former-commit-id: 1dc3c1153a1e3f2d21bf7981c4349abc29b305ee --- ci/jenkinsfile/milvus_build.groovy | 2 +- ci/jenkinsfile/milvus_build_no_ut.groovy | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/jenkinsfile/milvus_build.groovy b/ci/jenkinsfile/milvus_build.groovy index 11f8c51d33..e7341988b2 100644 --- a/ci/jenkinsfile/milvus_build.groovy +++ b/ci/jenkinsfile/milvus_build.groovy @@ -14,7 +14,7 @@ container('milvus-build-env') { sh "export JFROG_ARTFACTORY_URL='${params.JFROG_ARTFACTORY_URL}' \ && export JFROG_USER_NAME='${USERNAME}' \ && export JFROG_PASSWORD='${PASSWORD}' \ - && export FAISS_URL='http://192.168.1.105:6060/jinhai/faiss/-/archive/branch-0.2.1/faiss-branch-0.2.1.tar.gz' \ + && export FAISS_URL='http://192.168.1.105:6060/jinhai/faiss/-/archive/branch-0.3.0/faiss-branch-0.3.0.tar.gz' \ && ./build.sh -t ${params.BUILD_TYPE} -d /opt/milvus -j -u -c" sh "./coverage.sh -u root -p 123456 -t 192.168.1.194" diff --git a/ci/jenkinsfile/milvus_build_no_ut.groovy b/ci/jenkinsfile/milvus_build_no_ut.groovy index 1dd3361106..3f221b8947 100644 --- a/ci/jenkinsfile/milvus_build_no_ut.groovy +++ b/ci/jenkinsfile/milvus_build_no_ut.groovy @@ -14,7 +14,7 @@ container('milvus-build-env') { sh "export JFROG_ARTFACTORY_URL='${params.JFROG_ARTFACTORY_URL}' \ && export JFROG_USER_NAME='${USERNAME}' \ && export JFROG_PASSWORD='${PASSWORD}' \ - && export FAISS_URL='http://192.168.1.105:6060/jinhai/faiss/-/archive/branch-0.2.1/faiss-branch-0.2.1.tar.gz' \ + && export FAISS_URL='http://192.168.1.105:6060/jinhai/faiss/-/archive/branch-0.3.0/faiss-branch-0.3.0.tar.gz' \ && ./build.sh -t ${params.BUILD_TYPE} -j -d /opt/milvus" } } From cdb28ceaaea5c5af642f2d1c9403566bfafd0acf Mon Sep 17 00:00:00 2001 From: JinHai-CN Date: Sat, 19 Oct 2019 21:07:17 +0800 Subject: [PATCH 05/31] Update ci files Former-commit-id: 7972762f8bf8607d8f77e07a3226f08a24981308 --- core/src/index/thirdparty/versions.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/index/thirdparty/versions.txt b/core/src/index/thirdparty/versions.txt index 9ee845f1e3..a2b16414c2 100644 --- a/core/src/index/thirdparty/versions.txt +++ b/core/src/index/thirdparty/versions.txt @@ -3,4 +3,4 @@ BOOST_VERSION=1.70.0 GTEST_VERSION=1.8.1 LAPACK_VERSION=v3.8.0 OPENBLAS_VERSION=v0.3.6 -FAISS_VERSION=branch-0.2.1 \ No newline at end of file +FAISS_VERSION=branch-0.3.0 \ No newline at end of file From 635434126c8b7a6372ee0a46a653cdc517ecdf5d Mon Sep 17 00:00:00 2001 From: "yudong.cai" Date: Wed, 23 Oct 2019 17:44:57 +0800 Subject: [PATCH 06/31] #89 add faiss benchmark Former-commit-id: 12f9741900e36bb22cd8b7839f16174fd5d0c6f9 --- core/src/index/unittest/CMakeLists.txt | 1 + .../unittest/faiss_benchmark/CMakeLists.txt | 24 + .../faiss_benchmark/faiss_benchmark_test.cpp | 546 ++++++++++++++++++ 3 files changed, 571 insertions(+) create mode 100644 core/src/index/unittest/faiss_benchmark/CMakeLists.txt create mode 100644 core/src/index/unittest/faiss_benchmark/faiss_benchmark_test.cpp diff --git a/core/src/index/unittest/CMakeLists.txt b/core/src/index/unittest/CMakeLists.txt index f840b28e28..2e84908cd7 100644 --- a/core/src/index/unittest/CMakeLists.txt +++ b/core/src/index/unittest/CMakeLists.txt @@ -86,5 +86,6 @@ install(TARGETS test_gpuresource DESTINATION unittest) install(TARGETS test_customized_index DESTINATION unittest) #add_subdirectory(faiss_ori) +#add_subdirectory(faiss_benchmark) add_subdirectory(test_nsg) diff --git a/core/src/index/unittest/faiss_benchmark/CMakeLists.txt b/core/src/index/unittest/faiss_benchmark/CMakeLists.txt new file mode 100644 index 0000000000..556364b68a --- /dev/null +++ b/core/src/index/unittest/faiss_benchmark/CMakeLists.txt @@ -0,0 +1,24 @@ +include_directories(${INDEX_SOURCE_DIR}/thirdparty) +include_directories(${INDEX_SOURCE_DIR}/include) +include_directories(/usr/local/cuda/include) +include_directories(/usr/local/hdf5/include) + +link_directories(/usr/local/cuda/lib64) +link_directories(/usr/local/hdf5/lib) + +set(unittest_libs + gtest gmock gtest_main gmock_main) + +set(depend_libs + faiss openblas lapack hdf5 + arrow ${ARROW_PREFIX}/lib/libjemalloc_pic.a + ) + +set(basic_libs + cudart cublas + gomp gfortran pthread + ) + +add_executable(test_faiss_benchmark faiss_benchmark_test.cpp) +target_link_libraries(test_faiss_benchmark ${depend_libs} ${unittest_libs} ${basic_libs}) +install(TARGETS test_faiss_benchmark DESTINATION unittest) diff --git a/core/src/index/unittest/faiss_benchmark/faiss_benchmark_test.cpp b/core/src/index/unittest/faiss_benchmark/faiss_benchmark_test.cpp new file mode 100644 index 0000000000..5ece23c7aa --- /dev/null +++ b/core/src/index/unittest/faiss_benchmark/faiss_benchmark_test.cpp @@ -0,0 +1,546 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include +#include +#include +#include +#include + +/***************************************************** + * To run this test, please download the HDF5 from + * https://support.hdfgroup.org/ftp/HDF5/releases/ + * and install it to /usr/local/hdf5 . + *****************************************************/ + +double elapsed() { + struct timeval tv; + gettimeofday(&tv, nullptr); + return tv.tv_sec + tv.tv_usec * 1e-6; +} + +void* hdf5_read(const char *file_name, + const char *dataset_name, + H5T_class_t dataset_class, + size_t &d_out, + size_t &n_out) { + hid_t file, dataset, datatype, dataspace, memspace; + H5T_class_t t_class; /* data type class */ + H5T_order_t order; /* data order */ + size_t size; /* size of the data element stored in file */ + hsize_t dimsm[3]; /* memory space dimensions */ + hsize_t dims_out[2]; /* dataset dimensions */ + hsize_t count[2]; /* size of the hyperslab in the file */ + hsize_t offset[2]; /* hyperslab offset in the file */ + hsize_t count_out[3]; /* size of the hyperslab in memory */ + hsize_t offset_out[3]; /* hyperslab offset in memory */ + int rank; + void* data_out; /* output buffer */ + + /* Open the file and the dataset. */ + file = H5Fopen(file_name, H5F_ACC_RDONLY, H5P_DEFAULT); + dataset = H5Dopen2(file, dataset_name, H5P_DEFAULT); + + /* + * Get datatype and dataspace handles and then query + * dataset class, order, size, rank and dimensions. + */ + datatype = H5Dget_type(dataset); /* datatype handle */ + t_class = H5Tget_class(datatype); + assert(t_class == dataset_class || !"Illegal dataset class type"); + + order = H5Tget_order(datatype); + switch (order) { + case H5T_ORDER_LE: + printf("Little endian order \n"); + break; + case H5T_ORDER_BE: + printf("Big endian order \n"); + break; + default: + printf("Illegal endian order \n"); + break; + } + + size = H5Tget_size(datatype); + printf("Data size is %d \n", (int)size); + + dataspace = H5Dget_space(dataset); /* dataspace handle */ + rank = H5Sget_simple_extent_ndims(dataspace); + H5Sget_simple_extent_dims(dataspace, dims_out, NULL); + n_out = dims_out[0]; + d_out = dims_out[1]; + printf("rank %d, dimensions %lu x %lu \n", rank, n_out, d_out); + + /* Define hyperslab in the dataset. */ + offset[0] = offset[1] = 0; + count[0] = dims_out[0]; + count[1] = dims_out[1]; + H5Sselect_hyperslab(dataspace, H5S_SELECT_SET, offset, NULL, count, NULL); + + /* Define the memory dataspace. */ + dimsm[0] = dims_out[0]; + dimsm[1] = dims_out[1]; + dimsm[2] = 1; + memspace = H5Screate_simple(3, dimsm, NULL); + + /* Define memory hyperslab. */ + offset_out[0] = offset_out[1] = offset_out[2] = 0; + count_out[0] = dims_out[0]; + count_out[1] = dims_out[1]; + count_out[2] = 1; + H5Sselect_hyperslab(memspace, H5S_SELECT_SET, offset_out, NULL, count_out, NULL); + + /* Read data from hyperslab in the file into the hyperslab in memory and display. */ + switch (t_class) { + case H5T_INTEGER: + data_out = new int[dims_out[0] * dims_out[1]]; + H5Dread(dataset, H5T_NATIVE_INT, memspace, dataspace, H5P_DEFAULT, data_out); + break; + case H5T_FLOAT: + data_out = new float[dims_out[0] * dims_out[1]]; + H5Dread(dataset, H5T_NATIVE_FLOAT, memspace, dataspace, H5P_DEFAULT, data_out); + break; + default: + printf("Illegal dataset class type\n"); + break; + } + + /* Close/release resources. */ + H5Tclose(datatype); + H5Dclose(dataset); + H5Sclose(dataspace); + H5Sclose(memspace); + H5Fclose(file); + + return data_out; +} + +std::string get_index_file_name(const std::string& ann_test_name, + const std::string& index_key, + int32_t data_loops) { + size_t pos = index_key.find_first_of(',', 0); + std::string file_name = ann_test_name; + file_name = file_name + "_" + index_key.substr(0, pos) + "_" + index_key.substr(pos+1); + file_name = file_name + "_" + std::to_string(data_loops) + ".index"; + return file_name; +} + +bool parse_ann_test_name(const std::string& ann_test_name, + size_t &dim, + faiss::MetricType &metric_type) { + size_t pos1, pos2; + + if (ann_test_name.empty()) return false; + + pos1 = ann_test_name.find_first_of('-', 0); + if (pos1 == std::string::npos) return false; + pos2 = ann_test_name.find_first_of('-', pos1 + 1); + if (pos2 == std::string::npos) return false; + + dim = std::stoi(ann_test_name.substr(pos1+1, pos2-pos1-1)); + std::string metric_str = ann_test_name.substr(pos2+1); + if (metric_str == "angular") { + metric_type = faiss::METRIC_INNER_PRODUCT; + } else if (metric_str == "euclidean") { + metric_type = faiss::METRIC_L2; + } else { + return false; + } + + return true; +} + +void test_ann_hdf5(const std::string& ann_test_name, + const std::string& index_key, + int32_t index_add_loops, + const std::vector& nprobes) { + double t0 = elapsed(); + + const std::string ann_file_name = ann_test_name + ".hdf5"; + + faiss::MetricType metric_type; + size_t dim; + + if (!parse_ann_test_name(ann_test_name, dim, metric_type)) { + printf("Invalid ann test name: %s\n", ann_test_name.c_str()); + return; + } + + faiss::Index * index; + size_t d; + + std::string index_file_name = get_index_file_name(ann_test_name, index_key, index_add_loops); + try { + index = faiss::read_index(index_file_name.c_str()); + d = dim; + } + catch (...) { + printf("Cannot read index file: %s\n", index_file_name.c_str()); + + printf ("[%.3f s] Loading train set\n", elapsed() - t0); + + size_t nb; + float *xb = (float*)hdf5_read(ann_file_name.c_str(), "train", H5T_FLOAT, d, nb); + assert(d == dim || !"dataset does not have correct dimension"); + + printf ("[%.3f s] Preparing index \"%s\" d=%ld\n", + elapsed() - t0, index_key.c_str(), d); + + index = faiss::index_factory(d, index_key.c_str(), metric_type); + + printf ("[%.3f s] Training on %ld vectors\n", elapsed() - t0, nb); + + index->train(nb, xb); + + printf ("[%.3f s] Loading database\n", elapsed() - t0); + + // add index multiple times to get ~1G data set + for (int i = 0; i < index_add_loops; i++) { + printf ("[%.3f s] Indexing database, size %ld*%ld\n", elapsed() - t0, nb, d); + index->add(nb, xb); + } + + faiss::write_index(index, index_file_name.c_str()); + + delete [] xb; + } + + size_t nq; + float *xq; + { + printf ("[%.3f s] Loading queries\n", elapsed() - t0); + + size_t d2; + xq = (float*)hdf5_read(ann_file_name.c_str(), "test", H5T_FLOAT, d2, nq); + assert(d == d2 || !"query does not have same dimension as train set"); + } + + size_t k; // nb of results per query in the GT + faiss::Index::idx_t *gt; // nq * k matrix of ground-truth nearest-neighbors + { + printf ("[%.3f s] Loading ground truth for %ld queries\n", elapsed() - t0, nq); + + // load ground-truth and convert int to long + size_t nq2; + int *gt_int = (int*)hdf5_read(ann_file_name.c_str(), "neighbors", H5T_INTEGER, k, nq2); + assert(nq2 == nq || !"incorrect nb of ground truth entries"); + + gt = new faiss::Index::idx_t[k * nq]; + for(int i = 0; i < k * nq; i++) { + gt[i] = gt_int[i]; + } + delete [] gt_int; + } + + for (auto nprobe : nprobes) { + + faiss::ParameterSpace params; + + printf ("[%.3f s] Setting parameter configuration 'nprobe=%lu' on index\n", elapsed() - t0, nprobe); + + std::string nprobe_str = "nprobe=" + std::to_string(nprobe); + params.set_index_parameters(index, nprobe_str.c_str()); + + // output buffers +#if 1 + const size_t NQ = 1000, K = 1000; + faiss::Index::idx_t *I = new faiss::Index::idx_t[NQ * K]; + float *D = new float[NQ * K]; + + printf ("\n%s | %s | nprobe=%lu\n", ann_test_name.c_str(), index_key.c_str(), nprobe); + printf ("====================================================\n"); + for (size_t t_nq = 10; t_nq <= NQ; t_nq *= 10) { // nq = {10, 100, 1000} + for (size_t t_k = 100; t_k <= K; t_k *= 10) { // k = {100, 1000} + double t_start = elapsed(), t_end; + + index->search(t_nq, xq, t_k, D, I); + + t_end = elapsed(); + + // k = 100 for ground truth + int hit = 0; + for (int i = 0; i < t_nq; i++) { + // count the num of results exist in ground truth result set + // consider: each result replicates DATA_LOOPS times + for (int j_c = 0; j_c < k; j_c++) { + int r_c = I[i * t_k + j_c]; + for (int j_g = 0; j_g < k/index_add_loops; j_g++) { + if (gt[i * k + j_g] == r_c) { + hit++; + continue; + } + } + } + } + printf("nq = %4ld, k = %4ld, elapse = %fs, R@ = %.4f\n", + t_nq, t_k, (t_end - t_start), (hit / float(t_nq * k / index_add_loops))); + } + } + printf ("====================================================\n"); +#else + printf ("[%.3f s] Perform a search on %ld queries\n", elapsed() - t0, nq); + + faiss::Index::idx_t *I = new faiss::Index::idx_t[nq * k]; + float *D = new float[nq * k]; + + index->search(nq, xq, k, D, I); + + printf ("[%.3f s] Compute recalls\n", elapsed() - t0); + + // evaluate result by hand. + int n_1 = 0, n_10 = 0, n_100 = 0; + for(int i = 0; i < nq; i++) { + int gt_nn = gt[i * k]; + for(int j = 0; j < k; j++) { + if (I[i * k + j] == gt_nn) { + if(j < 1) n_1++; + if(j < 10) n_10++; + if(j < 100) n_100++; + } + } + } + printf("R@1 = %.4f\n", n_1 / float(nq)); + printf("R@10 = %.4f\n", n_10 / float(nq)); + printf("R@100 = %.4f\n", n_100 / float(nq)); +#endif + + printf ("[%.3f s] Search test done\n\n", elapsed() - t0); + + delete [] I; + delete [] D; + } + + delete [] xq; + delete [] gt; + delete index; +} + +#ifdef CUSTOMIZATION +void test_ivfsq8h_gpu(const std::string& ann_test_name, + int32_t index_add_loops, + const std::vector& nprobes){ + double t0 = elapsed(); + + const std::string ann_file_name = ann_test_name + ".hdf5"; + + faiss::MetricType metric_type; + size_t dim; + + if (!parse_ann_test_name(ann_test_name, dim, metric_type)) { + printf("Invalid ann test name: %s\n", ann_test_name.c_str()); + return; + } + + faiss::distance_compute_blas_threshold = 800; + faiss::gpu::StandardGpuResources res; + + const std::string index_key = "IVF16384,SQ8Hybrid"; + + faiss::Index* cpu_index = nullptr; + size_t d; + + std::string index_file_name = get_index_file_name(ann_test_name, index_key, index_add_loops); + try{ + cpu_index = faiss::read_index(index_file_name.c_str()); + d = dim; + } + catch (...){ + printf("Cannot read index file: %s\n", index_file_name.c_str()); + + printf ("[%.3f s] Loading train set\n", elapsed() - t0); + + size_t nb; + float *xb = (float*)hdf5_read(ann_file_name.c_str(), "train", H5T_FLOAT, d, nb); + assert(d == dim || !"dataset does not have correct dimension"); + + printf ("[%.3f s] Preparing index \"%s\" d=%ld\n", elapsed() - t0, index_key.c_str(), d); + + faiss::Index *ori_index = faiss::index_factory(d, index_key.c_str(), metric_type); + + auto device_index = faiss::gpu::index_cpu_to_gpu(&res, 0, ori_index); + + printf ("[%.3f s] Training on %ld vectors\n", elapsed() - t0, nb); + + device_index->train(nb, xb); + + printf ("[%.3f s] Loading database\n", elapsed() - t0); + + for (int i = 0; i < index_add_loops; i++) { + printf ("[%.3f s] Indexing database, size %ld*%ld\n", elapsed() - t0, nb, d); + device_index->add(nb, xb); + } + + cpu_index = faiss::gpu::index_gpu_to_cpu(device_index); + faiss::write_index(cpu_index, index_file_name.c_str()); + + delete []xb; + } + + faiss::IndexIVF *cpu_ivf_index = dynamic_cast(cpu_index); + if(cpu_ivf_index != nullptr) { + cpu_ivf_index->to_readonly(); + } + + faiss::gpu::GpuClonerOptions option; + option.allInGpu = true; + + faiss::IndexComposition index_composition; + index_composition.index = cpu_index; + index_composition.quantizer = nullptr; + index_composition.mode = 1; + + auto index = faiss::gpu::index_cpu_to_gpu(&res, 0, &index_composition, &option); + delete index; + + size_t nq; + float *xq; + { + printf ("[%.3f s] Loading queries\n", elapsed() - t0); + + size_t d2; + xq = (float*)hdf5_read(ann_file_name.c_str(), "test", H5T_FLOAT, d2, nq); + assert(d == d2 || !"query does not have same dimension as train set"); + } + + size_t k; + faiss::Index::idx_t *gt; + { + printf ("[%.3f s] Loading ground truth for %ld queries\n", elapsed() - t0, nq); + + size_t nq2; + int *gt_int = (int*)hdf5_read(ann_file_name.c_str(), "neighbors", H5T_INTEGER, k, nq2); + assert(nq2 == nq || !"incorrect nb of ground truth entries"); + + gt = new faiss::Index::idx_t[k * nq]; + for (unsigned long i = 0; i < k * nq; ++i) { + gt[i] = gt_int[i]; + } + delete []gt_int; + } + + for (auto nprobe : nprobes){ + printf ("[%.3f s] Setting parameter configuration 'nprobe=%lu' on index\n", + elapsed() - t0, nprobe); + + auto ivf_index = dynamic_cast(cpu_index); + ivf_index->nprobe = nprobe; + + auto is_gpu_flat_index = dynamic_cast(ivf_index->quantizer); + if(is_gpu_flat_index == nullptr) { + delete ivf_index->quantizer; + ivf_index->quantizer = index_composition.quantizer; + } + + const size_t NQ = 1000, K = 1000; + long *I = new faiss::Index::idx_t[NQ * K]; + float *D = new float[NQ * K]; + + printf ("\n%s %ld\n", index_key.c_str(), nprobe); + printf ("\n%s | %s | nprobe=%lu\n", ann_test_name.c_str(), index_key.c_str(), nprobe); + printf ("====================================================\n"); + + for (size_t t_nq = 10; t_nq <= NQ; t_nq *= 10) { // nq = {10, 100, 1000} + for (size_t t_k = 100; t_k <= K; t_k *= 10) { // k = {100, 1000} + double t_start = elapsed(), t_end; + + cpu_index->search(t_nq, xq, t_k, D, I); + + t_end = elapsed(); + + // k = 100 for ground truth + int hit = 0; + for (unsigned long i = 0; i < t_nq; i++) { + // count the num of results exist in ground truth result set + // consider: each result replicates DATA_LOOPS times + for (unsigned long j_c = 0; j_c < k; j_c++) { + int r_c = I[i * t_k + j_c]; + for (unsigned long j_g = 0; j_g < k/index_add_loops; j_g++) { + if (gt[i * k + j_g] == r_c) { + hit++; + continue; + } + } + } + } + printf("nq = %4ld, k = %4ld, elapse = %fs, R@ = %.4f\n", + t_nq, t_k, (t_end - t_start), (hit / float(t_nq * k / index_add_loops))); + } + } + printf ("====================================================\n"); + + printf ("[%.3f s] Search test done\n\n", elapsed() - t0); + + delete [] I; + delete [] D; + } + + delete [] xq; + delete [] gt; + delete cpu_index; +} +#endif + +/************************************************************************************ + * https://github.com/erikbern/ann-benchmarks + * + * Dataset Dimensions Train_size Test_size Neighbors Distance Download + * Fashion- + * MNIST 784 60,000 10,000 100 Euclidean HDF5 (217MB) + * GIST 960 1,000,000 1,000 100 Euclidean HDF5 (3.6GB) + * GloVe 100 1,183,514 10,000 100 Angular HDF5 (463MB) + * GloVe 200 1,183,514 10,000 100 Angular HDF5 (918MB) + * MNIST 784 60,000 10,000 100 Euclidean HDF5 (217MB) + * NYTimes 256 290,000 10,000 100 Angular HDF5 (301MB) + * SIFT 128 1,000,000 10,000 100 Euclidean HDF5 (501MB) +*************************************************************************************/ + +TEST(FAISSTEST, sift1m_L2) { + test_ann_hdf5("sift-128-euclidean", "IVF4096,Flat", 2, {8, 128}); + test_ann_hdf5("sift-128-euclidean", "IVF16384,SQ8", 2, {8, 128}); + test_ann_hdf5("sift-128-euclidean", "IVF16384,SQ8Hybrid", 2, {8, 128}); +#ifdef CUSTOMIZATION + test_ivfsq8h_gpu("sift-128-euclidean", 2, {8, 128}); +#endif + + test_ann_hdf5("glove-200-angular", "IVF4096,Flat", 1, {8, 128}); + test_ann_hdf5("glove-200-angular", "IVF16384,SQ8", 1, {8, 128}); + test_ann_hdf5("glove-200-angular", "IVF16384,SQ8Hybrid", 1, {8, 128}); +#ifdef CUSTOMIZATION + test_ivfsq8h_gpu("glove-200-angular", 2, {128, 1024}); +#endif +} + From 52ca4c4ae2578e270ca4b41a6f8f1f4e1b2a68d9 Mon Sep 17 00:00:00 2001 From: "yudong.cai" Date: Wed, 23 Oct 2019 17:46:27 +0800 Subject: [PATCH 07/31] update test parameter Former-commit-id: 0871f4f0acd87280b5327caf4f14dae5b0d5e165 --- .../src/index/unittest/faiss_benchmark/faiss_benchmark_test.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/index/unittest/faiss_benchmark/faiss_benchmark_test.cpp b/core/src/index/unittest/faiss_benchmark/faiss_benchmark_test.cpp index 5ece23c7aa..5f787a415c 100644 --- a/core/src/index/unittest/faiss_benchmark/faiss_benchmark_test.cpp +++ b/core/src/index/unittest/faiss_benchmark/faiss_benchmark_test.cpp @@ -540,7 +540,7 @@ TEST(FAISSTEST, sift1m_L2) { test_ann_hdf5("glove-200-angular", "IVF16384,SQ8", 1, {8, 128}); test_ann_hdf5("glove-200-angular", "IVF16384,SQ8Hybrid", 1, {8, 128}); #ifdef CUSTOMIZATION - test_ivfsq8h_gpu("glove-200-angular", 2, {128, 1024}); + test_ivfsq8h_gpu("glove-200-angular", 2, {8, 128}); #endif } From 4c7d590eb22323d1ef0f6a474a1f332d5c4ebd66 Mon Sep 17 00:00:00 2001 From: "yudong.cai" Date: Wed, 23 Oct 2019 18:24:35 +0800 Subject: [PATCH 08/31] 9 update unittest name Former-commit-id: 4bd87de1ceee872b9de13c5e5c65bab7291eb9c7 --- .../index/unittest/faiss_benchmark/faiss_benchmark_test.cpp | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/core/src/index/unittest/faiss_benchmark/faiss_benchmark_test.cpp b/core/src/index/unittest/faiss_benchmark/faiss_benchmark_test.cpp index 5f787a415c..f1dc060825 100644 --- a/core/src/index/unittest/faiss_benchmark/faiss_benchmark_test.cpp +++ b/core/src/index/unittest/faiss_benchmark/faiss_benchmark_test.cpp @@ -469,10 +469,8 @@ void test_ivfsq8h_gpu(const std::string& ann_test_name, long *I = new faiss::Index::idx_t[NQ * K]; float *D = new float[NQ * K]; - printf ("\n%s %ld\n", index_key.c_str(), nprobe); printf ("\n%s | %s | nprobe=%lu\n", ann_test_name.c_str(), index_key.c_str(), nprobe); printf ("====================================================\n"); - for (size_t t_nq = 10; t_nq <= NQ; t_nq *= 10) { // nq = {10, 100, 1000} for (size_t t_k = 100; t_k <= K; t_k *= 10) { // k = {100, 1000} double t_start = elapsed(), t_end; @@ -528,7 +526,7 @@ void test_ivfsq8h_gpu(const std::string& ann_test_name, * SIFT 128 1,000,000 10,000 100 Euclidean HDF5 (501MB) *************************************************************************************/ -TEST(FAISSTEST, sift1m_L2) { +TEST(FAISSTEST, BENCHMARK) { test_ann_hdf5("sift-128-euclidean", "IVF4096,Flat", 2, {8, 128}); test_ann_hdf5("sift-128-euclidean", "IVF16384,SQ8", 2, {8, 128}); test_ann_hdf5("sift-128-euclidean", "IVF16384,SQ8Hybrid", 2, {8, 128}); From 82ddcf8340b84a1eb2f4ea0209554a006f4dac6a Mon Sep 17 00:00:00 2001 From: "yudong.cai" Date: Wed, 23 Oct 2019 18:39:43 +0800 Subject: [PATCH 09/31] #89 display quant/search time Former-commit-id: 563141ab22274ca0e3e84253df8a79af58c16eca --- .../faiss_benchmark/faiss_benchmark_test.cpp | 28 +++++++++++++------ 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/core/src/index/unittest/faiss_benchmark/faiss_benchmark_test.cpp b/core/src/index/unittest/faiss_benchmark/faiss_benchmark_test.cpp index f1dc060825..5d63d63003 100644 --- a/core/src/index/unittest/faiss_benchmark/faiss_benchmark_test.cpp +++ b/core/src/index/unittest/faiss_benchmark/faiss_benchmark_test.cpp @@ -282,9 +282,12 @@ void test_ann_hdf5(const std::string& ann_test_name, float *D = new float[NQ * K]; printf ("\n%s | %s | nprobe=%lu\n", ann_test_name.c_str(), index_key.c_str(), nprobe); - printf ("====================================================\n"); + printf ("============================================================================================\n"); for (size_t t_nq = 10; t_nq <= NQ; t_nq *= 10) { // nq = {10, 100, 1000} for (size_t t_k = 100; t_k <= K; t_k *= 10) { // k = {100, 1000} + faiss::indexIVF_stats.quantization_time = 0.0; + faiss::indexIVF_stats.search_time = 0.0; + double t_start = elapsed(), t_end; index->search(t_nq, xq, t_k, D, I); @@ -306,11 +309,14 @@ void test_ann_hdf5(const std::string& ann_test_name, } } } - printf("nq = %4ld, k = %4ld, elapse = %fs, R@ = %.4f\n", - t_nq, t_k, (t_end - t_start), (hit / float(t_nq * k / index_add_loops))); + printf("nq = %4ld, k = %4ld, elapse = %fs (quant = %fs, search = %fs), R@ = %.4f\n", + t_nq, t_k, (t_end - t_start), + faiss::indexIVF_stats.quantization_time / 1000, + faiss::indexIVF_stats.search_time / 1000, + (hit / float(t_nq * k / index_add_loops))); } } - printf ("====================================================\n"); + printf ("============================================================================================\n"); #else printf ("[%.3f s] Perform a search on %ld queries\n", elapsed() - t0, nq); @@ -470,9 +476,12 @@ void test_ivfsq8h_gpu(const std::string& ann_test_name, float *D = new float[NQ * K]; printf ("\n%s | %s | nprobe=%lu\n", ann_test_name.c_str(), index_key.c_str(), nprobe); - printf ("====================================================\n"); + printf ("============================================================================================\n"); for (size_t t_nq = 10; t_nq <= NQ; t_nq *= 10) { // nq = {10, 100, 1000} for (size_t t_k = 100; t_k <= K; t_k *= 10) { // k = {100, 1000} + faiss::indexIVF_stats.quantization_time = 0.0; + faiss::indexIVF_stats.search_time = 0.0; + double t_start = elapsed(), t_end; cpu_index->search(t_nq, xq, t_k, D, I); @@ -494,11 +503,14 @@ void test_ivfsq8h_gpu(const std::string& ann_test_name, } } } - printf("nq = %4ld, k = %4ld, elapse = %fs, R@ = %.4f\n", - t_nq, t_k, (t_end - t_start), (hit / float(t_nq * k / index_add_loops))); + printf("nq = %4ld, k = %4ld, elapse = %fs (quant = %fs, search = %fs), R@ = %.4f\n", + t_nq, t_k, (t_end - t_start), + faiss::indexIVF_stats.quantization_time / 1000, + faiss::indexIVF_stats.search_time / 1000, + (hit / float(t_nq * k / index_add_loops))); } } - printf ("====================================================\n"); + printf ("============================================================================================\n"); printf ("[%.3f s] Search test done\n\n", elapsed() - t0); From ec022c330d264ec6fcbd28ef28e613b4d3804b45 Mon Sep 17 00:00:00 2001 From: "yudong.cai" Date: Thu, 24 Oct 2019 09:33:09 +0800 Subject: [PATCH 10/31] #89 update unittest Former-commit-id: f9b518f2961f3c7da30a76a53a49da8403208a0b --- .../index/unittest/faiss_benchmark/faiss_benchmark_test.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/src/index/unittest/faiss_benchmark/faiss_benchmark_test.cpp b/core/src/index/unittest/faiss_benchmark/faiss_benchmark_test.cpp index 5d63d63003..3d60574231 100644 --- a/core/src/index/unittest/faiss_benchmark/faiss_benchmark_test.cpp +++ b/core/src/index/unittest/faiss_benchmark/faiss_benchmark_test.cpp @@ -541,15 +541,15 @@ void test_ivfsq8h_gpu(const std::string& ann_test_name, TEST(FAISSTEST, BENCHMARK) { test_ann_hdf5("sift-128-euclidean", "IVF4096,Flat", 2, {8, 128}); test_ann_hdf5("sift-128-euclidean", "IVF16384,SQ8", 2, {8, 128}); - test_ann_hdf5("sift-128-euclidean", "IVF16384,SQ8Hybrid", 2, {8, 128}); #ifdef CUSTOMIZATION + test_ann_hdf5("sift-128-euclidean", "IVF16384,SQ8Hybrid", 2, {8, 128}); test_ivfsq8h_gpu("sift-128-euclidean", 2, {8, 128}); #endif test_ann_hdf5("glove-200-angular", "IVF4096,Flat", 1, {8, 128}); test_ann_hdf5("glove-200-angular", "IVF16384,SQ8", 1, {8, 128}); - test_ann_hdf5("glove-200-angular", "IVF16384,SQ8Hybrid", 1, {8, 128}); #ifdef CUSTOMIZATION + test_ann_hdf5("glove-200-angular", "IVF16384,SQ8Hybrid", 1, {8, 128}); test_ivfsq8h_gpu("glove-200-angular", 2, {8, 128}); #endif } From 4f5906b9bc6316523f2de708b2d3e6cb448aec2d Mon Sep 17 00:00:00 2001 From: "yudong.cai" Date: Thu, 24 Oct 2019 09:44:01 +0800 Subject: [PATCH 11/31] #89 update SQ8Hybrid-gpu log Former-commit-id: c2e70121ee65ed044c059ac3948b3412353e829e --- .../src/index/unittest/faiss_benchmark/faiss_benchmark_test.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/index/unittest/faiss_benchmark/faiss_benchmark_test.cpp b/core/src/index/unittest/faiss_benchmark/faiss_benchmark_test.cpp index 3d60574231..0c7cb97807 100644 --- a/core/src/index/unittest/faiss_benchmark/faiss_benchmark_test.cpp +++ b/core/src/index/unittest/faiss_benchmark/faiss_benchmark_test.cpp @@ -475,7 +475,7 @@ void test_ivfsq8h_gpu(const std::string& ann_test_name, long *I = new faiss::Index::idx_t[NQ * K]; float *D = new float[NQ * K]; - printf ("\n%s | %s | nprobe=%lu\n", ann_test_name.c_str(), index_key.c_str(), nprobe); + printf ("\n%s | %s-gpu | nprobe=%lu\n", ann_test_name.c_str(), index_key.c_str(), nprobe); printf ("============================================================================================\n"); for (size_t t_nq = 10; t_nq <= NQ; t_nq *= 10) { // nq = {10, 100, 1000} for (size_t t_k = 100; t_k <= K; t_k *= 10) { // k = {100, 1000} From d3d6077eb2daedb92b1a33a2e281e6445afd548f Mon Sep 17 00:00:00 2001 From: "yudong.cai" Date: Thu, 24 Oct 2019 10:24:02 +0800 Subject: [PATCH 12/31] #89 update unittest parameter Former-commit-id: 4692890b67109edefbd0cc0a0a5a628f6433306d --- .../src/index/unittest/faiss_benchmark/faiss_benchmark_test.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/index/unittest/faiss_benchmark/faiss_benchmark_test.cpp b/core/src/index/unittest/faiss_benchmark/faiss_benchmark_test.cpp index 0c7cb97807..ed00e74a98 100644 --- a/core/src/index/unittest/faiss_benchmark/faiss_benchmark_test.cpp +++ b/core/src/index/unittest/faiss_benchmark/faiss_benchmark_test.cpp @@ -550,7 +550,7 @@ TEST(FAISSTEST, BENCHMARK) { test_ann_hdf5("glove-200-angular", "IVF16384,SQ8", 1, {8, 128}); #ifdef CUSTOMIZATION test_ann_hdf5("glove-200-angular", "IVF16384,SQ8Hybrid", 1, {8, 128}); - test_ivfsq8h_gpu("glove-200-angular", 2, {8, 128}); + test_ivfsq8h_gpu("glove-200-angular", 1, {8, 128}); #endif } From 65b46de1ac7629948cd27a2ba00c5872b48b2f8b Mon Sep 17 00:00:00 2001 From: "yudong.cai" Date: Thu, 24 Oct 2019 10:56:12 +0800 Subject: [PATCH 13/31] #89 code format Former-commit-id: ced158f26d9c18e38c7afb84ad17fdb6f9057259 --- .../faiss_benchmark/faiss_benchmark_test.cpp | 282 +++++++++--------- 1 file changed, 136 insertions(+), 146 deletions(-) diff --git a/core/src/index/unittest/faiss_benchmark/faiss_benchmark_test.cpp b/core/src/index/unittest/faiss_benchmark/faiss_benchmark_test.cpp index ed00e74a98..d1db0e9049 100644 --- a/core/src/index/unittest/faiss_benchmark/faiss_benchmark_test.cpp +++ b/core/src/index/unittest/faiss_benchmark/faiss_benchmark_test.cpp @@ -17,28 +17,28 @@ #include +#include #include #include #include -#include #include #include #include #include -#include #include #include +#include #include #include #include -#include #include #include #include #include +#include /***************************************************** * To run this test, please download the HDF5 from @@ -46,29 +46,27 @@ * and install it to /usr/local/hdf5 . *****************************************************/ -double elapsed() { +double +elapsed() { struct timeval tv; gettimeofday(&tv, nullptr); return tv.tv_sec + tv.tv_usec * 1e-6; } -void* hdf5_read(const char *file_name, - const char *dataset_name, - H5T_class_t dataset_class, - size_t &d_out, - size_t &n_out) { - hid_t file, dataset, datatype, dataspace, memspace; - H5T_class_t t_class; /* data type class */ - H5T_order_t order; /* data order */ - size_t size; /* size of the data element stored in file */ - hsize_t dimsm[3]; /* memory space dimensions */ - hsize_t dims_out[2]; /* dataset dimensions */ - hsize_t count[2]; /* size of the hyperslab in the file */ - hsize_t offset[2]; /* hyperslab offset in the file */ - hsize_t count_out[3]; /* size of the hyperslab in memory */ - hsize_t offset_out[3]; /* hyperslab offset in memory */ - int rank; - void* data_out; /* output buffer */ +void* +hdf5_read(const char* file_name, const char* dataset_name, H5T_class_t dataset_class, size_t& d_out, size_t& n_out) { + hid_t file, dataset, datatype, dataspace, memspace; + H5T_class_t t_class; /* data type class */ + H5T_order_t order; /* data order */ + size_t size; /* size of the data element stored in file */ + hsize_t dimsm[3]; /* memory space dimensions */ + hsize_t dims_out[2]; /* dataset dimensions */ + hsize_t count[2]; /* size of the hyperslab in the file */ + hsize_t offset[2]; /* hyperslab offset in the file */ + hsize_t count_out[3]; /* size of the hyperslab in memory */ + hsize_t offset_out[3]; /* hyperslab offset in memory */ + int rank; + void* data_out; /* output buffer */ /* Open the file and the dataset. */ file = H5Fopen(file_name, H5F_ACC_RDONLY, H5P_DEFAULT); @@ -78,7 +76,7 @@ void* hdf5_read(const char *file_name, * Get datatype and dataspace handles and then query * dataset class, order, size, rank and dimensions. */ - datatype = H5Dget_type(dataset); /* datatype handle */ + datatype = H5Dget_type(dataset); /* datatype handle */ t_class = H5Tget_class(datatype); assert(t_class == dataset_class || !"Illegal dataset class type"); @@ -95,11 +93,11 @@ void* hdf5_read(const char *file_name, break; } - size = H5Tget_size(datatype); + size = H5Tget_size(datatype); printf("Data size is %d \n", (int)size); - dataspace = H5Dget_space(dataset); /* dataspace handle */ - rank = H5Sget_simple_extent_ndims(dataspace); + dataspace = H5Dget_space(dataset); /* dataspace handle */ + rank = H5Sget_simple_extent_ndims(dataspace); H5Sget_simple_extent_dims(dataspace, dims_out, NULL); n_out = dims_out[0]; d_out = dims_out[1]; @@ -107,8 +105,8 @@ void* hdf5_read(const char *file_name, /* Define hyperslab in the dataset. */ offset[0] = offset[1] = 0; - count[0] = dims_out[0]; - count[1] = dims_out[1]; + count[0] = dims_out[0]; + count[1] = dims_out[1]; H5Sselect_hyperslab(dataspace, H5S_SELECT_SET, offset, NULL, count, NULL); /* Define the memory dataspace. */ @@ -119,9 +117,9 @@ void* hdf5_read(const char *file_name, /* Define memory hyperslab. */ offset_out[0] = offset_out[1] = offset_out[2] = 0; - count_out[0] = dims_out[0]; - count_out[1] = dims_out[1]; - count_out[2] = 1; + count_out[0] = dims_out[0]; + count_out[1] = dims_out[1]; + count_out[2] = 1; H5Sselect_hyperslab(memspace, H5S_SELECT_SET, offset_out, NULL, count_out, NULL); /* Read data from hyperslab in the file into the hyperslab in memory and display. */ @@ -149,30 +147,31 @@ void* hdf5_read(const char *file_name, return data_out; } -std::string get_index_file_name(const std::string& ann_test_name, - const std::string& index_key, - int32_t data_loops) { +std::string +get_index_file_name(const std::string& ann_test_name, const std::string& index_key, int32_t data_loops) { size_t pos = index_key.find_first_of(',', 0); std::string file_name = ann_test_name; - file_name = file_name + "_" + index_key.substr(0, pos) + "_" + index_key.substr(pos+1); + file_name = file_name + "_" + index_key.substr(0, pos) + "_" + index_key.substr(pos + 1); file_name = file_name + "_" + std::to_string(data_loops) + ".index"; return file_name; } -bool parse_ann_test_name(const std::string& ann_test_name, - size_t &dim, - faiss::MetricType &metric_type) { +bool +parse_ann_test_name(const std::string& ann_test_name, size_t& dim, faiss::MetricType& metric_type) { size_t pos1, pos2; - if (ann_test_name.empty()) return false; + if (ann_test_name.empty()) + return false; pos1 = ann_test_name.find_first_of('-', 0); - if (pos1 == std::string::npos) return false; + if (pos1 == std::string::npos) + return false; pos2 = ann_test_name.find_first_of('-', pos1 + 1); - if (pos2 == std::string::npos) return false; + if (pos2 == std::string::npos) + return false; - dim = std::stoi(ann_test_name.substr(pos1+1, pos2-pos1-1)); - std::string metric_str = ann_test_name.substr(pos2+1); + dim = std::stoi(ann_test_name.substr(pos1 + 1, pos2 - pos1 - 1)); + std::string metric_str = ann_test_name.substr(pos2 + 1); if (metric_str == "angular") { metric_type = faiss::METRIC_INNER_PRODUCT; } else if (metric_str == "euclidean") { @@ -184,10 +183,9 @@ bool parse_ann_test_name(const std::string& ann_test_name, return true; } -void test_ann_hdf5(const std::string& ann_test_name, - const std::string& index_key, - int32_t index_add_loops, - const std::vector& nprobes) { +void +test_ann_hdf5(const std::string& ann_test_name, const std::string& index_key, int32_t index_add_loops, + const std::vector& nprobes) { double t0 = elapsed(); const std::string ann_file_name = ann_test_name + ".hdf5"; @@ -200,77 +198,74 @@ void test_ann_hdf5(const std::string& ann_test_name, return; } - faiss::Index * index; + faiss::Index* index; size_t d; std::string index_file_name = get_index_file_name(ann_test_name, index_key, index_add_loops); try { index = faiss::read_index(index_file_name.c_str()); d = dim; - } - catch (...) { + } catch (...) { printf("Cannot read index file: %s\n", index_file_name.c_str()); - printf ("[%.3f s] Loading train set\n", elapsed() - t0); + printf("[%.3f s] Loading train set\n", elapsed() - t0); size_t nb; - float *xb = (float*)hdf5_read(ann_file_name.c_str(), "train", H5T_FLOAT, d, nb); + float* xb = (float*)hdf5_read(ann_file_name.c_str(), "train", H5T_FLOAT, d, nb); assert(d == dim || !"dataset does not have correct dimension"); - printf ("[%.3f s] Preparing index \"%s\" d=%ld\n", - elapsed() - t0, index_key.c_str(), d); + printf("[%.3f s] Preparing index \"%s\" d=%ld\n", elapsed() - t0, index_key.c_str(), d); index = faiss::index_factory(d, index_key.c_str(), metric_type); - printf ("[%.3f s] Training on %ld vectors\n", elapsed() - t0, nb); + printf("[%.3f s] Training on %ld vectors\n", elapsed() - t0, nb); index->train(nb, xb); - printf ("[%.3f s] Loading database\n", elapsed() - t0); + printf("[%.3f s] Loading database\n", elapsed() - t0); // add index multiple times to get ~1G data set for (int i = 0; i < index_add_loops; i++) { - printf ("[%.3f s] Indexing database, size %ld*%ld\n", elapsed() - t0, nb, d); + printf("[%.3f s] Indexing database, size %ld*%ld\n", elapsed() - t0, nb, d); index->add(nb, xb); } faiss::write_index(index, index_file_name.c_str()); - delete [] xb; + delete[] xb; } size_t nq; - float *xq; + float* xq; { - printf ("[%.3f s] Loading queries\n", elapsed() - t0); + printf("[%.3f s] Loading queries\n", elapsed() - t0); size_t d2; xq = (float*)hdf5_read(ann_file_name.c_str(), "test", H5T_FLOAT, d2, nq); assert(d == d2 || !"query does not have same dimension as train set"); } - size_t k; // nb of results per query in the GT - faiss::Index::idx_t *gt; // nq * k matrix of ground-truth nearest-neighbors + size_t k; // nb of results per query in the GT + faiss::Index::idx_t* gt; // nq * k matrix of ground-truth nearest-neighbors { - printf ("[%.3f s] Loading ground truth for %ld queries\n", elapsed() - t0, nq); + printf("[%.3f s] Loading ground truth for %ld queries\n", elapsed() - t0, nq); // load ground-truth and convert int to long size_t nq2; - int *gt_int = (int*)hdf5_read(ann_file_name.c_str(), "neighbors", H5T_INTEGER, k, nq2); + int* gt_int = (int*)hdf5_read(ann_file_name.c_str(), "neighbors", H5T_INTEGER, k, nq2); assert(nq2 == nq || !"incorrect nb of ground truth entries"); gt = new faiss::Index::idx_t[k * nq]; - for(int i = 0; i < k * nq; i++) { + for (int i = 0; i < k * nq; i++) { gt[i] = gt_int[i]; } - delete [] gt_int; + delete[] gt_int; } for (auto nprobe : nprobes) { - faiss::ParameterSpace params; - printf ("[%.3f s] Setting parameter configuration 'nprobe=%lu' on index\n", elapsed() - t0, nprobe); + printf("[%.3f s] Setting parameter configuration 'nprobe=%lu' on index\n", elapsed() - t0, nprobe); std::string nprobe_str = "nprobe=" + std::to_string(nprobe); params.set_index_parameters(index, nprobe_str.c_str()); @@ -278,13 +273,13 @@ void test_ann_hdf5(const std::string& ann_test_name, // output buffers #if 1 const size_t NQ = 1000, K = 1000; - faiss::Index::idx_t *I = new faiss::Index::idx_t[NQ * K]; - float *D = new float[NQ * K]; + faiss::Index::idx_t* I = new faiss::Index::idx_t[NQ * K]; + float* D = new float[NQ * K]; - printf ("\n%s | %s | nprobe=%lu\n", ann_test_name.c_str(), index_key.c_str(), nprobe); - printf ("============================================================================================\n"); - for (size_t t_nq = 10; t_nq <= NQ; t_nq *= 10) { // nq = {10, 100, 1000} - for (size_t t_k = 100; t_k <= K; t_k *= 10) { // k = {100, 1000} + printf("\n%s | %s | nprobe=%lu\n", ann_test_name.c_str(), index_key.c_str(), nprobe); + printf("============================================================================================\n"); + for (size_t t_nq = 10; t_nq <= NQ; t_nq *= 10) { // nq = {10, 100, 1000} + for (size_t t_k = 100; t_k <= K; t_k *= 10) { // k = {100, 1000} faiss::indexIVF_stats.quantization_time = 0.0; faiss::indexIVF_stats.search_time = 0.0; @@ -301,7 +296,7 @@ void test_ann_hdf5(const std::string& ann_test_name, // consider: each result replicates DATA_LOOPS times for (int j_c = 0; j_c < k; j_c++) { int r_c = I[i * t_k + j_c]; - for (int j_g = 0; j_g < k/index_add_loops; j_g++) { + for (int j_g = 0; j_g < k / index_add_loops; j_g++) { if (gt[i * k + j_g] == r_c) { hit++; continue; @@ -309,33 +304,34 @@ void test_ann_hdf5(const std::string& ann_test_name, } } } - printf("nq = %4ld, k = %4ld, elapse = %fs (quant = %fs, search = %fs), R@ = %.4f\n", - t_nq, t_k, (t_end - t_start), - faiss::indexIVF_stats.quantization_time / 1000, - faiss::indexIVF_stats.search_time / 1000, - (hit / float(t_nq * k / index_add_loops))); + printf("nq = %4ld, k = %4ld, elapse = %fs (quant = %fs, search = %fs), R@ = %.4f\n", t_nq, t_k, + (t_end - t_start), faiss::indexIVF_stats.quantization_time / 1000, + faiss::indexIVF_stats.search_time / 1000, (hit / float(t_nq * k / index_add_loops))); } } - printf ("============================================================================================\n"); + printf("============================================================================================\n"); #else - printf ("[%.3f s] Perform a search on %ld queries\n", elapsed() - t0, nq); + printf("[%.3f s] Perform a search on %ld queries\n", elapsed() - t0, nq); - faiss::Index::idx_t *I = new faiss::Index::idx_t[nq * k]; - float *D = new float[nq * k]; + faiss::Index::idx_t* I = new faiss::Index::idx_t[nq * k]; + float* D = new float[nq * k]; index->search(nq, xq, k, D, I); - printf ("[%.3f s] Compute recalls\n", elapsed() - t0); + printf("[%.3f s] Compute recalls\n", elapsed() - t0); // evaluate result by hand. int n_1 = 0, n_10 = 0, n_100 = 0; - for(int i = 0; i < nq; i++) { + for (int i = 0; i < nq; i++) { int gt_nn = gt[i * k]; - for(int j = 0; j < k; j++) { + for (int j = 0; j < k; j++) { if (I[i * k + j] == gt_nn) { - if(j < 1) n_1++; - if(j < 10) n_10++; - if(j < 100) n_100++; + if (j < 1) + n_1++; + if (j < 10) + n_10++; + if (j < 100) + n_100++; } } } @@ -344,21 +340,20 @@ void test_ann_hdf5(const std::string& ann_test_name, printf("R@100 = %.4f\n", n_100 / float(nq)); #endif - printf ("[%.3f s] Search test done\n\n", elapsed() - t0); + printf("[%.3f s] Search test done\n\n", elapsed() - t0); - delete [] I; - delete [] D; + delete[] I; + delete[] D; } - delete [] xq; - delete [] gt; + delete[] xq; + delete[] gt; delete index; } #ifdef CUSTOMIZATION -void test_ivfsq8h_gpu(const std::string& ann_test_name, - int32_t index_add_loops, - const std::vector& nprobes){ +void +test_ivfsq8h_gpu(const std::string& ann_test_name, int32_t index_add_loops, const std::vector& nprobes) { double t0 = elapsed(); const std::string ann_file_name = ann_test_name + ".hdf5"; @@ -380,44 +375,43 @@ void test_ivfsq8h_gpu(const std::string& ann_test_name, size_t d; std::string index_file_name = get_index_file_name(ann_test_name, index_key, index_add_loops); - try{ + try { cpu_index = faiss::read_index(index_file_name.c_str()); d = dim; - } - catch (...){ + } catch (...) { printf("Cannot read index file: %s\n", index_file_name.c_str()); - printf ("[%.3f s] Loading train set\n", elapsed() - t0); + printf("[%.3f s] Loading train set\n", elapsed() - t0); - size_t nb; - float *xb = (float*)hdf5_read(ann_file_name.c_str(), "train", H5T_FLOAT, d, nb); + size_t nb; + float* xb = (float*)hdf5_read(ann_file_name.c_str(), "train", H5T_FLOAT, d, nb); assert(d == dim || !"dataset does not have correct dimension"); - printf ("[%.3f s] Preparing index \"%s\" d=%ld\n", elapsed() - t0, index_key.c_str(), d); + printf("[%.3f s] Preparing index \"%s\" d=%ld\n", elapsed() - t0, index_key.c_str(), d); - faiss::Index *ori_index = faiss::index_factory(d, index_key.c_str(), metric_type); + faiss::Index* ori_index = faiss::index_factory(d, index_key.c_str(), metric_type); auto device_index = faiss::gpu::index_cpu_to_gpu(&res, 0, ori_index); - printf ("[%.3f s] Training on %ld vectors\n", elapsed() - t0, nb); + printf("[%.3f s] Training on %ld vectors\n", elapsed() - t0, nb); device_index->train(nb, xb); - printf ("[%.3f s] Loading database\n", elapsed() - t0); + printf("[%.3f s] Loading database\n", elapsed() - t0); for (int i = 0; i < index_add_loops; i++) { - printf ("[%.3f s] Indexing database, size %ld*%ld\n", elapsed() - t0, nb, d); + printf("[%.3f s] Indexing database, size %ld*%ld\n", elapsed() - t0, nb, d); device_index->add(nb, xb); } cpu_index = faiss::gpu::index_gpu_to_cpu(device_index); faiss::write_index(cpu_index, index_file_name.c_str()); - delete []xb; + delete[] xb; } - faiss::IndexIVF *cpu_ivf_index = dynamic_cast(cpu_index); - if(cpu_ivf_index != nullptr) { + faiss::IndexIVF* cpu_ivf_index = dynamic_cast(cpu_index); + if (cpu_ivf_index != nullptr) { cpu_ivf_index->to_readonly(); } @@ -433,9 +427,9 @@ void test_ivfsq8h_gpu(const std::string& ann_test_name, delete index; size_t nq; - float *xq; + float* xq; { - printf ("[%.3f s] Loading queries\n", elapsed() - t0); + printf("[%.3f s] Loading queries\n", elapsed() - t0); size_t d2; xq = (float*)hdf5_read(ann_file_name.c_str(), "test", H5T_FLOAT, d2, nq); @@ -443,42 +437,41 @@ void test_ivfsq8h_gpu(const std::string& ann_test_name, } size_t k; - faiss::Index::idx_t *gt; + faiss::Index::idx_t* gt; { - printf ("[%.3f s] Loading ground truth for %ld queries\n", elapsed() - t0, nq); + printf("[%.3f s] Loading ground truth for %ld queries\n", elapsed() - t0, nq); size_t nq2; - int *gt_int = (int*)hdf5_read(ann_file_name.c_str(), "neighbors", H5T_INTEGER, k, nq2); + int* gt_int = (int*)hdf5_read(ann_file_name.c_str(), "neighbors", H5T_INTEGER, k, nq2); assert(nq2 == nq || !"incorrect nb of ground truth entries"); gt = new faiss::Index::idx_t[k * nq]; for (unsigned long i = 0; i < k * nq; ++i) { gt[i] = gt_int[i]; } - delete []gt_int; + delete[] gt_int; } - for (auto nprobe : nprobes){ - printf ("[%.3f s] Setting parameter configuration 'nprobe=%lu' on index\n", - elapsed() - t0, nprobe); + for (auto nprobe : nprobes) { + printf("[%.3f s] Setting parameter configuration 'nprobe=%lu' on index\n", elapsed() - t0, nprobe); - auto ivf_index = dynamic_cast(cpu_index); + auto ivf_index = dynamic_cast(cpu_index); ivf_index->nprobe = nprobe; auto is_gpu_flat_index = dynamic_cast(ivf_index->quantizer); - if(is_gpu_flat_index == nullptr) { + if (is_gpu_flat_index == nullptr) { delete ivf_index->quantizer; ivf_index->quantizer = index_composition.quantizer; } const size_t NQ = 1000, K = 1000; - long *I = new faiss::Index::idx_t[NQ * K]; - float *D = new float[NQ * K]; + long* I = new faiss::Index::idx_t[NQ * K]; + float* D = new float[NQ * K]; - printf ("\n%s | %s-gpu | nprobe=%lu\n", ann_test_name.c_str(), index_key.c_str(), nprobe); - printf ("============================================================================================\n"); - for (size_t t_nq = 10; t_nq <= NQ; t_nq *= 10) { // nq = {10, 100, 1000} - for (size_t t_k = 100; t_k <= K; t_k *= 10) { // k = {100, 1000} + printf("\n%s | %s-gpu | nprobe=%lu\n", ann_test_name.c_str(), index_key.c_str(), nprobe); + printf("============================================================================================\n"); + for (size_t t_nq = 10; t_nq <= NQ; t_nq *= 10) { // nq = {10, 100, 1000} + for (size_t t_k = 100; t_k <= K; t_k *= 10) { // k = {100, 1000} faiss::indexIVF_stats.quantization_time = 0.0; faiss::indexIVF_stats.search_time = 0.0; @@ -495,7 +488,7 @@ void test_ivfsq8h_gpu(const std::string& ann_test_name, // consider: each result replicates DATA_LOOPS times for (unsigned long j_c = 0; j_c < k; j_c++) { int r_c = I[i * t_k + j_c]; - for (unsigned long j_g = 0; j_g < k/index_add_loops; j_g++) { + for (unsigned long j_g = 0; j_g < k / index_add_loops; j_g++) { if (gt[i * k + j_g] == r_c) { hit++; continue; @@ -503,23 +496,21 @@ void test_ivfsq8h_gpu(const std::string& ann_test_name, } } } - printf("nq = %4ld, k = %4ld, elapse = %fs (quant = %fs, search = %fs), R@ = %.4f\n", - t_nq, t_k, (t_end - t_start), - faiss::indexIVF_stats.quantization_time / 1000, - faiss::indexIVF_stats.search_time / 1000, - (hit / float(t_nq * k / index_add_loops))); + printf("nq = %4ld, k = %4ld, elapse = %fs (quant = %fs, search = %fs), R@ = %.4f\n", t_nq, t_k, + (t_end - t_start), faiss::indexIVF_stats.quantization_time / 1000, + faiss::indexIVF_stats.search_time / 1000, (hit / float(t_nq * k / index_add_loops))); } } - printf ("============================================================================================\n"); + printf("============================================================================================\n"); - printf ("[%.3f s] Search test done\n\n", elapsed() - t0); + printf("[%.3f s] Search test done\n\n", elapsed() - t0); - delete [] I; - delete [] D; + delete[] I; + delete[] D; } - delete [] xq; - delete [] gt; + delete[] xq; + delete[] gt; delete cpu_index; } #endif @@ -536,21 +527,20 @@ void test_ivfsq8h_gpu(const std::string& ann_test_name, * MNIST 784 60,000 10,000 100 Euclidean HDF5 (217MB) * NYTimes 256 290,000 10,000 100 Angular HDF5 (301MB) * SIFT 128 1,000,000 10,000 100 Euclidean HDF5 (501MB) -*************************************************************************************/ + *************************************************************************************/ TEST(FAISSTEST, BENCHMARK) { - test_ann_hdf5("sift-128-euclidean", "IVF4096,Flat", 2, {8, 128}); - test_ann_hdf5("sift-128-euclidean", "IVF16384,SQ8", 2, {8, 128}); + test_ann_hdf5("sift-128-euclidean", "IVF4096,Flat", 2, {8, 128}); + test_ann_hdf5("sift-128-euclidean", "IVF16384,SQ8", 2, {8, 128}); #ifdef CUSTOMIZATION test_ann_hdf5("sift-128-euclidean", "IVF16384,SQ8Hybrid", 2, {8, 128}); test_ivfsq8h_gpu("sift-128-euclidean", 2, {8, 128}); #endif - test_ann_hdf5("glove-200-angular", "IVF4096,Flat", 1, {8, 128}); - test_ann_hdf5("glove-200-angular", "IVF16384,SQ8", 1, {8, 128}); + test_ann_hdf5("glove-200-angular", "IVF4096,Flat", 1, {8, 128}); + test_ann_hdf5("glove-200-angular", "IVF16384,SQ8", 1, {8, 128}); #ifdef CUSTOMIZATION test_ann_hdf5("glove-200-angular", "IVF16384,SQ8Hybrid", 1, {8, 128}); test_ivfsq8h_gpu("glove-200-angular", 1, {8, 128}); #endif } - From 4e5b778a3d79e2fd95251ddaf03f82152342e4d4 Mon Sep 17 00:00:00 2001 From: jielinxu <52057195+jielinxu@users.noreply.github.com> Date: Mon, 28 Oct 2019 11:19:09 +0800 Subject: [PATCH 14/31] [skip ci] Add Contributors section Former-commit-id: 6c3a3fa7b3556bf904eee77225730f3d9a1befba --- README.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/README.md b/README.md index f18328f748..43f117ff92 100644 --- a/README.md +++ b/README.md @@ -178,6 +178,13 @@ We use [GitHub issues](https://github.com/milvus-io/milvus/issues/new/choose) to To connect with other users and contributors, welcome to join our [slack channel](https://join.slack.com/t/milvusio/shared_invite/enQtNzY1OTQ0NDI3NjMzLWNmYmM1NmNjOTQ5MGI5NDhhYmRhMGU5M2NhNzhhMDMzY2MzNDdlYjM5ODQ5MmE3ODFlYzU3YjJkNmVlNDQ2ZTk). +## Contributors + +Deep thanks and appreciation go to the following people. + +- [akihoni](https://github.com/akihoni) found a broken link and a small typo in the README file. + + ## Milvus Roadmap Please read our [roadmap](https://milvus.io/docs/en/roadmap/) to learn about upcoming features. From a10f83c69c87f72a745aafc033f291f2740badf6 Mon Sep 17 00:00:00 2001 From: jielinxu <52057195+jielinxu@users.noreply.github.com> Date: Mon, 28 Oct 2019 17:12:51 +0800 Subject: [PATCH 15/31] [skip ci] Typo change Former-commit-id: 0438b3ee8770228d39d0146e35b154a88381bf1f --- README.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 43f117ff92..9ff0d4a3ae 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,7 @@ ## What is Milvus -Milvus is an open source similarity search engine for massive feature vectors. Designed with heterogeneous computing architecture for the best cost efficiency. Searches over billion-scale vectors take only milliseconds with minimum computing resources. +Milvus is an open source similarity search engine for massive-scale feature vectors. Built with heterogeneous computing architecture for the best cost efficiency. Searches over billion-scale vectors take only milliseconds with minimum computing resources. Milvus provides stable Python, Java and C++ APIs. @@ -28,7 +28,7 @@ Keep up-to-date with newest releases and latest updates by reading Milvus [relea - Heterogeneous computing - Milvus is designed with heterogeneous computing architecture for the best performance and cost efficiency. + Milvus is built with heterogeneous computing architecture for the best performance and cost efficiency. - Multiple indexes @@ -64,14 +64,14 @@ Keep up-to-date with newest releases and latest updates by reading Milvus [relea ## Get started -### Hardware Requirements +### Hardware requirements | Component | Recommended configuration | | --------- | ----------------------------------- | | CPU | Intel CPU Haswell or higher | | GPU | NVIDIA Pascal series or higher | -| Memory | 8 GB or more (depends on data size) | -| Storage | SATA 3.0 SSD or higher | +| RAM | 8 GB or more (depends on data size) | +| Hard drive| SATA 3.0 SSD or higher | ### Install using docker @@ -185,7 +185,7 @@ Deep thanks and appreciation go to the following people. - [akihoni](https://github.com/akihoni) found a broken link and a small typo in the README file. -## Milvus Roadmap +## Milvus roadmap Please read our [roadmap](https://milvus.io/docs/en/roadmap/) to learn about upcoming features. From baeff23c6bba1840ec9c62e1493a71e0bc53a5d0 Mon Sep 17 00:00:00 2001 From: jielinxu <52057195+jielinxu@users.noreply.github.com> Date: Mon, 28 Oct 2019 17:32:49 +0800 Subject: [PATCH 16/31] [skip ci] minor change Former-commit-id: 5b3a820dc9d017c5dfcb8aa60ed7d3d92d5ef114 --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 9ff0d4a3ae..3dfd95d2f4 100644 --- a/README.md +++ b/README.md @@ -180,7 +180,7 @@ To connect with other users and contributors, welcome to join our [slack channel ## Contributors -Deep thanks and appreciation go to the following people. +We greatly appreciate the help and contributions of the following people. - [akihoni](https://github.com/akihoni) found a broken link and a small typo in the README file. From a1d5ff06df232b9fdab143cedeac02505b177272 Mon Sep 17 00:00:00 2001 From: jielinxu <52057195+jielinxu@users.noreply.github.com> Date: Mon, 28 Oct 2019 17:48:00 +0800 Subject: [PATCH 17/31] [skip ci] minor change Former-commit-id: 10f0bf48d869fc193e33c1065f43d11c76560594 --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 3dfd95d2f4..5c5065442f 100644 --- a/README.md +++ b/README.md @@ -178,9 +178,9 @@ We use [GitHub issues](https://github.com/milvus-io/milvus/issues/new/choose) to To connect with other users and contributors, welcome to join our [slack channel](https://join.slack.com/t/milvusio/shared_invite/enQtNzY1OTQ0NDI3NjMzLWNmYmM1NmNjOTQ5MGI5NDhhYmRhMGU5M2NhNzhhMDMzY2MzNDdlYjM5ODQ5MmE3ODFlYzU3YjJkNmVlNDQ2ZTk). -## Contributors +## Thanks -We greatly appreciate the help and contributions of the following people. +We greatly appreciate the help of the following people. - [akihoni](https://github.com/akihoni) found a broken link and a small typo in the README file. From 7f6092b6baf8d529e762f508fbac9feff80b77b4 Mon Sep 17 00:00:00 2001 From: "yudong.cai" Date: Mon, 28 Oct 2019 19:34:38 +0800 Subject: [PATCH 18/31] #89 add SQ8Hybrid MIX test and pure-GPU test Former-commit-id: 25b5f419d3deedd4ec39c014ea47822a83a0af38 --- .../faiss_benchmark/faiss_benchmark_test.cpp | 228 +++++++++++------- 1 file changed, 143 insertions(+), 85 deletions(-) diff --git a/core/src/index/unittest/faiss_benchmark/faiss_benchmark_test.cpp b/core/src/index/unittest/faiss_benchmark/faiss_benchmark_test.cpp index d1db0e9049..bb50198f92 100644 --- a/core/src/index/unittest/faiss_benchmark/faiss_benchmark_test.cpp +++ b/core/src/index/unittest/faiss_benchmark/faiss_benchmark_test.cpp @@ -28,6 +28,7 @@ #include #include #include +#include #include #include #include @@ -183,9 +184,31 @@ parse_ann_test_name(const std::string& ann_test_name, size_t& dim, faiss::Metric return true; } +int32_t +GetResultHitCount(const faiss::Index::idx_t* ground_index, const faiss::Index::idx_t* index, size_t ground_k, size_t k, + size_t nq, int32_t index_add_loops) { + assert(ground_k <= k); + int hit = 0; + for (int i = 0; i < nq; i++) { + // count the num of results exist in ground truth result set + // each result replicates INDEX_ADD_LOOPS times + for (int j_c = 0; j_c < ground_k; j_c++) { + int r_c = index[i * k + j_c]; + int j_g = 0; + for (; j_g < ground_k / index_add_loops; j_g++) { + if (ground_index[i * ground_k + j_g] == r_c) { + hit++; + continue; + } + } + } + } + return hit; +} + void test_ann_hdf5(const std::string& ann_test_name, const std::string& index_key, int32_t index_add_loops, - const std::vector& nprobes) { + const std::vector& nprobes, int32_t search_loops) { double t0 = elapsed(); const std::string ann_file_name = ann_test_name + ".hdf5"; @@ -265,8 +288,6 @@ test_ann_hdf5(const std::string& ann_test_name, const std::string& index_key, in for (auto nprobe : nprobes) { faiss::ParameterSpace params; - printf("[%.3f s] Setting parameter configuration 'nprobe=%lu' on index\n", elapsed() - t0, nprobe); - std::string nprobe_str = "nprobe=" + std::to_string(nprobe); params.set_index_parameters(index, nprobe_str.c_str()); @@ -277,39 +298,28 @@ test_ann_hdf5(const std::string& ann_test_name, const std::string& index_key, in float* D = new float[NQ * K]; printf("\n%s | %s | nprobe=%lu\n", ann_test_name.c_str(), index_key.c_str(), nprobe); - printf("============================================================================================\n"); + printf("======================================================================================\n"); for (size_t t_nq = 10; t_nq <= NQ; t_nq *= 10) { // nq = {10, 100, 1000} for (size_t t_k = 100; t_k <= K; t_k *= 10) { // k = {100, 1000} faiss::indexIVF_stats.quantization_time = 0.0; faiss::indexIVF_stats.search_time = 0.0; double t_start = elapsed(), t_end; - - index->search(t_nq, xq, t_k, D, I); - + for (int i = 0; i < search_loops; i++) { + index->search(t_nq, xq, t_k, D, I); + } t_end = elapsed(); // k = 100 for ground truth - int hit = 0; - for (int i = 0; i < t_nq; i++) { - // count the num of results exist in ground truth result set - // consider: each result replicates DATA_LOOPS times - for (int j_c = 0; j_c < k; j_c++) { - int r_c = I[i * t_k + j_c]; - for (int j_g = 0; j_g < k / index_add_loops; j_g++) { - if (gt[i * k + j_g] == r_c) { - hit++; - continue; - } - } - } - } - printf("nq = %4ld, k = %4ld, elapse = %fs (quant = %fs, search = %fs), R@ = %.4f\n", t_nq, t_k, - (t_end - t_start), faiss::indexIVF_stats.quantization_time / 1000, - faiss::indexIVF_stats.search_time / 1000, (hit / float(t_nq * k / index_add_loops))); + int32_t hit = GetResultHitCount(gt, I, k, t_k, t_nq, index_add_loops); + + printf("nq = %4ld, k = %4ld, elapse = %.4fs (quant = %.4fs, search = %.4fs), R@ = %.4f\n", t_nq, t_k, + (t_end - t_start) / search_loops, faiss::indexIVF_stats.quantization_time / 1000 / search_loops, + faiss::indexIVF_stats.search_time / 1000 / search_loops, + (hit / float(t_nq * k / index_add_loops))); } } - printf("============================================================================================\n"); + printf("======================================================================================\n"); #else printf("[%.3f s] Perform a search on %ld queries\n", elapsed() - t0, nq); @@ -353,7 +363,8 @@ test_ann_hdf5(const std::string& ann_test_name, const std::string& index_key, in #ifdef CUSTOMIZATION void -test_ivfsq8h_gpu(const std::string& ann_test_name, int32_t index_add_loops, const std::vector& nprobes) { +test_ivfsq8h(const std::string& ann_test_name, int32_t index_add_loops, const std::vector& nprobes, + bool pure_gpu_mode, int32_t search_loops) { double t0 = elapsed(); const std::string ann_file_name = ann_test_name + ".hdf5"; @@ -423,9 +434,18 @@ test_ivfsq8h_gpu(const std::string& ann_test_name, int32_t index_add_loops, cons index_composition.quantizer = nullptr; index_composition.mode = 1; + double copy_time = elapsed(); auto index = faiss::gpu::index_cpu_to_gpu(&res, 0, &index_composition, &option); delete index; + if (pure_gpu_mode) { + index_composition.mode = 2; // 0: all data, 1: copy quantizer, 2: copy data + index = faiss::gpu::index_cpu_to_gpu(&res, 0, &index_composition, &option); + } + + copy_time = elapsed() - copy_time; + printf("[%.3f s] Copy quantizer completed, cost %f s\n", elapsed() - t0, copy_time); + size_t nq; float* xq; { @@ -446,67 +466,98 @@ test_ivfsq8h_gpu(const std::string& ann_test_name, int32_t index_add_loops, cons assert(nq2 == nq || !"incorrect nb of ground truth entries"); gt = new faiss::Index::idx_t[k * nq]; - for (unsigned long i = 0; i < k * nq; ++i) { + for (uint64_t i = 0; i < k * nq; ++i) { gt[i] = gt_int[i]; } delete[] gt_int; } - for (auto nprobe : nprobes) { - printf("[%.3f s] Setting parameter configuration 'nprobe=%lu' on index\n", elapsed() - t0, nprobe); + const size_t NQ = 1000, K = 1000; + if (!pure_gpu_mode) { + for (auto nprobe : nprobes) { + auto ivf_index = dynamic_cast(cpu_index); + ivf_index->nprobe = nprobe; - auto ivf_index = dynamic_cast(cpu_index); - ivf_index->nprobe = nprobe; - - auto is_gpu_flat_index = dynamic_cast(ivf_index->quantizer); - if (is_gpu_flat_index == nullptr) { - delete ivf_index->quantizer; - ivf_index->quantizer = index_composition.quantizer; - } - - const size_t NQ = 1000, K = 1000; - long* I = new faiss::Index::idx_t[NQ * K]; - float* D = new float[NQ * K]; - - printf("\n%s | %s-gpu | nprobe=%lu\n", ann_test_name.c_str(), index_key.c_str(), nprobe); - printf("============================================================================================\n"); - for (size_t t_nq = 10; t_nq <= NQ; t_nq *= 10) { // nq = {10, 100, 1000} - for (size_t t_k = 100; t_k <= K; t_k *= 10) { // k = {100, 1000} - faiss::indexIVF_stats.quantization_time = 0.0; - faiss::indexIVF_stats.search_time = 0.0; - - double t_start = elapsed(), t_end; - - cpu_index->search(t_nq, xq, t_k, D, I); - - t_end = elapsed(); - - // k = 100 for ground truth - int hit = 0; - for (unsigned long i = 0; i < t_nq; i++) { - // count the num of results exist in ground truth result set - // consider: each result replicates DATA_LOOPS times - for (unsigned long j_c = 0; j_c < k; j_c++) { - int r_c = I[i * t_k + j_c]; - for (unsigned long j_g = 0; j_g < k / index_add_loops; j_g++) { - if (gt[i * k + j_g] == r_c) { - hit++; - continue; - } - } - } - } - printf("nq = %4ld, k = %4ld, elapse = %fs (quant = %fs, search = %fs), R@ = %.4f\n", t_nq, t_k, - (t_end - t_start), faiss::indexIVF_stats.quantization_time / 1000, - faiss::indexIVF_stats.search_time / 1000, (hit / float(t_nq * k / index_add_loops))); + auto is_gpu_flat_index = dynamic_cast(ivf_index->quantizer); + if (is_gpu_flat_index == nullptr) { + delete ivf_index->quantizer; + ivf_index->quantizer = index_composition.quantizer; } + + int64_t* I = new faiss::Index::idx_t[NQ * K]; + float* D = new float[NQ * K]; + + printf("\n%s | %s-MIX | nprobe=%lu\n", ann_test_name.c_str(), index_key.c_str(), nprobe); + printf("======================================================================================\n"); + for (size_t t_nq = 10; t_nq <= NQ; t_nq *= 10) { // nq = {10, 100, 1000} + for (size_t t_k = 100; t_k <= K; t_k *= 10) { // k = {100, 1000} + faiss::indexIVF_stats.quantization_time = 0.0; + faiss::indexIVF_stats.search_time = 0.0; + + double t_start = elapsed(), t_end; + for (int32_t i = 0; i < search_loops; i++) { + cpu_index->search(t_nq, xq, t_k, D, I); + } + t_end = elapsed(); + + // k = 100 for ground truth + int32_t hit = GetResultHitCount(gt, I, k, t_k, t_nq, index_add_loops); + + printf("nq = %4ld, k = %4ld, elapse = %.4fs (quant = %.4fs, search = %.4fs), R@ = %.4f\n", t_nq, + t_k, (t_end - t_start) / search_loops, + faiss::indexIVF_stats.quantization_time / 1000 / search_loops, + faiss::indexIVF_stats.search_time / 1000 / search_loops, + (hit / float(t_nq * k / index_add_loops))); + } + } + printf("======================================================================================\n"); + + printf("[%.3f s] Search test done\n\n", elapsed() - t0); + + delete[] I; + delete[] D; } - printf("============================================================================================\n"); + } else { + std::shared_ptr gpu_index_ivf_ptr = std::shared_ptr(index); - printf("[%.3f s] Search test done\n\n", elapsed() - t0); + for (auto nprobe : nprobes) { + faiss::gpu::GpuIndexIVFSQHybrid* gpu_index_ivf_hybrid = + dynamic_cast(gpu_index_ivf_ptr.get()); + gpu_index_ivf_hybrid->setNumProbes(nprobe); - delete[] I; - delete[] D; + int64_t* I = new faiss::Index::idx_t[NQ * K]; + float* D = new float[NQ * K]; + + printf("\n%s | %s-GPU | nprobe=%lu\n", ann_test_name.c_str(), index_key.c_str(), nprobe); + printf("======================================================================================\n"); + for (size_t t_nq = 10; t_nq <= NQ; t_nq *= 10) { // nq = {10, 100, 1000} + for (size_t t_k = 100; t_k <= K; t_k *= 10) { // k = {100, 1000} + faiss::indexIVF_stats.quantization_time = 0.0; + faiss::indexIVF_stats.search_time = 0.0; + + double t_start = elapsed(), t_end; + for (int32_t i = 0; i < search_loops; i++) { + gpu_index_ivf_ptr->search(nq, xq, k, D, I); + } + t_end = elapsed(); + + // k = 100 for ground truth + int32_t hit = GetResultHitCount(gt, I, k, t_k, t_nq, index_add_loops); + + printf("nq = %4ld, k = %4ld, elapse = %.4fs (quant = %.4fs, search = %.4fs), R@ = %.4f\n", t_nq, + t_k, (t_end - t_start) / search_loops, + faiss::indexIVF_stats.quantization_time / 1000 / search_loops, + faiss::indexIVF_stats.search_time / 1000 / search_loops, + (hit / float(t_nq * k / index_add_loops))); + } + } + printf("======================================================================================\n"); + + printf("[%.3f s] Search test done\n\n", elapsed() - t0); + + delete[] I; + delete[] D; + } } delete[] xq; @@ -530,17 +581,24 @@ test_ivfsq8h_gpu(const std::string& ann_test_name, int32_t index_add_loops, cons *************************************************************************************/ TEST(FAISSTEST, BENCHMARK) { - test_ann_hdf5("sift-128-euclidean", "IVF4096,Flat", 2, {8, 128}); - test_ann_hdf5("sift-128-euclidean", "IVF16384,SQ8", 2, {8, 128}); + std::vector param_nprobes = {8, 128}; + const int32_t SEARCH_LOOPS = 5; + const int32_t SIFT_INSERT_LOOPS = 2; // insert twice to get ~1G data set + const int32_t GLOVE_INSERT_LOOPS = 1; + + test_ann_hdf5("sift-128-euclidean", "IVF4096,Flat", SIFT_INSERT_LOOPS, param_nprobes, SEARCH_LOOPS); + test_ann_hdf5("sift-128-euclidean", "IVF16384,SQ8", SIFT_INSERT_LOOPS, param_nprobes, SEARCH_LOOPS); #ifdef CUSTOMIZATION - test_ann_hdf5("sift-128-euclidean", "IVF16384,SQ8Hybrid", 2, {8, 128}); - test_ivfsq8h_gpu("sift-128-euclidean", 2, {8, 128}); + test_ann_hdf5("sift-128-euclidean", "IVF16384,SQ8Hybrid", SIFT_INSERT_LOOPS, param_nprobes, SEARCH_LOOPS); + test_ivfsq8h("sift-128-euclidean", SIFT_INSERT_LOOPS, param_nprobes, false, SEARCH_LOOPS); + test_ivfsq8h("sift-128-euclidean", SIFT_INSERT_LOOPS, param_nprobes, true, SEARCH_LOOPS); #endif - test_ann_hdf5("glove-200-angular", "IVF4096,Flat", 1, {8, 128}); - test_ann_hdf5("glove-200-angular", "IVF16384,SQ8", 1, {8, 128}); + test_ann_hdf5("glove-200-angular", "IVF4096,Flat", GLOVE_INSERT_LOOPS, param_nprobes, SEARCH_LOOPS); + test_ann_hdf5("glove-200-angular", "IVF16384,SQ8", GLOVE_INSERT_LOOPS, param_nprobes, SEARCH_LOOPS); #ifdef CUSTOMIZATION - test_ann_hdf5("glove-200-angular", "IVF16384,SQ8Hybrid", 1, {8, 128}); - test_ivfsq8h_gpu("glove-200-angular", 1, {8, 128}); + test_ann_hdf5("glove-200-angular", "IVF16384,SQ8Hybrid", GLOVE_INSERT_LOOPS, param_nprobes, SEARCH_LOOPS); + test_ivfsq8h("glove-200-angular", GLOVE_INSERT_LOOPS, param_nprobes, false, SEARCH_LOOPS); + test_ivfsq8h("glove-200-angular", GLOVE_INSERT_LOOPS, param_nprobes, true, SEARCH_LOOPS); #endif } From 97ae8a780ca0009e7f1749a339f50cb954cc4fa2 Mon Sep 17 00:00:00 2001 From: "yudong.cai" Date: Mon, 28 Oct 2019 19:55:43 +0800 Subject: [PATCH 19/31] #89 add README.txt Former-commit-id: a84501ce6c2c94275819ace9e7d7a4afc14fbeca --- core/src/index/unittest/faiss_benchmark/README.txt | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 core/src/index/unittest/faiss_benchmark/README.txt diff --git a/core/src/index/unittest/faiss_benchmark/README.txt b/core/src/index/unittest/faiss_benchmark/README.txt new file mode 100644 index 0000000000..81114d8381 --- /dev/null +++ b/core/src/index/unittest/faiss_benchmark/README.txt @@ -0,0 +1,13 @@ +To run this FAISS benchmark, please follow these steps: + +1. Download the HDF5 from: + https://support.hdfgroup.org/ftp/HDF5/releases/ + and install to /usr/local/hdf5 + +2. Download HDF5 data files from: + https://github.com/erikbern/ann-benchmarks + +3. Put HDF5 data files into the same directory with test binary + +4. Run the test binary + From b439da8a36dcfa2f63867cb70786859fe992dcf3 Mon Sep 17 00:00:00 2001 From: "yudong.cai" Date: Mon, 28 Oct 2019 20:17:25 +0800 Subject: [PATCH 20/31] #89 update README.md Former-commit-id: 0b85b430c0d4a69e0470e916a76adda16f96c12b --- .../index/unittest/faiss_benchmark/README.md | 25 +++++++++++++++++++ .../index/unittest/faiss_benchmark/README.txt | 13 ---------- 2 files changed, 25 insertions(+), 13 deletions(-) create mode 100644 core/src/index/unittest/faiss_benchmark/README.md delete mode 100644 core/src/index/unittest/faiss_benchmark/README.txt diff --git a/core/src/index/unittest/faiss_benchmark/README.md b/core/src/index/unittest/faiss_benchmark/README.md new file mode 100644 index 0000000000..c451ac13b0 --- /dev/null +++ b/core/src/index/unittest/faiss_benchmark/README.md @@ -0,0 +1,25 @@ +### To run this FAISS benchmark, please follow these steps: + +#### Step 1: +Download the HDF5 source from: + https://support.hdfgroup.org/ftp/HDF5/releases/ +and build/install to "/usr/local/hdf5". + +#### Step 2: +Download HDF5 data files from: + https://github.com/erikbern/ann-benchmarks + +#### Step 3: +Update 'milvus/core/src/index/unittest/CMakeLists.txt', +uncomment "#add_subdirectory(faiss_benchmark)". + +#### Step 4: +Build Milvus with unittest enabled: "./build.sh -t Release -u", +binary 'test_faiss_benchmark' will be generated. + +#### Step 5: +Put HDF5 data files into the same directory with binary 'test_faiss_benchmark'. + +#### Step 6: +Run test binary 'test_faiss_benchmark'. + diff --git a/core/src/index/unittest/faiss_benchmark/README.txt b/core/src/index/unittest/faiss_benchmark/README.txt deleted file mode 100644 index 81114d8381..0000000000 --- a/core/src/index/unittest/faiss_benchmark/README.txt +++ /dev/null @@ -1,13 +0,0 @@ -To run this FAISS benchmark, please follow these steps: - -1. Download the HDF5 from: - https://support.hdfgroup.org/ftp/HDF5/releases/ - and install to /usr/local/hdf5 - -2. Download HDF5 data files from: - https://github.com/erikbern/ann-benchmarks - -3. Put HDF5 data files into the same directory with test binary - -4. Run the test binary - From 12b7d6f5d8011ec9b60e3f6184467f11873ec9e7 Mon Sep 17 00:00:00 2001 From: starlord Date: Mon, 28 Oct 2019 20:24:13 +0800 Subject: [PATCH 21/31] #90 The server start error messages could be improved to enhance user experience Former-commit-id: 746b126621cd845c2848bb850b5c2eb6d3b65a6d --- CHANGELOG.md | 1 + core/src/server/Config.cpp | 134 +++++++++++++++++++++++--------- core/src/server/DBWrapper.cpp | 47 +++++++---- core/unittest/db/utils.cpp | 2 +- core/unittest/wrapper/utils.cpp | 2 +- 5 files changed, 133 insertions(+), 53 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0925fa1a68..c4643c0adc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ Please mark all change in change log and use the ticket from JIRA. # Milvus 0.5.1 (TODO) ## Bug +- \#90 - The server start error messages could be improved to enhance user experience - \#104 - test_scheduler core dump ## Improvement diff --git a/core/src/server/Config.cpp b/core/src/server/Config.cpp index 7de84cbccc..b20d6c2436 100644 --- a/core/src/server/Config.cpp +++ b/core/src/server/Config.cpp @@ -363,7 +363,9 @@ Config::PrintAll() { Status Config::CheckServerConfigAddress(const std::string& value) { if (!ValidationUtil::ValidateIpAddress(value).ok()) { - return Status(SERVER_INVALID_ARGUMENT, "Invalid server config address: " + value); + std::string msg = "Invalid server IP address: " + value + + ". Possible reason: server_config.address is invalid in server_config.yaml."; + return Status(SERVER_INVALID_ARGUMENT, msg); } return Status::OK(); } @@ -371,11 +373,15 @@ Config::CheckServerConfigAddress(const std::string& value) { Status Config::CheckServerConfigPort(const std::string& value) { if (!ValidationUtil::ValidateStringIsNumber(value).ok()) { - return Status(SERVER_INVALID_ARGUMENT, "Invalid server config port: " + value); + std::string msg = "Port " + value + " is not a number. " + + "Possible reason: server_config.port in server_config.yaml is invalid."; + return Status(SERVER_INVALID_ARGUMENT, msg); } else { int32_t port = std::stoi(value); if (!(port > 1024 && port < 65535)) { - return Status(SERVER_INVALID_ARGUMENT, "Server config port out of range (1024, 65535): " + value); + std::string msg = "Port " + value + " is not in range [1025, 65534]. " + + "Possible reason: server_config.port in server_config.yaml is invalid."; + return Status(SERVER_INVALID_ARGUMENT, msg); } } return Status::OK(); @@ -385,7 +391,8 @@ Status Config::CheckServerConfigDeployMode(const std::string& value) { if (value != "single" && value != "cluster_readonly" && value != "cluster_writable") { return Status(SERVER_INVALID_ARGUMENT, - "Invalid server config mode [single, cluster_readonly, cluster_writable]: " + value); + "Error: server_config.deploy_mode in server_config.yaml is not one of " + "single, cluster_readonly, and cluster_writable."); } return Status::OK(); } @@ -411,7 +418,8 @@ Config::CheckServerConfigTimeZone(const std::string& value) { Status Config::CheckDBConfigPrimaryPath(const std::string& value) { if (value.empty()) { - return Status(SERVER_INVALID_ARGUMENT, "DB config primary_path empty"); + return Status(SERVER_INVALID_ARGUMENT, + "db_path is empty. Possible reason: db_config.db_path in server_config.yaml is empty."); } return Status::OK(); } @@ -424,7 +432,11 @@ Config::CheckDBConfigSecondaryPath(const std::string& value) { Status Config::CheckDBConfigBackendUrl(const std::string& value) { if (!ValidationUtil::ValidateDbURI(value).ok()) { - return Status(SERVER_INVALID_ARGUMENT, "Invalid DB config backend_url: " + value); + std::string msg = + "Invalid db_backend_url: " + value + + ". Possible reason: db_config.db_backend_url is invalid in server_config.yaml. " + + "The correct format should be like sqlite://:@:/ or mysql://root:123456@127.0.0.1:3306/milvus."; + return Status(SERVER_INVALID_ARGUMENT, "invalid db_backend_url: " + value); } return Status::OK(); } @@ -432,7 +444,9 @@ Config::CheckDBConfigBackendUrl(const std::string& value) { Status Config::CheckDBConfigArchiveDiskThreshold(const std::string& value) { if (!ValidationUtil::ValidateStringIsNumber(value).ok()) { - return Status(SERVER_INVALID_ARGUMENT, "Invalid DB config archive_disk_threshold: " + value); + std::string msg = "Invalid archive disk threshold: " + value + + "Possible reason: db_config.archive_disk_threshold in server_config.yaml is invalid."; + return Status(SERVER_INVALID_ARGUMENT, msg); } return Status::OK(); } @@ -440,7 +454,9 @@ Config::CheckDBConfigArchiveDiskThreshold(const std::string& value) { Status Config::CheckDBConfigArchiveDaysThreshold(const std::string& value) { if (!ValidationUtil::ValidateStringIsNumber(value).ok()) { - return Status(SERVER_INVALID_ARGUMENT, "Invalid DB config archive_days_threshold: " + value); + std::string msg = "Invalid archive days threshold: " + value + + "Possible reason: db_config.archive_disk_threshold in server_config.yaml is invalid."; + return Status(SERVER_INVALID_ARGUMENT, msg); } return Status::OK(); } @@ -448,13 +464,17 @@ Config::CheckDBConfigArchiveDaysThreshold(const std::string& value) { Status Config::CheckDBConfigInsertBufferSize(const std::string& value) { if (!ValidationUtil::ValidateStringIsNumber(value).ok()) { - return Status(SERVER_INVALID_ARGUMENT, "Invalid DB config insert_buffer_size: " + value); + std::string msg = "Invalid insert buffer size: " + value + + "Possible reason: db_config.insert_buffer_size in server_config.yaml is invalid."; + return Status(SERVER_INVALID_ARGUMENT, msg); } else { int64_t buffer_size = std::stoi(value) * GB; uint64_t total_mem = 0, free_mem = 0; CommonUtil::GetSystemMemInfo(total_mem, free_mem); if (buffer_size >= total_mem) { - return Status(SERVER_INVALID_ARGUMENT, "DB config insert_buffer_size exceed system memory: " + value); + std::string msg = "Invalid insert buffer size: " + value + + "Possible reason: insert buffer size exceed system memory."; + return Status(SERVER_INVALID_ARGUMENT, msg); } } return Status::OK(); @@ -463,7 +483,9 @@ Config::CheckDBConfigInsertBufferSize(const std::string& value) { Status Config::CheckMetricConfigEnableMonitor(const std::string& value) { if (!ValidationUtil::ValidateStringIsBool(value).ok()) { - return Status(SERVER_INVALID_ARGUMENT, "Invalid metric config auto_bootup: " + value); + std::string msg = "Invalid metric config: " + value + + "Possible reason: metric_config.enable_monitor is invalid."; + return Status(SERVER_INVALID_ARGUMENT, msg); } return Status::OK(); } @@ -471,7 +493,9 @@ Config::CheckMetricConfigEnableMonitor(const std::string& value) { Status Config::CheckMetricConfigCollector(const std::string& value) { if (value != "prometheus") { - return Status(SERVER_INVALID_ARGUMENT, "Invalid metric config collector: " + value); + std::string msg = "Invalid metric config: " + value + + "Possible reason: metric_config.collector is invalid."; + return Status(SERVER_INVALID_ARGUMENT, msg); } return Status::OK(); } @@ -479,6 +503,8 @@ Config::CheckMetricConfigCollector(const std::string& value) { Status Config::CheckMetricConfigPrometheusPort(const std::string& value) { if (!ValidationUtil::ValidateStringIsNumber(value).ok()) { + std::string msg = "Invalid metric config: " + value + + "Possible reason: metric_config.prometheus_config.port is invalid."; return Status(SERVER_INVALID_ARGUMENT, "Invalid metric config prometheus_port: " + value); } return Status::OK(); @@ -487,15 +513,19 @@ Config::CheckMetricConfigPrometheusPort(const std::string& value) { Status Config::CheckCacheConfigCpuCacheCapacity(const std::string& value) { if (!ValidationUtil::ValidateStringIsNumber(value).ok()) { - return Status(SERVER_INVALID_ARGUMENT, "Invalid cache config cpu_cache_capacity: " + value); + std::string msg = "Invalid cpu cache capacity: " + value + + "Possible reason: cache_config.cpu_cache_capacity is invalid."; + return Status(SERVER_INVALID_ARGUMENT, msg); } else { uint64_t cpu_cache_capacity = std::stoi(value) * GB; uint64_t total_mem = 0, free_mem = 0; CommonUtil::GetSystemMemInfo(total_mem, free_mem); if (cpu_cache_capacity >= total_mem) { - return Status(SERVER_INVALID_ARGUMENT, "Cache config cpu_cache_capacity exceed system memory: " + value); + std::string msg = "Invalid cpu cache capacity: " + value + + "Possible reason: Cache config cpu_cache_capacity exceed system memory."; + return Status(SERVER_INVALID_ARGUMENT, msg); } else if (cpu_cache_capacity > static_cast(total_mem * 0.9)) { - std::cerr << "Warning: cpu_cache_capacity value is too big" << std::endl; + std::cerr << "WARNING: cpu cache capacity value is too big" << std::endl; } int32_t buffer_value; @@ -506,7 +536,10 @@ Config::CheckCacheConfigCpuCacheCapacity(const std::string& value) { int64_t insert_buffer_size = buffer_value * GB; if (insert_buffer_size + cpu_cache_capacity >= total_mem) { - return Status(SERVER_INVALID_ARGUMENT, "Sum of cpu_cache_capacity and buffer_size exceed system memory"); + std::string msg = "Invalid cpu cache capacity: " + value + + "Possible reason: sum of cache_config.cpu_cache_capacity and " + "db_config.insert_buffer_size exceeds system memory."; + return Status(SERVER_INVALID_ARGUMENT, msg); } } return Status::OK(); @@ -515,11 +548,15 @@ Config::CheckCacheConfigCpuCacheCapacity(const std::string& value) { Status Config::CheckCacheConfigCpuCacheThreshold(const std::string& value) { if (!ValidationUtil::ValidateStringIsFloat(value).ok()) { - return Status(SERVER_INVALID_ARGUMENT, "Invalid cache config cpu_cache_threshold: " + value); + std::string msg = "Invalid cpu cache threshold: " + value + + "Possible reason: cache_config.cpu_cache_threshold is invalid."; + return Status(SERVER_INVALID_ARGUMENT, msg); } else { float cpu_cache_threshold = std::stof(value); if (cpu_cache_threshold <= 0.0 || cpu_cache_threshold >= 1.0) { - return Status(SERVER_INVALID_ARGUMENT, "Invalid cache config cpu_cache_threshold: " + value); + std::string msg = "Invalid cpu cache threshold: " + value + + "Possible reason: cache_config.cpu_cache_threshold is invalid."; + return Status(SERVER_INVALID_ARGUMENT, msg); } } return Status::OK(); @@ -528,7 +565,9 @@ Config::CheckCacheConfigCpuCacheThreshold(const std::string& value) { Status Config::CheckCacheConfigGpuCacheCapacity(const std::string& value) { if (!ValidationUtil::ValidateStringIsNumber(value).ok()) { - return Status(SERVER_INVALID_ARGUMENT, "Invalid cache config gpu_cache_capacity: " + value); + std::string msg = "Invalid gpu cache capacity: " + value + + "Possible reason: cache_config.gpu_cache_capacity is invalid."; + return Status(SERVER_INVALID_ARGUMENT, msg); } else { uint64_t gpu_cache_capacity = std::stoi(value) * GB; int gpu_index; @@ -539,13 +578,14 @@ Config::CheckCacheConfigGpuCacheCapacity(const std::string& value) { size_t gpu_memory; if (!ValidationUtil::GetGpuMemory(gpu_index, gpu_memory).ok()) { - return Status(SERVER_UNEXPECTED_ERROR, - "Fail to get GPU memory for GPU device: " + std::to_string(gpu_index)); + std::string msg = "Fail to get GPU memory for GPU device: " + std::to_string(gpu_index); + return Status(SERVER_UNEXPECTED_ERROR, msg); } else if (gpu_cache_capacity >= gpu_memory) { - return Status(SERVER_INVALID_ARGUMENT, - "Cache config gpu_cache_capacity exceed GPU memory: " + std::to_string(gpu_memory)); + std::string msg = "Invalid gpu cache capacity: " + value + + "Possible reason: cache_config.gpu_cache_capacity exceed GPU memory."; + return Status(SERVER_INVALID_ARGUMENT, msg); } else if (gpu_cache_capacity > (double)gpu_memory * 0.9) { - std::cerr << "Warning: gpu_cache_capacity value is too big" << std::endl; + std::cerr << "Warning: gpu cache capacity value is too big" << std::endl; } } return Status::OK(); @@ -554,11 +594,15 @@ Config::CheckCacheConfigGpuCacheCapacity(const std::string& value) { Status Config::CheckCacheConfigGpuCacheThreshold(const std::string& value) { if (!ValidationUtil::ValidateStringIsFloat(value).ok()) { - return Status(SERVER_INVALID_ARGUMENT, "Invalid cache config gpu_cache_threshold: " + value); + std::string msg = "Invalid gpu cache threshold: " + value + + "Possible reason: cache_config.gpu_cache_threshold is invalid."; + return Status(SERVER_INVALID_ARGUMENT, msg); } else { float gpu_cache_threshold = std::stof(value); if (gpu_cache_threshold <= 0.0 || gpu_cache_threshold >= 1.0) { - return Status(SERVER_INVALID_ARGUMENT, "Invalid cache config gpu_cache_threshold: " + value); + std::string msg = "Invalid gpu cache threshold: " + value + + "Possible reason: cache_config.gpu_cache_threshold is invalid."; + return Status(SERVER_INVALID_ARGUMENT, msg); } } return Status::OK(); @@ -567,7 +611,9 @@ Config::CheckCacheConfigGpuCacheThreshold(const std::string& value) { Status Config::CheckCacheConfigCacheInsertData(const std::string& value) { if (!ValidationUtil::ValidateStringIsBool(value).ok()) { - return Status(SERVER_INVALID_ARGUMENT, "Invalid cache config cache_insert_data: " + value); + std::string msg = "Invalid cache insert option: " + value + + "Possible reason: cache_config.cache_insert_data is invalid."; + return Status(SERVER_INVALID_ARGUMENT, msg); } return Status::OK(); } @@ -575,7 +621,9 @@ Config::CheckCacheConfigCacheInsertData(const std::string& value) { Status Config::CheckEngineConfigUseBlasThreshold(const std::string& value) { if (!ValidationUtil::ValidateStringIsNumber(value).ok()) { - return Status(SERVER_INVALID_ARGUMENT, "Invalid engine config use_blas_threshold: " + value); + std::string msg = "Invalid blas threshold: " + value + + "Possible reason: engine_config.use_blas_threshold is invalid."; + return Status(SERVER_INVALID_ARGUMENT, msg); } return Status::OK(); } @@ -583,14 +631,18 @@ Config::CheckEngineConfigUseBlasThreshold(const std::string& value) { Status Config::CheckEngineConfigOmpThreadNum(const std::string& value) { if (!ValidationUtil::ValidateStringIsNumber(value).ok()) { - return Status(SERVER_INVALID_ARGUMENT, "Invalid engine config omp_thread_num: " + value); + std::string msg = "Invalid omp thread number: " + value + + "Possible reason: engine_config.omp_thread_num is invalid."; + return Status(SERVER_INVALID_ARGUMENT, msg); } int32_t omp_thread = std::stoi(value); uint32_t sys_thread_cnt = 8; CommonUtil::GetSystemAvailableThreads(sys_thread_cnt); if (omp_thread > static_cast(sys_thread_cnt)) { - return Status(SERVER_INVALID_ARGUMENT, "Invalid engine config omp_thread_num: " + value); + std::string msg = "Invalid omp thread number: " + value + + "Possible reason: engine_config.omp_thread_num is invalid."; + return Status(SERVER_INVALID_ARGUMENT, msg); } return Status::OK(); } @@ -598,7 +650,9 @@ Config::CheckEngineConfigOmpThreadNum(const std::string& value) { Status Config::CheckResourceConfigMode(const std::string& value) { if (value != "simple") { - return Status(SERVER_INVALID_ARGUMENT, "Invalid resource config mode: " + value); + std::string msg = "Invalid resource mode: " + value + + "Possible reason: resource_config.mode is invalid."; + return Status(SERVER_INVALID_ARGUMENT, msg); } return Status::OK(); } @@ -608,12 +662,16 @@ CheckGpuDevice(const std::string& value) { const std::regex pat("gpu(\\d+)"); std::cmatch m; if (!std::regex_match(value.c_str(), m, pat)) { - return Status(SERVER_INVALID_ARGUMENT, "Invalid gpu device: " + value); + std::string msg = "Invalid gpu device: " + value + + "Possible reason: resource_config.search_resources is invalid."; + return Status(SERVER_INVALID_ARGUMENT, msg); } int32_t gpu_index = std::stoi(value.substr(3)); if (!ValidationUtil::ValidateGpuIndex(gpu_index).ok()) { - return Status(SERVER_INVALID_ARGUMENT, "Invalid gpu device: " + value); + std::string msg = "Invalid gpu device: " + value + + "Possible reason: resource_config.search_resources is invalid."; + return Status(SERVER_INVALID_ARGUMENT, msg); } return Status::OK(); } @@ -621,11 +679,15 @@ CheckGpuDevice(const std::string& value) { Status Config::CheckResourceConfigSearchResources(const std::vector& value) { if (value.empty()) { - return Status(SERVER_INVALID_ARGUMENT, "Empty resource config search_resources"); + std::string msg = "Invalid search resource. " + "Possible reason: resource_config.search_resources is empty."; + return Status(SERVER_INVALID_ARGUMENT, msg); } for (auto& gpu_device : value) { if (!CheckGpuDevice(gpu_device).ok()) { + std::string msg = "Invalid search resource: " + gpu_device + + "Possible reason: resource_config.search_resources is invalid."; return Status(SERVER_INVALID_ARGUMENT, "Invalid resource config search_resources: " + gpu_device); } } @@ -635,7 +697,9 @@ Config::CheckResourceConfigSearchResources(const std::vector& value Status Config::CheckResourceConfigIndexBuildDevice(const std::string& value) { if (!CheckGpuDevice(value).ok()) { - return Status(SERVER_INVALID_ARGUMENT, "Invalid resource config index_build_device: " + value); + std::string msg = "Invalid index build device: " + value + + "Possible reason: resource_config.index_build_device is invalid."; + return Status(SERVER_INVALID_ARGUMENT, msg); } return Status::OK(); } diff --git a/core/src/server/DBWrapper.cpp b/core/src/server/DBWrapper.cpp index a5b892ad47..2217b29e1c 100644 --- a/core/src/server/DBWrapper.cpp +++ b/core/src/server/DBWrapper.cpp @@ -54,7 +54,8 @@ DBWrapper::StartService() { std::string db_slave_path; s = config.GetDBConfigSecondaryPath(db_slave_path); if (!s.ok()) { - return s; + std::cerr << s.ToString() << std::endl; + kill(0, SIGUSR1); } StringHelpFunctions::SplitStringByDelimeter(db_slave_path, ";", opt.meta_.slave_paths_); @@ -62,13 +63,15 @@ DBWrapper::StartService() { // cache config s = config.GetCacheConfigCacheInsertData(opt.insert_cache_immediately_); if (!s.ok()) { - return s; + std::cerr << s.ToString() << std::endl; + kill(0, SIGUSR1); } std::string mode; s = config.GetServerConfigDeployMode(mode); if (!s.ok()) { - return s; + std::cerr << s.ToString() << std::endl; + kill(0, SIGUSR1); } if (mode == "single") { @@ -78,7 +81,8 @@ DBWrapper::StartService() { } else if (mode == "cluster_writable") { opt.mode_ = engine::DBOptions::MODE::CLUSTER_WRITABLE; } else { - std::cerr << "ERROR: mode specified in server_config must be ['single', 'cluster_readonly', 'cluster_writable']" + std::cerr << "Error: server_config.deploy_mode in server_config.yaml is not one of " + << "single, cluster_readonly, and cluster_writable." << std::endl; kill(0, SIGUSR1); } @@ -87,7 +91,8 @@ DBWrapper::StartService() { int32_t omp_thread; s = config.GetEngineConfigOmpThreadNum(omp_thread); if (!s.ok()) { - return s; + std::cerr << s.ToString() << std::endl; + kill(0, SIGUSR1); } if (omp_thread > 0) { @@ -105,7 +110,8 @@ DBWrapper::StartService() { int32_t use_blas_threshold; s = config.GetEngineConfigUseBlasThreshold(use_blas_threshold); if (!s.ok()) { - return s; + std::cerr << s.ToString() << std::endl; + kill(0, SIGUSR1); } faiss::distance_compute_blas_threshold = use_blas_threshold; @@ -115,7 +121,8 @@ DBWrapper::StartService() { int32_t disk, days; s = config.GetDBConfigArchiveDiskThreshold(disk); if (!s.ok()) { - return s; + std::cerr << s.ToString() << std::endl; + kill(0, SIGUSR1); } if (disk > 0) { @@ -124,7 +131,8 @@ DBWrapper::StartService() { s = config.GetDBConfigArchiveDaysThreshold(days); if (!s.ok()) { - return s; + std::cerr << s.ToString() << std::endl; + kill(0, SIGUSR1); } if (days > 0) { @@ -133,16 +141,20 @@ DBWrapper::StartService() { opt.meta_.archive_conf_.SetCriterias(criterial); // create db root folder - Status status = CommonUtil::CreateDirectory(opt.meta_.path_); - if (!status.ok()) { - std::cerr << "ERROR! Failed to create database root path: " << opt.meta_.path_ << std::endl; + s = CommonUtil::CreateDirectory(opt.meta_.path_); + if (!s.ok()) { + std::cerr << "Error: Failed to create database primary path: " << path + << ". Possible reason: db_config.primary_path is wrong in server_config.yaml or not available." + << std::endl; kill(0, SIGUSR1); } for (auto& path : opt.meta_.slave_paths_) { - status = CommonUtil::CreateDirectory(path); - if (!status.ok()) { - std::cerr << "ERROR! Failed to create database slave path: " << path << std::endl; + s = CommonUtil::CreateDirectory(path); + if (!s.ok()) { + std::cerr << "Error: Failed to create database secondary path: " << path + << ". Possible reason: db_config.secondary_path is wrong in server_config.yaml or not available." + << std::endl; kill(0, SIGUSR1); } } @@ -151,7 +163,9 @@ DBWrapper::StartService() { try { db_ = engine::DBFactory::Build(opt); } catch (std::exception& ex) { - std::cerr << "ERROR! Failed to open database: " << ex.what() << std::endl; + std::cerr << "Error: failed to open database: " << ex.what() + << ". Possible reason: the meta system does not work." + << std::endl; kill(0, SIGUSR1); } @@ -161,7 +175,8 @@ DBWrapper::StartService() { std::string preload_tables; s = config.GetDBConfigPreloadTable(preload_tables); if (!s.ok()) { - return s; + std::cerr << s.ToString() << std::endl; + kill(0, SIGUSR1); } s = PreloadTables(preload_tables); diff --git a/core/unittest/db/utils.cpp b/core/unittest/db/utils.cpp index 8903ce14ea..7cc2f28745 100644 --- a/core/unittest/db/utils.cpp +++ b/core/unittest/db/utils.cpp @@ -68,7 +68,7 @@ static const char " blas_threshold: 20\n" "\n" "resource_config:\n" - " resource_pool:\n" + " search_resources:\n" " - gpu0\n" " index_build_device: gpu0 # GPU used for building index"; diff --git a/core/unittest/wrapper/utils.cpp b/core/unittest/wrapper/utils.cpp index 6204ac0c05..b397a35d7c 100644 --- a/core/unittest/wrapper/utils.cpp +++ b/core/unittest/wrapper/utils.cpp @@ -58,7 +58,7 @@ static const char " blas_threshold: 20\n" "\n" "resource_config:\n" - " resource_pool:\n" + " search_resources:\n" " - gpu0\n" " index_build_device: gpu0 # GPU used for building index"; From d95b3906c8afdc47d34d1ba43d045967993320a2 Mon Sep 17 00:00:00 2001 From: Heisenberg Date: Mon, 28 Oct 2019 20:41:08 +0800 Subject: [PATCH 22/31] [skip ci] refine the copy time test Former-commit-id: d370cec5e350d3f3740d9b56182ad0a990ed2ec8 --- .../faiss_benchmark/faiss_benchmark_test.cpp | 50 +++++++++++-------- 1 file changed, 30 insertions(+), 20 deletions(-) diff --git a/core/src/index/unittest/faiss_benchmark/faiss_benchmark_test.cpp b/core/src/index/unittest/faiss_benchmark/faiss_benchmark_test.cpp index bb50198f92..e80b85e024 100644 --- a/core/src/index/unittest/faiss_benchmark/faiss_benchmark_test.cpp +++ b/core/src/index/unittest/faiss_benchmark/faiss_benchmark_test.cpp @@ -426,26 +426,6 @@ test_ivfsq8h(const std::string& ann_test_name, int32_t index_add_loops, const st cpu_ivf_index->to_readonly(); } - faiss::gpu::GpuClonerOptions option; - option.allInGpu = true; - - faiss::IndexComposition index_composition; - index_composition.index = cpu_index; - index_composition.quantizer = nullptr; - index_composition.mode = 1; - - double copy_time = elapsed(); - auto index = faiss::gpu::index_cpu_to_gpu(&res, 0, &index_composition, &option); - delete index; - - if (pure_gpu_mode) { - index_composition.mode = 2; // 0: all data, 1: copy quantizer, 2: copy data - index = faiss::gpu::index_cpu_to_gpu(&res, 0, &index_composition, &option); - } - - copy_time = elapsed() - copy_time; - printf("[%.3f s] Copy quantizer completed, cost %f s\n", elapsed() - t0, copy_time); - size_t nq; float* xq; { @@ -472,6 +452,36 @@ test_ivfsq8h(const std::string& ann_test_name, int32_t index_add_loops, const st delete[] gt_int; } + faiss::gpu::GpuClonerOptions option; + option.allInGpu = true; + + faiss::IndexComposition index_composition; + index_composition.index = cpu_index; + index_composition.quantizer = nullptr; + + faiss::Index* index; + double copy_time; + + if (!pure_gpu_mode) { + index_composition.mode = 1; // 0: all data, 1: copy quantizer, 2: copy data + index = faiss::gpu::index_cpu_to_gpu(&res, 0, &index_composition, &option); + delete index; + + copy_time = elapsed(); + index = faiss::gpu::index_cpu_to_gpu(&res, 0, &index_composition, &option); + delete index; + } else { + index_composition.mode = 2; + index = faiss::gpu::index_cpu_to_gpu(&res, 0, &index_composition, &option); + delete index; + + copy_time = elapsed(); + index = faiss::gpu::index_cpu_to_gpu(&res, 0, &index_composition, &option); + } + + copy_time = elapsed() - copy_time; + printf("[%.3f s] Copy quantizer completed, cost %f s\n", elapsed() - t0, copy_time); + const size_t NQ = 1000, K = 1000; if (!pure_gpu_mode) { for (auto nprobe : nprobes) { From d109a3778c4849548298921833390acc18a1b680 Mon Sep 17 00:00:00 2001 From: jielinxu <52057195+jielinxu@users.noreply.github.com> Date: Tue, 29 Oct 2019 09:32:36 +0800 Subject: [PATCH 23/31] [skip ci] Move Roadmap section ahead Former-commit-id: c2a065f0192c4bc6ab948d8c4c972847bfcb771b --- README.md | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 5c5065442f..884ddb01ca 100644 --- a/README.md +++ b/README.md @@ -168,6 +168,10 @@ Make sure Java 8 or higher is already installed. Refer to [this link](https://github.com/milvus-io/milvus-sdk-java/tree/master/examples) for the example code. +## Milvus roadmap + +Please read our [roadmap](https://milvus.io/docs/en/roadmap/) to learn about upcoming features. + ## Contribution guidelines Contributions are welcomed and greatly appreciated. Please read our [contribution guidelines](CONTRIBUTING.md) for detailed contribution workflow. This project adheres to the [code of conduct](CODE_OF_CONDUCT.md) of Milvus. By participating, you are expected to uphold this code. @@ -184,11 +188,6 @@ We greatly appreciate the help of the following people. - [akihoni](https://github.com/akihoni) found a broken link and a small typo in the README file. - -## Milvus roadmap - -Please read our [roadmap](https://milvus.io/docs/en/roadmap/) to learn about upcoming features. - ## Resources [Milvus official website](https://www.milvus.io) @@ -203,7 +202,6 @@ Please read our [roadmap](https://milvus.io/docs/en/roadmap/) to learn about upc [Milvus roadmap](https://milvus.io/docs/en/roadmap/) - ## License [Apache License 2.0](LICENSE) From a78e928dfaa389d2c6b02ae1dc7cc60bc0a712f0 Mon Sep 17 00:00:00 2001 From: starlord Date: Tue, 29 Oct 2019 15:16:04 +0800 Subject: [PATCH 24/31] #90 The server start error messages could be improved to enhance user experience Former-commit-id: e3ad89ab67f2c46bc4cb4e6d094a7763fb098664 --- core/conf/server_config.template | 20 ++--- core/src/server/Config.cpp | 145 +++++++++++++++++-------------- core/src/server/DBWrapper.cpp | 6 +- 3 files changed, 91 insertions(+), 80 deletions(-) diff --git a/core/conf/server_config.template b/core/conf/server_config.template index 7abfb8b055..3b366f1bd4 100644 --- a/core/conf/server_config.template +++ b/core/conf/server_config.template @@ -4,7 +4,7 @@ server_config: address: 0.0.0.0 # milvus server ip address (IPv4) port: 19530 # port range: 1025 ~ 65534 deploy_mode: single # deployment type: single, cluster_readonly, cluster_writable - time_zone: UTC+8 + time_zone: UTC+8 # time zone, must be in format: UTC+X db_config: primary_path: @MILVUS_DB_PATH@ # path used to store data and meta @@ -14,30 +14,30 @@ db_config: # Keep 'dialect://:@:/', and replace other texts with real values # Replace 'dialect' with 'mysql' or 'sqlite' - insert_buffer_size: 4 # GB, maximum insert buffer size allowed + insert_buffer_size: 4 # GB, maximum insert buffer size allowed, must be a positive integer # sum of insert_buffer_size and cpu_cache_capacity cannot exceed total memory preload_table: # preload data at startup, '*' means load all tables, empty value means no preload # you can specify preload tables like this: table1,table2,table3 metric_config: - enable_monitor: false # enable monitoring or not + enable_monitor: false # enable monitoring or not, must be a boolean collector: prometheus # prometheus prometheus_config: - port: 8080 # port prometheus uses to fetch metrics + port: 8080 # port prometheus uses to fetch metrics, range: 1025 ~ 65534 cache_config: - cpu_cache_capacity: 16 # GB, CPU memory used for cache - cpu_cache_threshold: 0.85 # percentage of data that will be kept when cache cleanup is triggered - gpu_cache_capacity: 4 # GB, GPU memory used for cache - gpu_cache_threshold: 0.85 # percentage of data that will be kept when cache cleanup is triggered - cache_insert_data: false # whether to load inserted data into cache + cpu_cache_capacity: 16 # GB, CPU memory used for cache, must be a positive integer + cpu_cache_threshold: 0.85 # percentage of data that will be kept when cache cleanup is triggered, must be in range (0.0, 1.0] + gpu_cache_capacity: 4 # GB, GPU memory used for cache, must be a positive integer + gpu_cache_threshold: 0.85 # percentage of data that will be kept when cache cleanup is triggered, must be in range (0.0, 1.0] + cache_insert_data: false # whether to load inserted data into cache, must be a boolean engine_config: use_blas_threshold: 20 # if nq < use_blas_threshold, use SSE, faster with fluctuated response times # if nq >= use_blas_threshold, use OpenBlas, slower with stable response times resource_config: - search_resources: # define the GPUs used for search computation, valid value: gpux + search_resources: # define the GPUs used for search computation, must be in format: gpux - gpu0 index_build_device: gpu0 # GPU used for building index \ No newline at end of file diff --git a/core/src/server/Config.cpp b/core/src/server/Config.cpp index b20d6c2436..51449cb1de 100644 --- a/core/src/server/Config.cpp +++ b/core/src/server/Config.cpp @@ -363,8 +363,8 @@ Config::PrintAll() { Status Config::CheckServerConfigAddress(const std::string& value) { if (!ValidationUtil::ValidateIpAddress(value).ok()) { - std::string msg = "Invalid server IP address: " + value - + ". Possible reason: server_config.address is invalid in server_config.yaml."; + std::string msg = "Invalid server IP address: " + value + + ". Possible reason: server_config.address is invalid in server_config.yaml."; return Status(SERVER_INVALID_ARGUMENT, msg); } return Status::OK(); @@ -373,14 +373,14 @@ Config::CheckServerConfigAddress(const std::string& value) { Status Config::CheckServerConfigPort(const std::string& value) { if (!ValidationUtil::ValidateStringIsNumber(value).ok()) { - std::string msg = "Port " + value + " is not a number. " - + "Possible reason: server_config.port in server_config.yaml is invalid."; + std::string msg = "Port " + value + " is not a number. " + + "Possible reason: server_config.port in server_config.yaml is invalid."; return Status(SERVER_INVALID_ARGUMENT, msg); } else { int32_t port = std::stoi(value); if (!(port > 1024 && port < 65535)) { - std::string msg = "Port " + value + " is not in range [1025, 65534]. " - + "Possible reason: server_config.port in server_config.yaml is invalid."; + std::string msg = "Port " + value + " is not in range [1025, 65534]. " + + "Possible reason: server_config.port in server_config.yaml is invalid."; return Status(SERVER_INVALID_ARGUMENT, msg); } } @@ -433,9 +433,9 @@ Status Config::CheckDBConfigBackendUrl(const std::string& value) { if (!ValidationUtil::ValidateDbURI(value).ok()) { std::string msg = - "Invalid db_backend_url: " + value - + ". Possible reason: db_config.db_backend_url is invalid in server_config.yaml. " - + "The correct format should be like sqlite://:@:/ or mysql://root:123456@127.0.0.1:3306/milvus."; + "Invalid db_backend_url: " + value + + ". Possible reason: db_config.db_backend_url is invalid in server_config.yaml. " + + "The correct format should be like sqlite://:@:/ or mysql://root:123456@127.0.0.1:3306/milvus."; return Status(SERVER_INVALID_ARGUMENT, "invalid db_backend_url: " + value); } return Status::OK(); @@ -444,8 +444,8 @@ Config::CheckDBConfigBackendUrl(const std::string& value) { Status Config::CheckDBConfigArchiveDiskThreshold(const std::string& value) { if (!ValidationUtil::ValidateStringIsNumber(value).ok()) { - std::string msg = "Invalid archive disk threshold: " + value - + "Possible reason: db_config.archive_disk_threshold in server_config.yaml is invalid."; + std::string msg = "Invalid archive disk threshold: " + value + + "Possible reason: db_config.archive_disk_threshold in server_config.yaml is invalid."; return Status(SERVER_INVALID_ARGUMENT, msg); } return Status::OK(); @@ -454,8 +454,8 @@ Config::CheckDBConfigArchiveDiskThreshold(const std::string& value) { Status Config::CheckDBConfigArchiveDaysThreshold(const std::string& value) { if (!ValidationUtil::ValidateStringIsNumber(value).ok()) { - std::string msg = "Invalid archive days threshold: " + value - + "Possible reason: db_config.archive_disk_threshold in server_config.yaml is invalid."; + std::string msg = "Invalid archive days threshold: " + value + + "Possible reason: db_config.archive_disk_threshold in server_config.yaml is invalid."; return Status(SERVER_INVALID_ARGUMENT, msg); } return Status::OK(); @@ -464,16 +464,24 @@ Config::CheckDBConfigArchiveDaysThreshold(const std::string& value) { Status Config::CheckDBConfigInsertBufferSize(const std::string& value) { if (!ValidationUtil::ValidateStringIsNumber(value).ok()) { - std::string msg = "Invalid insert buffer size: " + value - + "Possible reason: db_config.insert_buffer_size in server_config.yaml is invalid."; + std::string msg = "Invalid insert buffer size: " + value + + "Possible reason: db_config.insert_buffer_size in server_config.yaml " + "is not a positive integer."; return Status(SERVER_INVALID_ARGUMENT, msg); } else { int64_t buffer_size = std::stoi(value) * GB; + if (buffer_size <= 0) { + std::string msg = "Invalid insert buffer size: " + value + + "Possible reason: db_config.insert_buffer_size in server_config.yaml " + "is not a positive integer."; + return Status(SERVER_INVALID_ARGUMENT, msg); + } + uint64_t total_mem = 0, free_mem = 0; CommonUtil::GetSystemMemInfo(total_mem, free_mem); if (buffer_size >= total_mem) { - std::string msg = "Invalid insert buffer size: " + value - + "Possible reason: insert buffer size exceed system memory."; + std::string msg = + "Invalid insert buffer size: " + value + "Possible reason: insert buffer size exceeds system memory."; return Status(SERVER_INVALID_ARGUMENT, msg); } } @@ -483,8 +491,8 @@ Config::CheckDBConfigInsertBufferSize(const std::string& value) { Status Config::CheckMetricConfigEnableMonitor(const std::string& value) { if (!ValidationUtil::ValidateStringIsBool(value).ok()) { - std::string msg = "Invalid metric config: " + value - + "Possible reason: metric_config.enable_monitor is invalid."; + std::string msg = + "Invalid metric config: " + value + "Possible reason: metric_config.enable_monitor is not a boolean."; return Status(SERVER_INVALID_ARGUMENT, msg); } return Status::OK(); @@ -493,8 +501,7 @@ Config::CheckMetricConfigEnableMonitor(const std::string& value) { Status Config::CheckMetricConfigCollector(const std::string& value) { if (value != "prometheus") { - std::string msg = "Invalid metric config: " + value - + "Possible reason: metric_config.collector is invalid."; + std::string msg = "Invalid metric config: " + value + "Possible reason: metric_config.collector is invalid."; return Status(SERVER_INVALID_ARGUMENT, msg); } return Status::OK(); @@ -503,8 +510,8 @@ Config::CheckMetricConfigCollector(const std::string& value) { Status Config::CheckMetricConfigPrometheusPort(const std::string& value) { if (!ValidationUtil::ValidateStringIsNumber(value).ok()) { - std::string msg = "Invalid metric config: " + value - + "Possible reason: metric_config.prometheus_config.port is invalid."; + std::string msg = "Invalid metric config: " + value + + "Possible reason: metric_config.prometheus_config.port is not in range [1025, 65534]."; return Status(SERVER_INVALID_ARGUMENT, "Invalid metric config prometheus_port: " + value); } return Status::OK(); @@ -513,18 +520,24 @@ Config::CheckMetricConfigPrometheusPort(const std::string& value) { Status Config::CheckCacheConfigCpuCacheCapacity(const std::string& value) { if (!ValidationUtil::ValidateStringIsNumber(value).ok()) { - std::string msg = "Invalid cpu cache capacity: " + value - + "Possible reason: cache_config.cpu_cache_capacity is invalid."; + std::string msg = "Invalid cpu cache capacity: " + value + + "Possible reason: cache_config.cpu_cache_capacity is not a positive integer."; return Status(SERVER_INVALID_ARGUMENT, msg); } else { - uint64_t cpu_cache_capacity = std::stoi(value) * GB; + int64_t cpu_cache_capacity = std::stoi(value) * GB; + if (cpu_cache_capacity <= 0) { + std::string msg = "Invalid cpu cache capacity: " + value + + "Possible reason: cache_config.cpu_cache_capacity is not a positive integer."; + return Status(SERVER_INVALID_ARGUMENT, msg); + } + uint64_t total_mem = 0, free_mem = 0; CommonUtil::GetSystemMemInfo(total_mem, free_mem); - if (cpu_cache_capacity >= total_mem) { - std::string msg = "Invalid cpu cache capacity: " + value - + "Possible reason: Cache config cpu_cache_capacity exceed system memory."; + if (static_cast(cpu_cache_capacity) >= total_mem) { + std::string msg = "Invalid cpu cache capacity: " + value + + "Possible reason: Cache config cpu_cache_capacity exceeds system memory."; return Status(SERVER_INVALID_ARGUMENT, msg); - } else if (cpu_cache_capacity > static_cast(total_mem * 0.9)) { + } else if (static_cast(cpu_cache_capacity) > static_cast(total_mem * 0.9)) { std::cerr << "WARNING: cpu cache capacity value is too big" << std::endl; } @@ -536,9 +549,9 @@ Config::CheckCacheConfigCpuCacheCapacity(const std::string& value) { int64_t insert_buffer_size = buffer_value * GB; if (insert_buffer_size + cpu_cache_capacity >= total_mem) { - std::string msg = "Invalid cpu cache capacity: " + value - + "Possible reason: sum of cache_config.cpu_cache_capacity and " - "db_config.insert_buffer_size exceeds system memory."; + std::string msg = "Invalid cpu cache capacity: " + value + + "Possible reason: sum of cache_config.cpu_cache_capacity and " + "db_config.insert_buffer_size exceeds system memory."; return Status(SERVER_INVALID_ARGUMENT, msg); } } @@ -548,14 +561,14 @@ Config::CheckCacheConfigCpuCacheCapacity(const std::string& value) { Status Config::CheckCacheConfigCpuCacheThreshold(const std::string& value) { if (!ValidationUtil::ValidateStringIsFloat(value).ok()) { - std::string msg = "Invalid cpu cache threshold: " + value - + "Possible reason: cache_config.cpu_cache_threshold is invalid."; + std::string msg = "Invalid cpu cache threshold: " + value + + "Possible reason: cache_config.cpu_cache_threshold is not in range (0.0, 1.0]."; return Status(SERVER_INVALID_ARGUMENT, msg); } else { float cpu_cache_threshold = std::stof(value); if (cpu_cache_threshold <= 0.0 || cpu_cache_threshold >= 1.0) { - std::string msg = "Invalid cpu cache threshold: " + value - + "Possible reason: cache_config.cpu_cache_threshold is invalid."; + std::string msg = "Invalid cpu cache threshold: " + value + + "Possible reason: cache_config.cpu_cache_threshold is not in range (0.0, 1.0]."; return Status(SERVER_INVALID_ARGUMENT, msg); } } @@ -565,8 +578,8 @@ Config::CheckCacheConfigCpuCacheThreshold(const std::string& value) { Status Config::CheckCacheConfigGpuCacheCapacity(const std::string& value) { if (!ValidationUtil::ValidateStringIsNumber(value).ok()) { - std::string msg = "Invalid gpu cache capacity: " + value - + "Possible reason: cache_config.gpu_cache_capacity is invalid."; + std::string msg = "Invalid gpu cache capacity: " + value + + "Possible reason: cache_config.gpu_cache_capacity is not a positive integer."; return Status(SERVER_INVALID_ARGUMENT, msg); } else { uint64_t gpu_cache_capacity = std::stoi(value) * GB; @@ -581,8 +594,8 @@ Config::CheckCacheConfigGpuCacheCapacity(const std::string& value) { std::string msg = "Fail to get GPU memory for GPU device: " + std::to_string(gpu_index); return Status(SERVER_UNEXPECTED_ERROR, msg); } else if (gpu_cache_capacity >= gpu_memory) { - std::string msg = "Invalid gpu cache capacity: " + value - + "Possible reason: cache_config.gpu_cache_capacity exceed GPU memory."; + std::string msg = "Invalid gpu cache capacity: " + value + + "Possible reason: cache_config.gpu_cache_capacity exceeds GPU memory."; return Status(SERVER_INVALID_ARGUMENT, msg); } else if (gpu_cache_capacity > (double)gpu_memory * 0.9) { std::cerr << "Warning: gpu cache capacity value is too big" << std::endl; @@ -594,14 +607,14 @@ Config::CheckCacheConfigGpuCacheCapacity(const std::string& value) { Status Config::CheckCacheConfigGpuCacheThreshold(const std::string& value) { if (!ValidationUtil::ValidateStringIsFloat(value).ok()) { - std::string msg = "Invalid gpu cache threshold: " + value - + "Possible reason: cache_config.gpu_cache_threshold is invalid."; + std::string msg = "Invalid gpu cache threshold: " + value + + "Possible reason: cache_config.gpu_cache_threshold is not in range (0.0, 1.0]."; return Status(SERVER_INVALID_ARGUMENT, msg); } else { float gpu_cache_threshold = std::stof(value); if (gpu_cache_threshold <= 0.0 || gpu_cache_threshold >= 1.0) { - std::string msg = "Invalid gpu cache threshold: " + value - + "Possible reason: cache_config.gpu_cache_threshold is invalid."; + std::string msg = "Invalid gpu cache threshold: " + value + + "Possible reason: cache_config.gpu_cache_threshold is not in range (0.0, 1.0]."; return Status(SERVER_INVALID_ARGUMENT, msg); } } @@ -611,8 +624,8 @@ Config::CheckCacheConfigGpuCacheThreshold(const std::string& value) { Status Config::CheckCacheConfigCacheInsertData(const std::string& value) { if (!ValidationUtil::ValidateStringIsBool(value).ok()) { - std::string msg = "Invalid cache insert option: " + value - + "Possible reason: cache_config.cache_insert_data is invalid."; + std::string msg = "Invalid cache insert option: " + value + + "Possible reason: cache_config.cache_insert_data is not a boolean."; return Status(SERVER_INVALID_ARGUMENT, msg); } return Status::OK(); @@ -621,8 +634,8 @@ Config::CheckCacheConfigCacheInsertData(const std::string& value) { Status Config::CheckEngineConfigUseBlasThreshold(const std::string& value) { if (!ValidationUtil::ValidateStringIsNumber(value).ok()) { - std::string msg = "Invalid blas threshold: " + value - + "Possible reason: engine_config.use_blas_threshold is invalid."; + std::string msg = "Invalid blas threshold: " + value + + "Possible reason: engine_config.use_blas_threshold is not a positive integer."; return Status(SERVER_INVALID_ARGUMENT, msg); } return Status::OK(); @@ -631,8 +644,8 @@ Config::CheckEngineConfigUseBlasThreshold(const std::string& value) { Status Config::CheckEngineConfigOmpThreadNum(const std::string& value) { if (!ValidationUtil::ValidateStringIsNumber(value).ok()) { - std::string msg = "Invalid omp thread number: " + value - + "Possible reason: engine_config.omp_thread_num is invalid."; + std::string msg = "Invalid omp thread number: " + value + + "Possible reason: engine_config.omp_thread_num is not a positive integer."; return Status(SERVER_INVALID_ARGUMENT, msg); } @@ -640,8 +653,8 @@ Config::CheckEngineConfigOmpThreadNum(const std::string& value) { uint32_t sys_thread_cnt = 8; CommonUtil::GetSystemAvailableThreads(sys_thread_cnt); if (omp_thread > static_cast(sys_thread_cnt)) { - std::string msg = "Invalid omp thread number: " + value - + "Possible reason: engine_config.omp_thread_num is invalid."; + std::string msg = "Invalid omp thread number: " + value + + "Possible reason: engine_config.omp_thread_num exceeds system cpu cores."; return Status(SERVER_INVALID_ARGUMENT, msg); } return Status::OK(); @@ -650,8 +663,7 @@ Config::CheckEngineConfigOmpThreadNum(const std::string& value) { Status Config::CheckResourceConfigMode(const std::string& value) { if (value != "simple") { - std::string msg = "Invalid resource mode: " + value - + "Possible reason: resource_config.mode is invalid."; + std::string msg = "Invalid resource mode: " + value + "Possible reason: resource_config.mode is invalid."; return Status(SERVER_INVALID_ARGUMENT, msg); } return Status::OK(); @@ -662,15 +674,15 @@ CheckGpuDevice(const std::string& value) { const std::regex pat("gpu(\\d+)"); std::cmatch m; if (!std::regex_match(value.c_str(), m, pat)) { - std::string msg = "Invalid gpu device: " + value - + "Possible reason: resource_config.search_resources is invalid."; + std::string msg = "Invalid gpu device: " + value + + "Possible reason: resource_config.search_resources does not match your hardware."; return Status(SERVER_INVALID_ARGUMENT, msg); } int32_t gpu_index = std::stoi(value.substr(3)); if (!ValidationUtil::ValidateGpuIndex(gpu_index).ok()) { - std::string msg = "Invalid gpu device: " + value - + "Possible reason: resource_config.search_resources is invalid."; + std::string msg = "Invalid gpu device: " + value + + "Possible reason: resource_config.search_resources does not match your hardware."; return Status(SERVER_INVALID_ARGUMENT, msg); } return Status::OK(); @@ -679,16 +691,17 @@ CheckGpuDevice(const std::string& value) { Status Config::CheckResourceConfigSearchResources(const std::vector& value) { if (value.empty()) { - std::string msg = "Invalid search resource. " - "Possible reason: resource_config.search_resources is empty."; + std::string msg = + "Invalid search resource. " + "Possible reason: resource_config.search_resources is empty."; return Status(SERVER_INVALID_ARGUMENT, msg); } for (auto& gpu_device : value) { if (!CheckGpuDevice(gpu_device).ok()) { - std::string msg = "Invalid search resource: " + gpu_device - + "Possible reason: resource_config.search_resources is invalid."; - return Status(SERVER_INVALID_ARGUMENT, "Invalid resource config search_resources: " + gpu_device); + std::string msg = "Invalid search resource: " + gpu_device + + "Possible reason: resource_config.search_resources does not match your hardware."; + return Status(SERVER_INVALID_ARGUMENT, msg); } } return Status::OK(); @@ -697,8 +710,8 @@ Config::CheckResourceConfigSearchResources(const std::vector& value Status Config::CheckResourceConfigIndexBuildDevice(const std::string& value) { if (!CheckGpuDevice(value).ok()) { - std::string msg = "Invalid index build device: " + value - + "Possible reason: resource_config.index_build_device is invalid."; + std::string msg = "Invalid index build device: " + value + + "Possible reason: resource_config.index_build_device does not match your hardware."; return Status(SERVER_INVALID_ARGUMENT, msg); } return Status::OK(); diff --git a/core/src/server/DBWrapper.cpp b/core/src/server/DBWrapper.cpp index 2217b29e1c..7efb71075a 100644 --- a/core/src/server/DBWrapper.cpp +++ b/core/src/server/DBWrapper.cpp @@ -82,8 +82,7 @@ DBWrapper::StartService() { opt.mode_ = engine::DBOptions::MODE::CLUSTER_WRITABLE; } else { std::cerr << "Error: server_config.deploy_mode in server_config.yaml is not one of " - << "single, cluster_readonly, and cluster_writable." - << std::endl; + << "single, cluster_readonly, and cluster_writable." << std::endl; kill(0, SIGUSR1); } @@ -164,8 +163,7 @@ DBWrapper::StartService() { db_ = engine::DBFactory::Build(opt); } catch (std::exception& ex) { std::cerr << "Error: failed to open database: " << ex.what() - << ". Possible reason: the meta system does not work." - << std::endl; + << ". Possible reason: the meta system does not work." << std::endl; kill(0, SIGUSR1); } From a05cebf5133538650fcbee0601bcc1bdebdfdfa9 Mon Sep 17 00:00:00 2001 From: starlord Date: Tue, 29 Oct 2019 15:18:54 +0800 Subject: [PATCH 25/31] fix job.h header cpplint error Former-commit-id: c5bfb2f7acdfad00adf818b9cc4b20ce42b7c9e1 --- core/src/scheduler/job/Job.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/scheduler/job/Job.cpp b/core/src/scheduler/job/Job.cpp index 1199fe17a6..06a163b959 100644 --- a/core/src/scheduler/job/Job.cpp +++ b/core/src/scheduler/job/Job.cpp @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -#include "Job.h" +#include "scheduler/job/Job.h" namespace milvus { namespace scheduler { From 96dffcf8ab7219b20621a64efe62ffbf2f7996e1 Mon Sep 17 00:00:00 2001 From: starlord Date: Tue, 29 Oct 2019 15:37:08 +0800 Subject: [PATCH 26/31] format code Former-commit-id: f809ffd4505ada713620e4996d6fe1004bcc69dc --- core/src/server/DBWrapper.cpp | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/core/src/server/DBWrapper.cpp b/core/src/server/DBWrapper.cpp index 7efb71075a..e3d319ac53 100644 --- a/core/src/server/DBWrapper.cpp +++ b/core/src/server/DBWrapper.cpp @@ -40,12 +40,14 @@ DBWrapper::StartService() { engine::DBOptions opt; s = config.GetDBConfigBackendUrl(opt.meta_.backend_uri_); if (!s.ok()) { + std::cerr << s.ToString() << std::endl; return s; } std::string path; s = config.GetDBConfigPrimaryPath(path); if (!s.ok()) { + std::cerr << s.ToString() << std::endl; return s; } @@ -55,7 +57,7 @@ DBWrapper::StartService() { s = config.GetDBConfigSecondaryPath(db_slave_path); if (!s.ok()) { std::cerr << s.ToString() << std::endl; - kill(0, SIGUSR1); + return s; } StringHelpFunctions::SplitStringByDelimeter(db_slave_path, ";", opt.meta_.slave_paths_); @@ -64,14 +66,14 @@ DBWrapper::StartService() { s = config.GetCacheConfigCacheInsertData(opt.insert_cache_immediately_); if (!s.ok()) { std::cerr << s.ToString() << std::endl; - kill(0, SIGUSR1); + return s; } std::string mode; s = config.GetServerConfigDeployMode(mode); if (!s.ok()) { std::cerr << s.ToString() << std::endl; - kill(0, SIGUSR1); + return s; } if (mode == "single") { @@ -91,7 +93,7 @@ DBWrapper::StartService() { s = config.GetEngineConfigOmpThreadNum(omp_thread); if (!s.ok()) { std::cerr << s.ToString() << std::endl; - kill(0, SIGUSR1); + return s; } if (omp_thread > 0) { @@ -110,7 +112,7 @@ DBWrapper::StartService() { s = config.GetEngineConfigUseBlasThreshold(use_blas_threshold); if (!s.ok()) { std::cerr << s.ToString() << std::endl; - kill(0, SIGUSR1); + return s; } faiss::distance_compute_blas_threshold = use_blas_threshold; @@ -121,7 +123,7 @@ DBWrapper::StartService() { s = config.GetDBConfigArchiveDiskThreshold(disk); if (!s.ok()) { std::cerr << s.ToString() << std::endl; - kill(0, SIGUSR1); + return s; } if (disk > 0) { @@ -131,7 +133,7 @@ DBWrapper::StartService() { s = config.GetDBConfigArchiveDaysThreshold(days); if (!s.ok()) { std::cerr << s.ToString() << std::endl; - kill(0, SIGUSR1); + return s; } if (days > 0) { @@ -174,7 +176,7 @@ DBWrapper::StartService() { s = config.GetDBConfigPreloadTable(preload_tables); if (!s.ok()) { std::cerr << s.ToString() << std::endl; - kill(0, SIGUSR1); + return s; } s = PreloadTables(preload_tables); From c63c622cf5c4bcec43a33dacfa2b68518a5a1ddc Mon Sep 17 00:00:00 2001 From: starlord Date: Tue, 29 Oct 2019 16:06:27 +0800 Subject: [PATCH 27/31] update message Former-commit-id: 568dbdc5d6dc233716682ebec33e6bd3dfe45500 --- core/conf/server_config.template | 6 +++--- core/src/server/Config.cpp | 29 ++++++++++++----------------- 2 files changed, 15 insertions(+), 20 deletions(-) diff --git a/core/conf/server_config.template b/core/conf/server_config.template index 3b366f1bd4..3cb899d1b9 100644 --- a/core/conf/server_config.template +++ b/core/conf/server_config.template @@ -2,7 +2,7 @@ server_config: address: 0.0.0.0 # milvus server ip address (IPv4) - port: 19530 # port range: 1025 ~ 65534 + port: 19530 # milvus server port, must in range [1025, 6553] deploy_mode: single # deployment type: single, cluster_readonly, cluster_writable time_zone: UTC+8 # time zone, must be in format: UTC+X @@ -24,7 +24,7 @@ metric_config: enable_monitor: false # enable monitoring or not, must be a boolean collector: prometheus # prometheus prometheus_config: - port: 8080 # port prometheus uses to fetch metrics, range: 1025 ~ 65534 + port: 8080 # port prometheus uses to fetch metrics, must in range [1025, 6553] cache_config: cpu_cache_capacity: 16 # GB, CPU memory used for cache, must be a positive integer @@ -40,4 +40,4 @@ engine_config: resource_config: search_resources: # define the GPUs used for search computation, must be in format: gpux - gpu0 - index_build_device: gpu0 # GPU used for building index \ No newline at end of file + index_build_device: gpu0 # GPU used for building index, must be in format: gpux \ No newline at end of file diff --git a/core/src/server/Config.cpp b/core/src/server/Config.cpp index 51449cb1de..684d92674c 100644 --- a/core/src/server/Config.cpp +++ b/core/src/server/Config.cpp @@ -363,8 +363,8 @@ Config::PrintAll() { Status Config::CheckServerConfigAddress(const std::string& value) { if (!ValidationUtil::ValidateIpAddress(value).ok()) { - std::string msg = "Invalid server IP address: " + value + - ". Possible reason: server_config.address is invalid in server_config.yaml."; + std::string msg = + "Invalid server IP address: " + value + ". Possible reason: server_config.address is invalid."; return Status(SERVER_INVALID_ARGUMENT, msg); } return Status::OK(); @@ -373,14 +373,13 @@ Config::CheckServerConfigAddress(const std::string& value) { Status Config::CheckServerConfigPort(const std::string& value) { if (!ValidationUtil::ValidateStringIsNumber(value).ok()) { - std::string msg = "Port " + value + " is not a number. " + - "Possible reason: server_config.port in server_config.yaml is invalid."; + std::string msg = "Port " + value + " is not a number. " + "Possible reason: server_config.port is invalid."; return Status(SERVER_INVALID_ARGUMENT, msg); } else { int32_t port = std::stoi(value); if (!(port > 1024 && port < 65535)) { std::string msg = "Port " + value + " is not in range [1025, 65534]. " + - "Possible reason: server_config.port in server_config.yaml is invalid."; + "Possible reason: server_config.port is invalid."; return Status(SERVER_INVALID_ARGUMENT, msg); } } @@ -391,7 +390,7 @@ Status Config::CheckServerConfigDeployMode(const std::string& value) { if (value != "single" && value != "cluster_readonly" && value != "cluster_writable") { return Status(SERVER_INVALID_ARGUMENT, - "Error: server_config.deploy_mode in server_config.yaml is not one of " + "Error: server_config.deploy_mode is not one of " "single, cluster_readonly, and cluster_writable."); } return Status::OK(); @@ -418,8 +417,7 @@ Config::CheckServerConfigTimeZone(const std::string& value) { Status Config::CheckDBConfigPrimaryPath(const std::string& value) { if (value.empty()) { - return Status(SERVER_INVALID_ARGUMENT, - "db_path is empty. Possible reason: db_config.db_path in server_config.yaml is empty."); + return Status(SERVER_INVALID_ARGUMENT, "db_path is empty. Possible reason: db_config.db_path is empty."); } return Status::OK(); } @@ -433,8 +431,7 @@ Status Config::CheckDBConfigBackendUrl(const std::string& value) { if (!ValidationUtil::ValidateDbURI(value).ok()) { std::string msg = - "Invalid db_backend_url: " + value + - ". Possible reason: db_config.db_backend_url is invalid in server_config.yaml. " + + "Invalid db_backend_url: " + value + ". Possible reason: db_config.db_backend_url is invalid. " + "The correct format should be like sqlite://:@:/ or mysql://root:123456@127.0.0.1:3306/milvus."; return Status(SERVER_INVALID_ARGUMENT, "invalid db_backend_url: " + value); } @@ -445,7 +442,7 @@ Status Config::CheckDBConfigArchiveDiskThreshold(const std::string& value) { if (!ValidationUtil::ValidateStringIsNumber(value).ok()) { std::string msg = "Invalid archive disk threshold: " + value + - "Possible reason: db_config.archive_disk_threshold in server_config.yaml is invalid."; + "Possible reason: db_config.archive_disk_threshold is invalid."; return Status(SERVER_INVALID_ARGUMENT, msg); } return Status::OK(); @@ -455,7 +452,7 @@ Status Config::CheckDBConfigArchiveDaysThreshold(const std::string& value) { if (!ValidationUtil::ValidateStringIsNumber(value).ok()) { std::string msg = "Invalid archive days threshold: " + value + - "Possible reason: db_config.archive_disk_threshold in server_config.yaml is invalid."; + "Possible reason: db_config.archive_disk_threshold is invalid."; return Status(SERVER_INVALID_ARGUMENT, msg); } return Status::OK(); @@ -465,15 +462,13 @@ Status Config::CheckDBConfigInsertBufferSize(const std::string& value) { if (!ValidationUtil::ValidateStringIsNumber(value).ok()) { std::string msg = "Invalid insert buffer size: " + value + - "Possible reason: db_config.insert_buffer_size in server_config.yaml " - "is not a positive integer."; + "Possible reason: db_config.insert_buffer_size is not a positive integer."; return Status(SERVER_INVALID_ARGUMENT, msg); } else { int64_t buffer_size = std::stoi(value) * GB; if (buffer_size <= 0) { std::string msg = "Invalid insert buffer size: " + value + - "Possible reason: db_config.insert_buffer_size in server_config.yaml " - "is not a positive integer."; + "Possible reason: db_config.insert_buffer_size is not a positive integer."; return Status(SERVER_INVALID_ARGUMENT, msg); } @@ -535,7 +530,7 @@ Config::CheckCacheConfigCpuCacheCapacity(const std::string& value) { CommonUtil::GetSystemMemInfo(total_mem, free_mem); if (static_cast(cpu_cache_capacity) >= total_mem) { std::string msg = "Invalid cpu cache capacity: " + value + - "Possible reason: Cache config cpu_cache_capacity exceeds system memory."; + "Possible reason: cache_config.cpu_cache_capacity exceeds system memory."; return Status(SERVER_INVALID_ARGUMENT, msg); } else if (static_cast(cpu_cache_capacity) > static_cast(total_mem * 0.9)) { std::cerr << "WARNING: cpu cache capacity value is too big" << std::endl; From bfdc80401d4c560b256cb9e887543cbd17b57e7b Mon Sep 17 00:00:00 2001 From: starlord Date: Tue, 29 Oct 2019 16:15:14 +0800 Subject: [PATCH 28/31] fix typo Former-commit-id: bc177ec75ef30265a1358961f89966c8d6ea535c --- core/conf/server_config.template | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/conf/server_config.template b/core/conf/server_config.template index 3cb899d1b9..3feb16fd63 100644 --- a/core/conf/server_config.template +++ b/core/conf/server_config.template @@ -2,7 +2,7 @@ server_config: address: 0.0.0.0 # milvus server ip address (IPv4) - port: 19530 # milvus server port, must in range [1025, 6553] + port: 19530 # milvus server port, must in range [1025, 65534] deploy_mode: single # deployment type: single, cluster_readonly, cluster_writable time_zone: UTC+8 # time zone, must be in format: UTC+X @@ -24,7 +24,7 @@ metric_config: enable_monitor: false # enable monitoring or not, must be a boolean collector: prometheus # prometheus prometheus_config: - port: 8080 # port prometheus uses to fetch metrics, must in range [1025, 6553] + port: 8080 # port prometheus uses to fetch metrics, must in range [1025, 65534] cache_config: cpu_cache_capacity: 16 # GB, CPU memory used for cache, must be a positive integer From dcc7fbd439ca674bc39aeb859e001c6e4fbe4311 Mon Sep 17 00:00:00 2001 From: starlord Date: Tue, 29 Oct 2019 16:50:55 +0800 Subject: [PATCH 29/31] update message Former-commit-id: 84f9b8ce2ac8050128f678754123ae2716e3b342 --- core/src/server/Config.cpp | 75 +++++++++++++++++++------------------- 1 file changed, 38 insertions(+), 37 deletions(-) diff --git a/core/src/server/Config.cpp b/core/src/server/Config.cpp index 684d92674c..937556824e 100644 --- a/core/src/server/Config.cpp +++ b/core/src/server/Config.cpp @@ -373,13 +373,13 @@ Config::CheckServerConfigAddress(const std::string& value) { Status Config::CheckServerConfigPort(const std::string& value) { if (!ValidationUtil::ValidateStringIsNumber(value).ok()) { - std::string msg = "Port " + value + " is not a number. " + "Possible reason: server_config.port is invalid."; + std::string msg = "Invalid server port: " + value + ". Possible reason: server_config.port is not a number."; return Status(SERVER_INVALID_ARGUMENT, msg); } else { int32_t port = std::stoi(value); if (!(port > 1024 && port < 65535)) { - std::string msg = "Port " + value + " is not in range [1025, 65534]. " + - "Possible reason: server_config.port is invalid."; + std::string msg = "Invalid server port: " + value + + ". Possible reason: server_config.port is not in range [1025, 65534]."; return Status(SERVER_INVALID_ARGUMENT, msg); } } @@ -390,7 +390,7 @@ Status Config::CheckServerConfigDeployMode(const std::string& value) { if (value != "single" && value != "cluster_readonly" && value != "cluster_writable") { return Status(SERVER_INVALID_ARGUMENT, - "Error: server_config.deploy_mode is not one of " + "server_config.deploy_mode is not one of " "single, cluster_readonly, and cluster_writable."); } return Status::OK(); @@ -399,15 +399,15 @@ Config::CheckServerConfigDeployMode(const std::string& value) { Status Config::CheckServerConfigTimeZone(const std::string& value) { if (value.length() <= 3) { - return Status(SERVER_INVALID_ARGUMENT, "Invalid server config time_zone: " + value); + return Status(SERVER_INVALID_ARGUMENT, "Invalid server_config.time_zone: " + value); } else { if (value.substr(0, 3) != "UTC") { - return Status(SERVER_INVALID_ARGUMENT, "Invalid server config time_zone: " + value); + return Status(SERVER_INVALID_ARGUMENT, "Invalid server_config.time_zone: " + value); } else { try { stoi(value.substr(3)); } catch (...) { - return Status(SERVER_INVALID_ARGUMENT, "Invalid server config time_zone: " + value); + return Status(SERVER_INVALID_ARGUMENT, "Invalid server_config.time_zone: " + value); } } } @@ -417,7 +417,7 @@ Config::CheckServerConfigTimeZone(const std::string& value) { Status Config::CheckDBConfigPrimaryPath(const std::string& value) { if (value.empty()) { - return Status(SERVER_INVALID_ARGUMENT, "db_path is empty. Possible reason: db_config.db_path is empty."); + return Status(SERVER_INVALID_ARGUMENT, "db_config.db_path is empty."); } return Status::OK(); } @@ -431,7 +431,7 @@ Status Config::CheckDBConfigBackendUrl(const std::string& value) { if (!ValidationUtil::ValidateDbURI(value).ok()) { std::string msg = - "Invalid db_backend_url: " + value + ". Possible reason: db_config.db_backend_url is invalid. " + + "Invalid backend url: " + value + ". Possible reason: db_config.db_backend_url is invalid. " + "The correct format should be like sqlite://:@:/ or mysql://root:123456@127.0.0.1:3306/milvus."; return Status(SERVER_INVALID_ARGUMENT, "invalid db_backend_url: " + value); } @@ -442,7 +442,7 @@ Status Config::CheckDBConfigArchiveDiskThreshold(const std::string& value) { if (!ValidationUtil::ValidateStringIsNumber(value).ok()) { std::string msg = "Invalid archive disk threshold: " + value + - "Possible reason: db_config.archive_disk_threshold is invalid."; + ". Possible reason: db_config.archive_disk_threshold is invalid."; return Status(SERVER_INVALID_ARGUMENT, msg); } return Status::OK(); @@ -452,7 +452,7 @@ Status Config::CheckDBConfigArchiveDaysThreshold(const std::string& value) { if (!ValidationUtil::ValidateStringIsNumber(value).ok()) { std::string msg = "Invalid archive days threshold: " + value + - "Possible reason: db_config.archive_disk_threshold is invalid."; + ". Possible reason: db_config.archive_disk_threshold is invalid."; return Status(SERVER_INVALID_ARGUMENT, msg); } return Status::OK(); @@ -462,13 +462,13 @@ Status Config::CheckDBConfigInsertBufferSize(const std::string& value) { if (!ValidationUtil::ValidateStringIsNumber(value).ok()) { std::string msg = "Invalid insert buffer size: " + value + - "Possible reason: db_config.insert_buffer_size is not a positive integer."; + ". Possible reason: db_config.insert_buffer_size is not a positive integer."; return Status(SERVER_INVALID_ARGUMENT, msg); } else { int64_t buffer_size = std::stoi(value) * GB; if (buffer_size <= 0) { std::string msg = "Invalid insert buffer size: " + value + - "Possible reason: db_config.insert_buffer_size is not a positive integer."; + ". Possible reason: db_config.insert_buffer_size is not a positive integer."; return Status(SERVER_INVALID_ARGUMENT, msg); } @@ -476,7 +476,7 @@ Config::CheckDBConfigInsertBufferSize(const std::string& value) { CommonUtil::GetSystemMemInfo(total_mem, free_mem); if (buffer_size >= total_mem) { std::string msg = - "Invalid insert buffer size: " + value + "Possible reason: insert buffer size exceeds system memory."; + "Invalid insert buffer size: " + value + ". Possible reason: insert buffer size exceeds system memory."; return Status(SERVER_INVALID_ARGUMENT, msg); } } @@ -487,7 +487,7 @@ Status Config::CheckMetricConfigEnableMonitor(const std::string& value) { if (!ValidationUtil::ValidateStringIsBool(value).ok()) { std::string msg = - "Invalid metric config: " + value + "Possible reason: metric_config.enable_monitor is not a boolean."; + "Invalid metric config: " + value + ". Possible reason: metric_config.enable_monitor is not a boolean."; return Status(SERVER_INVALID_ARGUMENT, msg); } return Status::OK(); @@ -496,7 +496,8 @@ Config::CheckMetricConfigEnableMonitor(const std::string& value) { Status Config::CheckMetricConfigCollector(const std::string& value) { if (value != "prometheus") { - std::string msg = "Invalid metric config: " + value + "Possible reason: metric_config.collector is invalid."; + std::string msg = + "Invalid metric collector: " + value + ". Possible reason: metric_config.collector is invalid."; return Status(SERVER_INVALID_ARGUMENT, msg); } return Status::OK(); @@ -505,8 +506,8 @@ Config::CheckMetricConfigCollector(const std::string& value) { Status Config::CheckMetricConfigPrometheusPort(const std::string& value) { if (!ValidationUtil::ValidateStringIsNumber(value).ok()) { - std::string msg = "Invalid metric config: " + value + - "Possible reason: metric_config.prometheus_config.port is not in range [1025, 65534]."; + std::string msg = "Invalid metric port: " + value + + ". Possible reason: metric_config.prometheus_config.port is not in range [1025, 65534]."; return Status(SERVER_INVALID_ARGUMENT, "Invalid metric config prometheus_port: " + value); } return Status::OK(); @@ -516,13 +517,13 @@ Status Config::CheckCacheConfigCpuCacheCapacity(const std::string& value) { if (!ValidationUtil::ValidateStringIsNumber(value).ok()) { std::string msg = "Invalid cpu cache capacity: " + value + - "Possible reason: cache_config.cpu_cache_capacity is not a positive integer."; + ". Possible reason: cache_config.cpu_cache_capacity is not a positive integer."; return Status(SERVER_INVALID_ARGUMENT, msg); } else { int64_t cpu_cache_capacity = std::stoi(value) * GB; if (cpu_cache_capacity <= 0) { std::string msg = "Invalid cpu cache capacity: " + value + - "Possible reason: cache_config.cpu_cache_capacity is not a positive integer."; + ". Possible reason: cache_config.cpu_cache_capacity is not a positive integer."; return Status(SERVER_INVALID_ARGUMENT, msg); } @@ -530,7 +531,7 @@ Config::CheckCacheConfigCpuCacheCapacity(const std::string& value) { CommonUtil::GetSystemMemInfo(total_mem, free_mem); if (static_cast(cpu_cache_capacity) >= total_mem) { std::string msg = "Invalid cpu cache capacity: " + value + - "Possible reason: cache_config.cpu_cache_capacity exceeds system memory."; + ". Possible reason: cache_config.cpu_cache_capacity exceeds system memory."; return Status(SERVER_INVALID_ARGUMENT, msg); } else if (static_cast(cpu_cache_capacity) > static_cast(total_mem * 0.9)) { std::cerr << "WARNING: cpu cache capacity value is too big" << std::endl; @@ -545,7 +546,7 @@ Config::CheckCacheConfigCpuCacheCapacity(const std::string& value) { int64_t insert_buffer_size = buffer_value * GB; if (insert_buffer_size + cpu_cache_capacity >= total_mem) { std::string msg = "Invalid cpu cache capacity: " + value + - "Possible reason: sum of cache_config.cpu_cache_capacity and " + ". Possible reason: sum of cache_config.cpu_cache_capacity and " "db_config.insert_buffer_size exceeds system memory."; return Status(SERVER_INVALID_ARGUMENT, msg); } @@ -557,13 +558,13 @@ Status Config::CheckCacheConfigCpuCacheThreshold(const std::string& value) { if (!ValidationUtil::ValidateStringIsFloat(value).ok()) { std::string msg = "Invalid cpu cache threshold: " + value + - "Possible reason: cache_config.cpu_cache_threshold is not in range (0.0, 1.0]."; + ". Possible reason: cache_config.cpu_cache_threshold is not in range (0.0, 1.0]."; return Status(SERVER_INVALID_ARGUMENT, msg); } else { float cpu_cache_threshold = std::stof(value); if (cpu_cache_threshold <= 0.0 || cpu_cache_threshold >= 1.0) { std::string msg = "Invalid cpu cache threshold: " + value + - "Possible reason: cache_config.cpu_cache_threshold is not in range (0.0, 1.0]."; + ". Possible reason: cache_config.cpu_cache_threshold is not in range (0.0, 1.0]."; return Status(SERVER_INVALID_ARGUMENT, msg); } } @@ -574,7 +575,7 @@ Status Config::CheckCacheConfigGpuCacheCapacity(const std::string& value) { if (!ValidationUtil::ValidateStringIsNumber(value).ok()) { std::string msg = "Invalid gpu cache capacity: " + value + - "Possible reason: cache_config.gpu_cache_capacity is not a positive integer."; + ". Possible reason: cache_config.gpu_cache_capacity is not a positive integer."; return Status(SERVER_INVALID_ARGUMENT, msg); } else { uint64_t gpu_cache_capacity = std::stoi(value) * GB; @@ -590,7 +591,7 @@ Config::CheckCacheConfigGpuCacheCapacity(const std::string& value) { return Status(SERVER_UNEXPECTED_ERROR, msg); } else if (gpu_cache_capacity >= gpu_memory) { std::string msg = "Invalid gpu cache capacity: " + value + - "Possible reason: cache_config.gpu_cache_capacity exceeds GPU memory."; + ". Possible reason: cache_config.gpu_cache_capacity exceeds GPU memory."; return Status(SERVER_INVALID_ARGUMENT, msg); } else if (gpu_cache_capacity > (double)gpu_memory * 0.9) { std::cerr << "Warning: gpu cache capacity value is too big" << std::endl; @@ -603,13 +604,13 @@ Status Config::CheckCacheConfigGpuCacheThreshold(const std::string& value) { if (!ValidationUtil::ValidateStringIsFloat(value).ok()) { std::string msg = "Invalid gpu cache threshold: " + value + - "Possible reason: cache_config.gpu_cache_threshold is not in range (0.0, 1.0]."; + ". Possible reason: cache_config.gpu_cache_threshold is not in range (0.0, 1.0]."; return Status(SERVER_INVALID_ARGUMENT, msg); } else { float gpu_cache_threshold = std::stof(value); if (gpu_cache_threshold <= 0.0 || gpu_cache_threshold >= 1.0) { std::string msg = "Invalid gpu cache threshold: " + value + - "Possible reason: cache_config.gpu_cache_threshold is not in range (0.0, 1.0]."; + ". Possible reason: cache_config.gpu_cache_threshold is not in range (0.0, 1.0]."; return Status(SERVER_INVALID_ARGUMENT, msg); } } @@ -620,7 +621,7 @@ Status Config::CheckCacheConfigCacheInsertData(const std::string& value) { if (!ValidationUtil::ValidateStringIsBool(value).ok()) { std::string msg = "Invalid cache insert option: " + value + - "Possible reason: cache_config.cache_insert_data is not a boolean."; + ". Possible reason: cache_config.cache_insert_data is not a boolean."; return Status(SERVER_INVALID_ARGUMENT, msg); } return Status::OK(); @@ -630,7 +631,7 @@ Status Config::CheckEngineConfigUseBlasThreshold(const std::string& value) { if (!ValidationUtil::ValidateStringIsNumber(value).ok()) { std::string msg = "Invalid blas threshold: " + value + - "Possible reason: engine_config.use_blas_threshold is not a positive integer."; + ". Possible reason: engine_config.use_blas_threshold is not a positive integer."; return Status(SERVER_INVALID_ARGUMENT, msg); } return Status::OK(); @@ -640,7 +641,7 @@ Status Config::CheckEngineConfigOmpThreadNum(const std::string& value) { if (!ValidationUtil::ValidateStringIsNumber(value).ok()) { std::string msg = "Invalid omp thread number: " + value + - "Possible reason: engine_config.omp_thread_num is not a positive integer."; + ". Possible reason: engine_config.omp_thread_num is not a positive integer."; return Status(SERVER_INVALID_ARGUMENT, msg); } @@ -649,7 +650,7 @@ Config::CheckEngineConfigOmpThreadNum(const std::string& value) { CommonUtil::GetSystemAvailableThreads(sys_thread_cnt); if (omp_thread > static_cast(sys_thread_cnt)) { std::string msg = "Invalid omp thread number: " + value + - "Possible reason: engine_config.omp_thread_num exceeds system cpu cores."; + ". Possible reason: engine_config.omp_thread_num exceeds system cpu cores."; return Status(SERVER_INVALID_ARGUMENT, msg); } return Status::OK(); @@ -658,7 +659,7 @@ Config::CheckEngineConfigOmpThreadNum(const std::string& value) { Status Config::CheckResourceConfigMode(const std::string& value) { if (value != "simple") { - std::string msg = "Invalid resource mode: " + value + "Possible reason: resource_config.mode is invalid."; + std::string msg = "Invalid resource mode: " + value + ". Possible reason: resource_config.mode is invalid."; return Status(SERVER_INVALID_ARGUMENT, msg); } return Status::OK(); @@ -670,14 +671,14 @@ CheckGpuDevice(const std::string& value) { std::cmatch m; if (!std::regex_match(value.c_str(), m, pat)) { std::string msg = "Invalid gpu device: " + value + - "Possible reason: resource_config.search_resources does not match your hardware."; + ". Possible reason: resource_config.search_resources does not match your hardware."; return Status(SERVER_INVALID_ARGUMENT, msg); } int32_t gpu_index = std::stoi(value.substr(3)); if (!ValidationUtil::ValidateGpuIndex(gpu_index).ok()) { std::string msg = "Invalid gpu device: " + value + - "Possible reason: resource_config.search_resources does not match your hardware."; + ". Possible reason: resource_config.search_resources does not match your hardware."; return Status(SERVER_INVALID_ARGUMENT, msg); } return Status::OK(); @@ -695,7 +696,7 @@ Config::CheckResourceConfigSearchResources(const std::vector& value for (auto& gpu_device : value) { if (!CheckGpuDevice(gpu_device).ok()) { std::string msg = "Invalid search resource: " + gpu_device + - "Possible reason: resource_config.search_resources does not match your hardware."; + ". Possible reason: resource_config.search_resources does not match your hardware."; return Status(SERVER_INVALID_ARGUMENT, msg); } } @@ -706,7 +707,7 @@ Status Config::CheckResourceConfigIndexBuildDevice(const std::string& value) { if (!CheckGpuDevice(value).ok()) { std::string msg = "Invalid index build device: " + value + - "Possible reason: resource_config.index_build_device does not match your hardware."; + ". Possible reason: resource_config.index_build_device does not match your hardware."; return Status(SERVER_INVALID_ARGUMENT, msg); } return Status::OK(); From bb43a771306dd8ef4237d72189a260fe5a28296c Mon Sep 17 00:00:00 2001 From: starlord Date: Tue, 29 Oct 2019 17:19:02 +0800 Subject: [PATCH 30/31] update message Former-commit-id: 0a5ad5dfd42d76032c5008d9d9e4e2213b161232 --- core/src/server/Config.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/src/server/Config.cpp b/core/src/server/Config.cpp index 937556824e..111cc26f9c 100644 --- a/core/src/server/Config.cpp +++ b/core/src/server/Config.cpp @@ -475,8 +475,8 @@ Config::CheckDBConfigInsertBufferSize(const std::string& value) { uint64_t total_mem = 0, free_mem = 0; CommonUtil::GetSystemMemInfo(total_mem, free_mem); if (buffer_size >= total_mem) { - std::string msg = - "Invalid insert buffer size: " + value + ". Possible reason: insert buffer size exceeds system memory."; + std::string msg = "Invalid insert buffer size: " + value + + ". Possible reason: db_config.insert_buffer_size exceeds system memory."; return Status(SERVER_INVALID_ARGUMENT, msg); } } From 39e127d70752737351efaa1a6ed07d05a6437296 Mon Sep 17 00:00:00 2001 From: "xiaojun.lin" Date: Tue, 29 Oct 2019 20:05:01 +0800 Subject: [PATCH 31/31] Upgrade Faiss to v1.6 Former-commit-id: 637f6933c77c2644d4d90120d9d2aed77e4e70ff --- core/src/db/engine/ExecutionEngineImpl.cpp | 2 -- core/src/index/cmake/ThirdPartyPackagesCore.cmake | 6 +++--- .../knowhere/knowhere/index/vector_index/IndexGPUIVF.cpp | 3 ++- .../knowhere/index/vector_index/IndexIVFSQHybrid.cpp | 7 ++++++- core/src/scheduler/job/Job.cpp | 2 +- core/unittest/db/test_engine.cpp | 1 - core/unittest/db/utils.cpp | 4 ++-- 7 files changed, 14 insertions(+), 11 deletions(-) diff --git a/core/src/db/engine/ExecutionEngineImpl.cpp b/core/src/db/engine/ExecutionEngineImpl.cpp index 1ecba677fe..66e9795ff3 100644 --- a/core/src/db/engine/ExecutionEngineImpl.cpp +++ b/core/src/db/engine/ExecutionEngineImpl.cpp @@ -257,7 +257,6 @@ ExecutionEngineImpl::Load(bool to_cache) { Status ExecutionEngineImpl::CopyToGpu(uint64_t device_id, bool hybrid) { if (hybrid) { -#if 1 const std::string key = location_ + ".quantizer"; std::vector gpus = scheduler::get_gpu_pool(); @@ -306,7 +305,6 @@ ExecutionEngineImpl::CopyToGpu(uint64_t device_id, bool hybrid) { auto cached_quantizer = std::make_shared(pair.second); cache::GpuCacheMgr::GetInstance(device_id)->InsertItem(key, cached_quantizer); } -#endif return Status::OK(); } diff --git a/core/src/index/cmake/ThirdPartyPackagesCore.cmake b/core/src/index/cmake/ThirdPartyPackagesCore.cmake index 3635b4fa1b..e8a5c8a995 100644 --- a/core/src/index/cmake/ThirdPartyPackagesCore.cmake +++ b/core/src/index/cmake/ThirdPartyPackagesCore.cmake @@ -245,11 +245,11 @@ if(CUSTOMIZATION) # set(FAISS_MD5 "072db398351cca6e88f52d743bbb9fa0") # commit-id 3a2344d04744166af41ef1a74449d68a315bfe17 branch-0.2.1 # set(FAISS_MD5 "c89ea8e655f5cdf58f42486f13614714") # commit-id 9c28a1cbb88f41fa03b03d7204106201ad33276b branch-0.2.1 # set(FAISS_MD5 "87fdd86351ffcaf3f80dc26ade63c44b") # commit-id 841a156e67e8e22cd8088e1b58c00afbf2efc30b branch-0.2.1 - set(FAISS_MD5 "f3b2ce3364c3fa7febd3aa7fdd0fe380") # commit-id 694e03458e6b69ce8a62502f71f69a614af5af8f branch-0.3.0 + set(FAISS_MD5 "f3b2ce3364c3fa7febd3aa7fdd0fe380") # commit-id 694e03458e6b69ce8a62502f71f69a614af5af8f branch-0.3.0 endif() else() - set(FAISS_SOURCE_URL "https://github.com/facebookresearch/faiss/archive/v1.5.3.tar.gz") - set(FAISS_MD5 "0bc12737b23def156f6a1eb782050135") + set(FAISS_SOURCE_URL "https://github.com/milvus-io/faiss/archive/1.6.0.tar.gz") + set(FAISS_MD5 "eb96d84f98b078a9eec04a796f5c792e") endif() message(STATUS "FAISS URL = ${FAISS_SOURCE_URL}") diff --git a/core/src/index/knowhere/knowhere/index/vector_index/IndexGPUIVF.cpp b/core/src/index/knowhere/knowhere/index/vector_index/IndexGPUIVF.cpp index a26f947181..251dfc12ed 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/IndexGPUIVF.cpp +++ b/core/src/index/knowhere/knowhere/index/vector_index/IndexGPUIVF.cpp @@ -86,7 +86,8 @@ GPUIVF::SerializeImpl() { faiss::Index* index = index_.get(); faiss::Index* host_index = faiss::gpu::index_gpu_to_cpu(index); - SealImpl(); + // TODO(linxj): support seal + // SealImpl(); faiss::write_index(host_index, &writer); delete host_index; diff --git a/core/src/index/knowhere/knowhere/index/vector_index/IndexIVFSQHybrid.cpp b/core/src/index/knowhere/knowhere/index/vector_index/IndexIVFSQHybrid.cpp index 7b229db21e..84bf594421 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/IndexIVFSQHybrid.cpp +++ b/core/src/index/knowhere/knowhere/index/vector_index/IndexIVFSQHybrid.cpp @@ -160,7 +160,7 @@ IVFSQHybrid::LoadQuantizer(const Config& conf) { index_composition->quantizer = nullptr; index_composition->mode = quantizer_conf->mode; // only 1 - auto gpu_index = faiss::gpu::index_cpu_to_gpu(res->faiss_res.get(), gpu_id_, index_composition, &option); + auto gpu_index = faiss::gpu::index_cpu_to_gpu(res->faiss_res.get(), gpu_id, index_composition, &option); delete gpu_index; auto q = std::make_shared(); @@ -354,5 +354,10 @@ IVFSQHybrid::LoadImpl(const BinarySet& index_binary) { GPUIVF::LoadImpl(index_binary); } +void +IVFSQHybrid::set_index_model(IndexModelPtr model) { + GPUIVF::set_index_model(model); +} + #endif } // namespace knowhere diff --git a/core/src/scheduler/job/Job.cpp b/core/src/scheduler/job/Job.cpp index 1199fe17a6..06a163b959 100644 --- a/core/src/scheduler/job/Job.cpp +++ b/core/src/scheduler/job/Job.cpp @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -#include "Job.h" +#include "scheduler/job/Job.h" namespace milvus { namespace scheduler { diff --git a/core/unittest/db/test_engine.cpp b/core/unittest/db/test_engine.cpp index 147de5399c..eb2c60ec4b 100644 --- a/core/unittest/db/test_engine.cpp +++ b/core/unittest/db/test_engine.cpp @@ -108,7 +108,6 @@ TEST_F(EngineTest, ENGINE_IMPL_TEST) { ASSERT_EQ(engine_ptr->Dimension(), dimension); ASSERT_EQ(engine_ptr->Count(), ids.size()); - status = engine_ptr->CopyToGpu(0, true); status = engine_ptr->CopyToGpu(0, false); //ASSERT_TRUE(status.ok()); diff --git a/core/unittest/db/utils.cpp b/core/unittest/db/utils.cpp index 8903ce14ea..16e195079c 100644 --- a/core/unittest/db/utils.cpp +++ b/core/unittest/db/utils.cpp @@ -65,10 +65,10 @@ static const char " cache_insert_data: false # whether load inserted data into cache\n" "\n" "engine_config:\n" - " blas_threshold: 20\n" + " use_blas_threshold: 20\n" "\n" "resource_config:\n" - " resource_pool:\n" + " search_resources:\n" " - gpu0\n" " index_build_device: gpu0 # GPU used for building index";