From 4be6a442e8319a017d5d7866c1f6c807de63e2a5 Mon Sep 17 00:00:00 2001 From: Cai Yudong Date: Sat, 25 Dec 2021 21:38:20 +0800 Subject: [PATCH] Remove index build suspend support (#14219) Signed-off-by: yudong.cai --- .../knowhere/index/vector_index/IndexHNSW.cpp | 2 -- .../knowhere/index/vector_index/IndexIVF.cpp | 4 --- .../index/vector_index/IndexRHNSW.cpp | 1 - .../index/vector_index/IndexRHNSWFlat.cpp | 1 - .../index/vector_index/IndexRHNSWPQ.cpp | 1 - .../index/vector_index/IndexRHNSWSQ.cpp | 1 - .../vector_index/helpers/BuilderSuspend.h | 30 ---------------- .../index/vector_index/impl/nsg/NSG.cpp | 4 --- .../index/vector_offset_index/IndexIVF_NM.cpp | 4 --- .../src/index/thirdparty/annoy/src/annoylib.h | 3 -- .../index/thirdparty/faiss/BuilderSuspend.cpp | 35 ------------------- .../index/thirdparty/faiss/BuilderSuspend.h | 33 ----------------- .../thirdparty/faiss/utils/distances.cpp | 3 -- 13 files changed, 122 deletions(-) delete mode 100644 internal/core/src/index/knowhere/knowhere/index/vector_index/helpers/BuilderSuspend.h delete mode 100644 internal/core/src/index/thirdparty/faiss/BuilderSuspend.cpp delete mode 100644 internal/core/src/index/thirdparty/faiss/BuilderSuspend.h diff --git a/internal/core/src/index/knowhere/knowhere/index/vector_index/IndexHNSW.cpp b/internal/core/src/index/knowhere/knowhere/index/vector_index/IndexHNSW.cpp index 881b3b4bfd..a1672d5bf7 100644 --- a/internal/core/src/index/knowhere/knowhere/index/vector_index/IndexHNSW.cpp +++ b/internal/core/src/index/knowhere/knowhere/index/vector_index/IndexHNSW.cpp @@ -19,7 +19,6 @@ #include #include -#include "faiss/BuilderSuspend.h" #include "hnswlib/hnswalg.h" #include "hnswlib/hnswlib.h" #include "hnswlib/space_ip.h" @@ -122,7 +121,6 @@ IndexHNSW::AddWithoutIds(const DatasetPtr& dataset_ptr, const Config& config) { index_->addPoint(p_data, 0); #pragma omp parallel for for (int i = 1; i < rows; ++i) { - faiss::BuilderSuspend::check_wait(); index_->addPoint((reinterpret_cast(p_data) + Dim() * i), i); } if (STATISTICS_LEVEL >= 3) { diff --git a/internal/core/src/index/knowhere/knowhere/index/vector_index/IndexIVF.cpp b/internal/core/src/index/knowhere/knowhere/index/vector_index/IndexIVF.cpp index 08a5cc278f..187b938d3a 100644 --- a/internal/core/src/index/knowhere/knowhere/index/vector_index/IndexIVF.cpp +++ b/internal/core/src/index/knowhere/knowhere/index/vector_index/IndexIVF.cpp @@ -30,7 +30,6 @@ #include #include -#include "faiss/BuilderSuspend.h" #include "knowhere/common/Exception.h" #include "knowhere/common/Log.h" #include "knowhere/index/vector_index/IndexIVF.h" @@ -285,9 +284,6 @@ IVF::GenGraph(const float* data, const int64_t k, GraphType& graph, const Config graph.resize(ntotal); GraphType res_vec(total_search_count); for (int i = 0; i < total_search_count; ++i) { - // it is usually used in NSG::train, to check BuilderSuspend - faiss::BuilderSuspend::check_wait(); - auto b_size = (i == (total_search_count - 1)) && tail_batch_size != 0 ? tail_batch_size : batch_size; auto& res = res_vec[i]; diff --git a/internal/core/src/index/knowhere/knowhere/index/vector_index/IndexRHNSW.cpp b/internal/core/src/index/knowhere/knowhere/index/vector_index/IndexRHNSW.cpp index ec2d38a417..a8f6d514fe 100644 --- a/internal/core/src/index/knowhere/knowhere/index/vector_index/IndexRHNSW.cpp +++ b/internal/core/src/index/knowhere/knowhere/index/vector_index/IndexRHNSW.cpp @@ -18,7 +18,6 @@ #include #include -#include "faiss/BuilderSuspend.h" #include "knowhere/common/Exception.h" #include "knowhere/common/Log.h" #include "knowhere/index/vector_index/adapter/VectorAdapter.h" diff --git a/internal/core/src/index/knowhere/knowhere/index/vector_index/IndexRHNSWFlat.cpp b/internal/core/src/index/knowhere/knowhere/index/vector_index/IndexRHNSWFlat.cpp index a6e3a64e7f..c791f46691 100644 --- a/internal/core/src/index/knowhere/knowhere/index/vector_index/IndexRHNSWFlat.cpp +++ b/internal/core/src/index/knowhere/knowhere/index/vector_index/IndexRHNSWFlat.cpp @@ -18,7 +18,6 @@ #include #include -#include "faiss/BuilderSuspend.h" #include "knowhere/common/Exception.h" #include "knowhere/common/Log.h" #include "knowhere/index/vector_index/adapter/VectorAdapter.h" diff --git a/internal/core/src/index/knowhere/knowhere/index/vector_index/IndexRHNSWPQ.cpp b/internal/core/src/index/knowhere/knowhere/index/vector_index/IndexRHNSWPQ.cpp index 195547fbb6..b209170e78 100644 --- a/internal/core/src/index/knowhere/knowhere/index/vector_index/IndexRHNSWPQ.cpp +++ b/internal/core/src/index/knowhere/knowhere/index/vector_index/IndexRHNSWPQ.cpp @@ -17,7 +17,6 @@ #include #include -#include "faiss/BuilderSuspend.h" #include "knowhere/common/Exception.h" #include "knowhere/common/Log.h" #include "knowhere/index/vector_index/adapter/VectorAdapter.h" diff --git a/internal/core/src/index/knowhere/knowhere/index/vector_index/IndexRHNSWSQ.cpp b/internal/core/src/index/knowhere/knowhere/index/vector_index/IndexRHNSWSQ.cpp index 566e1b47c1..8ca847aecd 100644 --- a/internal/core/src/index/knowhere/knowhere/index/vector_index/IndexRHNSWSQ.cpp +++ b/internal/core/src/index/knowhere/knowhere/index/vector_index/IndexRHNSWSQ.cpp @@ -18,7 +18,6 @@ #include #include -#include "faiss/BuilderSuspend.h" #include "knowhere/common/Exception.h" #include "knowhere/common/Log.h" #include "knowhere/index/vector_index/adapter/VectorAdapter.h" diff --git a/internal/core/src/index/knowhere/knowhere/index/vector_index/helpers/BuilderSuspend.h b/internal/core/src/index/knowhere/knowhere/index/vector_index/helpers/BuilderSuspend.h deleted file mode 100644 index d77d9a9dea..0000000000 --- a/internal/core/src/index/knowhere/knowhere/index/vector_index/helpers/BuilderSuspend.h +++ /dev/null @@ -1,30 +0,0 @@ -// Copyright (C) 2019-2020 Zilliz. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software distributed under the License -// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -// or implied. See the License for the specific language governing permissions and limitations under the License - -#pragma once - -#include "faiss/BuilderSuspend.h" - -namespace milvus { -namespace knowhere { - -inline void -BuilderSuspend() { - faiss::BuilderSuspend::suspend(); -} - -inline void -BuildResume() { - faiss::BuilderSuspend::resume(); -} - -} // namespace knowhere -} // namespace milvus diff --git a/internal/core/src/index/knowhere/knowhere/index/vector_index/impl/nsg/NSG.cpp b/internal/core/src/index/knowhere/knowhere/index/vector_index/impl/nsg/NSG.cpp index 523b1e1019..084f9b22d3 100644 --- a/internal/core/src/index/knowhere/knowhere/index/vector_index/impl/nsg/NSG.cpp +++ b/internal/core/src/index/knowhere/knowhere/index/vector_index/impl/nsg/NSG.cpp @@ -19,7 +19,6 @@ #include #include -#include "faiss/BuilderSuspend.h" #include "knowhere/common/Exception.h" #include "knowhere/common/Log.h" #include "knowhere/common/Timer.h" @@ -456,7 +455,6 @@ NsgIndex::Link(float* data) { boost::dynamic_bitset<> flags{ntotal, 0}; #pragma omp for schedule(dynamic, 100) for (size_t n = 0; n < ntotal; ++n) { - faiss::BuilderSuspend::check_wait(); fullset.clear(); temp.clear(); flags.reset(); @@ -486,7 +484,6 @@ NsgIndex::Link(float* data) { std::vector mutex_vec(ntotal); #pragma omp for schedule(dynamic, 100) for (unsigned n = 0; n < ntotal; ++n) { - faiss::BuilderSuspend::check_wait(); InterInsert(data, n, mutex_vec, cut_graph_dist); } delete[] cut_graph_dist; @@ -645,7 +642,6 @@ NsgIndex::CheckConnectivity(float* data) { int64_t linked_count = 0; while (linked_count < static_cast(ntotal)) { - faiss::BuilderSuspend::check_wait(); DFS(root, has_linked, linked_count); if (linked_count >= static_cast(ntotal)) { break; diff --git a/internal/core/src/index/knowhere/knowhere/index/vector_offset_index/IndexIVF_NM.cpp b/internal/core/src/index/knowhere/knowhere/index/vector_offset_index/IndexIVF_NM.cpp index b47befe21f..ec5e8fd5ca 100644 --- a/internal/core/src/index/knowhere/knowhere/index/vector_offset_index/IndexIVF_NM.cpp +++ b/internal/core/src/index/knowhere/knowhere/index/vector_offset_index/IndexIVF_NM.cpp @@ -29,7 +29,6 @@ #include #include -#include "faiss/BuilderSuspend.h" #include "knowhere/common/Exception.h" #include "knowhere/common/Log.h" #include "knowhere/index/vector_index/adapter/VectorAdapter.h" @@ -287,9 +286,6 @@ IVF_NM::GenGraph(const float* data, const int64_t k, GraphType& graph, const Con graph.resize(ntotal); GraphType res_vec(total_search_count); for (int i = 0; i < total_search_count; ++i) { - // it is usually used in NSG::train, to check BuilderSuspend - faiss::BuilderSuspend::check_wait(); - auto b_size = (i == (total_search_count - 1)) && tail_batch_size != 0 ? tail_batch_size : batch_size; auto& res = res_vec[i]; diff --git a/internal/core/src/index/thirdparty/annoy/src/annoylib.h b/internal/core/src/index/thirdparty/annoy/src/annoylib.h index dcd8c346cc..deecc8fa41 100644 --- a/internal/core/src/index/thirdparty/annoy/src/annoylib.h +++ b/internal/core/src/index/thirdparty/annoy/src/annoylib.h @@ -127,7 +127,6 @@ inline void set_error_from_string(char **error, const char* msg) { #endif #include -#include using std::vector; using std::pair; @@ -1283,7 +1282,6 @@ protected: vector children_indices[2]; Node* m = (Node*)alloca(_s); D::create_split(children, _f, _s, _random, m); - faiss::BuilderSuspend::check_wait(); for (size_t i = 0; i < indices.size(); i++) { S j = indices[i]; @@ -1323,7 +1321,6 @@ protected: m->n_descendants = is_root ? _n_items : (S)indices.size(); for (int side = 0; side < 2; side++) { // run _make_tree for the smallest child first (for cache locality) - faiss::BuilderSuspend::check_wait(); m->children[side^flip] = _make_tree(children_indices[side^flip], false); } diff --git a/internal/core/src/index/thirdparty/faiss/BuilderSuspend.cpp b/internal/core/src/index/thirdparty/faiss/BuilderSuspend.cpp deleted file mode 100644 index dd32b630f7..0000000000 --- a/internal/core/src/index/thirdparty/faiss/BuilderSuspend.cpp +++ /dev/null @@ -1,35 +0,0 @@ -// Copyright (C) 2019-2020 Zilliz. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software distributed under the License -// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -// or implied. See the License for the specific language governing permissions and limitations under the License. - -#include "BuilderSuspend.h" - -namespace faiss { - -std::atomic BuilderSuspend::suspend_flag_(false); -std::mutex BuilderSuspend::mutex_; -std::condition_variable BuilderSuspend::cv_; - -void BuilderSuspend::suspend() { - suspend_flag_ = true; -} - -void BuilderSuspend::resume() { - suspend_flag_ = false; -} - -void BuilderSuspend::check_wait() { - while (suspend_flag_) { - std::unique_lock lck(mutex_); - cv_.wait_for(lck, std::chrono::seconds(5)); - } -} - -} // namespace faiss diff --git a/internal/core/src/index/thirdparty/faiss/BuilderSuspend.h b/internal/core/src/index/thirdparty/faiss/BuilderSuspend.h deleted file mode 100644 index d5291a9628..0000000000 --- a/internal/core/src/index/thirdparty/faiss/BuilderSuspend.h +++ /dev/null @@ -1,33 +0,0 @@ -// Copyright (C) 2019-2020 Zilliz. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software distributed under the License -// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -// or implied. See the License for the specific language governing permissions and limitations under the License. - -#pragma once - -#include -#include -#include - -namespace faiss { - -class BuilderSuspend { -public: - static void suspend(); - static void resume(); - static void check_wait(); - -private: - static std::atomic suspend_flag_; - static std::mutex mutex_; - static std::condition_variable cv_; - -}; - -} // namespace faiss diff --git a/internal/core/src/index/thirdparty/faiss/utils/distances.cpp b/internal/core/src/index/thirdparty/faiss/utils/distances.cpp index 8da34b9b57..7746e9df2e 100644 --- a/internal/core/src/index/thirdparty/faiss/utils/distances.cpp +++ b/internal/core/src/index/thirdparty/faiss/utils/distances.cpp @@ -15,7 +15,6 @@ #include #include -#include #include #include #include @@ -1087,8 +1086,6 @@ void elkan_L2_sse ( float *data = (float *) malloc((bs_y * (bs_y - 1) / 2) * sizeof (float)); for (size_t j0 = 0; j0 < ny; j0 += bs_y) { - BuilderSuspend::check_wait(); - size_t j1 = j0 + bs_y; if (j1 > ny) j1 = ny;