mirror of
https://gitee.com/milvus-io/milvus.git
synced 2026-01-07 19:31:51 +08:00
Remove index build suspend support (#14219)
Signed-off-by: yudong.cai <yudong.cai@zilliz.com>
This commit is contained in:
Cai Yudong
GitHub
parent
6177d4651e
commit
4be6a442e8
@ -19,7 +19,6 @@
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
#include "faiss/BuilderSuspend.h"
|
||||
#include "hnswlib/hnswalg.h"
|
||||
#include "hnswlib/hnswlib.h"
|
||||
#include "hnswlib/space_ip.h"
|
||||
@ -122,7 +121,6 @@ IndexHNSW::AddWithoutIds(const DatasetPtr& dataset_ptr, const Config& config) {
|
||||
index_->addPoint(p_data, 0);
|
||||
#pragma omp parallel for
|
||||
for (int i = 1; i < rows; ++i) {
|
||||
faiss::BuilderSuspend::check_wait();
|
||||
index_->addPoint((reinterpret_cast<const float*>(p_data) + Dim() * i), i);
|
||||
}
|
||||
if (STATISTICS_LEVEL >= 3) {
|
||||
|
||||
@ -30,7 +30,6 @@
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
#include "faiss/BuilderSuspend.h"
|
||||
#include "knowhere/common/Exception.h"
|
||||
#include "knowhere/common/Log.h"
|
||||
#include "knowhere/index/vector_index/IndexIVF.h"
|
||||
@ -285,9 +284,6 @@ IVF::GenGraph(const float* data, const int64_t k, GraphType& graph, const Config
|
||||
graph.resize(ntotal);
|
||||
GraphType res_vec(total_search_count);
|
||||
for (int i = 0; i < total_search_count; ++i) {
|
||||
// it is usually used in NSG::train, to check BuilderSuspend
|
||||
faiss::BuilderSuspend::check_wait();
|
||||
|
||||
auto b_size = (i == (total_search_count - 1)) && tail_batch_size != 0 ? tail_batch_size : batch_size;
|
||||
|
||||
auto& res = res_vec[i];
|
||||
|
||||
@ -18,7 +18,6 @@
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
#include "faiss/BuilderSuspend.h"
|
||||
#include "knowhere/common/Exception.h"
|
||||
#include "knowhere/common/Log.h"
|
||||
#include "knowhere/index/vector_index/adapter/VectorAdapter.h"
|
||||
|
||||
@ -18,7 +18,6 @@
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
#include "faiss/BuilderSuspend.h"
|
||||
#include "knowhere/common/Exception.h"
|
||||
#include "knowhere/common/Log.h"
|
||||
#include "knowhere/index/vector_index/adapter/VectorAdapter.h"
|
||||
|
||||
@ -17,7 +17,6 @@
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
#include "faiss/BuilderSuspend.h"
|
||||
#include "knowhere/common/Exception.h"
|
||||
#include "knowhere/common/Log.h"
|
||||
#include "knowhere/index/vector_index/adapter/VectorAdapter.h"
|
||||
|
||||
@ -18,7 +18,6 @@
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
#include "faiss/BuilderSuspend.h"
|
||||
#include "knowhere/common/Exception.h"
|
||||
#include "knowhere/common/Log.h"
|
||||
#include "knowhere/index/vector_index/adapter/VectorAdapter.h"
|
||||
|
||||
@ -1,30 +0,0 @@
|
||||
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software distributed under the License
|
||||
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
|
||||
// or implied. See the License for the specific language governing permissions and limitations under the License
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "faiss/BuilderSuspend.h"
|
||||
|
||||
namespace milvus {
|
||||
namespace knowhere {
|
||||
|
||||
inline void
|
||||
BuilderSuspend() {
|
||||
faiss::BuilderSuspend::suspend();
|
||||
}
|
||||
|
||||
inline void
|
||||
BuildResume() {
|
||||
faiss::BuilderSuspend::resume();
|
||||
}
|
||||
|
||||
} // namespace knowhere
|
||||
} // namespace milvus
|
||||
@ -19,7 +19,6 @@
|
||||
#include <string>
|
||||
#include <utility>
|
||||
|
||||
#include "faiss/BuilderSuspend.h"
|
||||
#include "knowhere/common/Exception.h"
|
||||
#include "knowhere/common/Log.h"
|
||||
#include "knowhere/common/Timer.h"
|
||||
@ -456,7 +455,6 @@ NsgIndex::Link(float* data) {
|
||||
boost::dynamic_bitset<> flags{ntotal, 0};
|
||||
#pragma omp for schedule(dynamic, 100)
|
||||
for (size_t n = 0; n < ntotal; ++n) {
|
||||
faiss::BuilderSuspend::check_wait();
|
||||
fullset.clear();
|
||||
temp.clear();
|
||||
flags.reset();
|
||||
@ -486,7 +484,6 @@ NsgIndex::Link(float* data) {
|
||||
std::vector<std::mutex> mutex_vec(ntotal);
|
||||
#pragma omp for schedule(dynamic, 100)
|
||||
for (unsigned n = 0; n < ntotal; ++n) {
|
||||
faiss::BuilderSuspend::check_wait();
|
||||
InterInsert(data, n, mutex_vec, cut_graph_dist);
|
||||
}
|
||||
delete[] cut_graph_dist;
|
||||
@ -645,7 +642,6 @@ NsgIndex::CheckConnectivity(float* data) {
|
||||
int64_t linked_count = 0;
|
||||
|
||||
while (linked_count < static_cast<int64_t>(ntotal)) {
|
||||
faiss::BuilderSuspend::check_wait();
|
||||
DFS(root, has_linked, linked_count);
|
||||
if (linked_count >= static_cast<int64_t>(ntotal)) {
|
||||
break;
|
||||
|
||||
@ -29,7 +29,6 @@
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
#include "faiss/BuilderSuspend.h"
|
||||
#include "knowhere/common/Exception.h"
|
||||
#include "knowhere/common/Log.h"
|
||||
#include "knowhere/index/vector_index/adapter/VectorAdapter.h"
|
||||
@ -287,9 +286,6 @@ IVF_NM::GenGraph(const float* data, const int64_t k, GraphType& graph, const Con
|
||||
graph.resize(ntotal);
|
||||
GraphType res_vec(total_search_count);
|
||||
for (int i = 0; i < total_search_count; ++i) {
|
||||
// it is usually used in NSG::train, to check BuilderSuspend
|
||||
faiss::BuilderSuspend::check_wait();
|
||||
|
||||
auto b_size = (i == (total_search_count - 1)) && tail_batch_size != 0 ? tail_batch_size : batch_size;
|
||||
|
||||
auto& res = res_vec[i];
|
||||
|
||||
@ -127,7 +127,6 @@ inline void set_error_from_string(char **error, const char* msg) {
|
||||
#endif
|
||||
|
||||
#include <faiss/FaissHook.h>
|
||||
#include <faiss/BuilderSuspend.h>
|
||||
|
||||
using std::vector;
|
||||
using std::pair;
|
||||
@ -1283,7 +1282,6 @@ protected:
|
||||
vector<S> children_indices[2];
|
||||
Node* m = (Node*)alloca(_s);
|
||||
D::create_split(children, _f, _s, _random, m);
|
||||
faiss::BuilderSuspend::check_wait();
|
||||
|
||||
for (size_t i = 0; i < indices.size(); i++) {
|
||||
S j = indices[i];
|
||||
@ -1323,7 +1321,6 @@ protected:
|
||||
m->n_descendants = is_root ? _n_items : (S)indices.size();
|
||||
for (int side = 0; side < 2; side++) {
|
||||
// run _make_tree for the smallest child first (for cache locality)
|
||||
faiss::BuilderSuspend::check_wait();
|
||||
m->children[side^flip] = _make_tree(children_indices[side^flip], false);
|
||||
}
|
||||
|
||||
|
||||
@ -1,35 +0,0 @@
|
||||
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software distributed under the License
|
||||
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
|
||||
// or implied. See the License for the specific language governing permissions and limitations under the License.
|
||||
|
||||
#include "BuilderSuspend.h"
|
||||
|
||||
namespace faiss {
|
||||
|
||||
std::atomic<bool> BuilderSuspend::suspend_flag_(false);
|
||||
std::mutex BuilderSuspend::mutex_;
|
||||
std::condition_variable BuilderSuspend::cv_;
|
||||
|
||||
void BuilderSuspend::suspend() {
|
||||
suspend_flag_ = true;
|
||||
}
|
||||
|
||||
void BuilderSuspend::resume() {
|
||||
suspend_flag_ = false;
|
||||
}
|
||||
|
||||
void BuilderSuspend::check_wait() {
|
||||
while (suspend_flag_) {
|
||||
std::unique_lock<std::mutex> lck(mutex_);
|
||||
cv_.wait_for(lck, std::chrono::seconds(5));
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace faiss
|
||||
@ -1,33 +0,0 @@
|
||||
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software distributed under the License
|
||||
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
|
||||
// or implied. See the License for the specific language governing permissions and limitations under the License.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <atomic>
|
||||
#include <condition_variable>
|
||||
#include <mutex>
|
||||
|
||||
namespace faiss {
|
||||
|
||||
class BuilderSuspend {
|
||||
public:
|
||||
static void suspend();
|
||||
static void resume();
|
||||
static void check_wait();
|
||||
|
||||
private:
|
||||
static std::atomic<bool> suspend_flag_;
|
||||
static std::mutex mutex_;
|
||||
static std::condition_variable cv_;
|
||||
|
||||
};
|
||||
|
||||
} // namespace faiss
|
||||
@ -15,7 +15,6 @@
|
||||
#include <cmath>
|
||||
|
||||
#include <omp.h>
|
||||
#include <faiss/BuilderSuspend.h>
|
||||
#include <faiss/FaissHook.h>
|
||||
#include <faiss/impl/AuxIndexStructures.h>
|
||||
#include <faiss/impl/FaissAssert.h>
|
||||
@ -1087,8 +1086,6 @@ void elkan_L2_sse (
|
||||
float *data = (float *) malloc((bs_y * (bs_y - 1) / 2) * sizeof (float));
|
||||
|
||||
for (size_t j0 = 0; j0 < ny; j0 += bs_y) {
|
||||
BuilderSuspend::check_wait();
|
||||
|
||||
size_t j1 = j0 + bs_y;
|
||||
if (j1 > ny) j1 = ny;
|
||||
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user