admin/milvus
1
0
Fork 0
mirror of https://gitee.com/milvus-io/milvus.git synced 2026-02-02 01:06:41 +08:00
Code Issues Packages Projects Releases Wiki Activity

Create index failed caused by server crashed (#2736)

Browse Source 
Signed-off-by: shengjun.li <shengjun.li@zilliz.com>

Co-authored-by: groot <yhmo@zeronedata.com>
This commit is contained in:
shengjun.li 2020-07-04 09:25:48 +08:00 committed by GitHub
parent e892626a1c
commit 0672347775
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
15 changed files with 109 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
CHANGELOG.md
View File

@ -14,6 +14,7 @@ Please mark all changes in change log and use the issue from GitHub
- \#2598 Fix Milvus docker image report illegal instruction
- \#2617 Fix HNSW and RNSG index files size
- \#2637 Suit the range of HNSW parameters
- \#2642 Create index failed and server crashed
- \#2649 Search parameter of annoy has conflict with document
- \#2690 Remove body parser in show-partitions endpoints

16
core/src/index/knowhere/knowhere/index/vector_index/IndexBinaryIDMAP.cpp
View File

@ -112,6 +112,22 @@ BinaryIDMAP::QueryById(const DatasetPtr& dataset_ptr, const Config& config) {
}
#endif
int64_t
BinaryIDMAP::Count() {
if (!index_) {
KNOWHERE_THROW_MSG("index not initialize");
}
return index_->ntotal;
}
int64_t
BinaryIDMAP::Dim() {
if (!index_) {
KNOWHERE_THROW_MSG("index not initialize");
}
return index_->d;
}
void
BinaryIDMAP::Add(const DatasetPtr& dataset_ptr, const Config& config) {
if (!index_) {

8
core/src/index/knowhere/knowhere/index/vector_index/IndexBinaryIDMAP.h
View File

@ -56,14 +56,10 @@ class BinaryIDMAP : public VecIndex, public FaissBaseBinaryIndex {
#endif
int64_t
Count() override {
return index_->ntotal;
}
Count() override;
int64_t
Dim() override {
return index_->d;
}
Dim() override;
int64_t
IndexSize() override {

16
core/src/index/knowhere/knowhere/index/vector_index/IndexBinaryIVF.cpp
View File

@ -129,6 +129,22 @@ BinaryIVF::QueryById(const DatasetPtr& dataset_ptr, const Config& config) {
}
#endif
int64_t
BinaryIVF::Count() {
if (!index_) {
KNOWHERE_THROW_MSG("index not initialize");
}
return index_->ntotal;
}
int64_t
BinaryIVF::Dim() {
if (!index_) {
KNOWHERE_THROW_MSG("index not initialize");
}
return index_->d;
}
void
BinaryIVF::Train(const DatasetPtr& dataset_ptr, const Config& config) {
GETTENSORWITHIDS(dataset_ptr)

8
core/src/index/knowhere/knowhere/index/vector_index/IndexBinaryIVF.h
View File

@ -68,14 +68,10 @@ class BinaryIVF : public VecIndex, public FaissBaseBinaryIndex {
#endif
int64_t
Count() override {
return index_->ntotal;
}
Count() override;
int64_t
Dim() override {
return index_->d;
}
Dim() override;
#if 0
DatasetPtr

16
core/src/index/knowhere/knowhere/index/vector_index/IndexIDMAP.cpp
View File

@ -142,6 +142,22 @@ IDMAP::QueryById(const DatasetPtr& dataset_ptr, const Config& config) {
}
#endif
int64_t
IDMAP::Count() {
if (!index_) {
KNOWHERE_THROW_MSG("index not initialize");
}
return index_->ntotal;
}
int64_t
IDMAP::Dim() {
if (!index_) {
KNOWHERE_THROW_MSG("index not initialize");
}
return index_->d;
}
VecIndexPtr
IDMAP::CopyCpuToGpu(const int64_t device_id, const Config& config) {
#ifdef MILVUS_GPU_VERSION

8
core/src/index/knowhere/knowhere/index/vector_index/IndexIDMAP.h
View File

@ -54,14 +54,10 @@ class IDMAP : public VecIndex, public FaissBaseIndex {
#endif
int64_t
Count() override {
return index_->ntotal;
}
Count() override;
int64_t
Dim() override {
return index_->d;
}
Dim() override;
int64_t
IndexSize() override {

16
core/src/index/knowhere/knowhere/index/vector_index/IndexIVF.cpp
View File

@ -217,6 +217,22 @@ IVF::GetVectorById(const DatasetPtr& dataset_ptr, const Config& config) {
}
#endif
int64_t
IVF::Count() {
if (!index_) {
KNOWHERE_THROW_MSG("index not initialize");
}
return index_->ntotal;
}
int64_t
IVF::Dim() {
if (!index_) {
KNOWHERE_THROW_MSG("index not initialize");
}
return index_->d;
}
void
IVF::Seal() {
if (!index_ || !index_->is_trained) {

8
core/src/index/knowhere/knowhere/index/vector_index/IndexIVF.h
View File

@ -59,14 +59,10 @@ class IVF : public VecIndex, public FaissBaseIndex {
#endif
int64_t
Count() override {
return index_->ntotal;
}
Count() override;
int64_t
Dim() override {
return index_->d;
}
Dim() override;
#if 0
DatasetPtr

6
core/src/index/knowhere/knowhere/index/vector_index/IndexNSG.cpp
View File

@ -149,11 +149,17 @@ NSG::Train(const DatasetPtr& dataset_ptr, const Config& config) {
int64_t
NSG::Count() {
if (!index_) {
KNOWHERE_THROW_MSG("index not initialize");
}
return index_->ntotal;
}
int64_t
NSG::Dim() {
if (!index_) {
KNOWHERE_THROW_MSG("index not initialize");
}
return index_->dimension;
}

6
core/src/index/knowhere/knowhere/index/vector_index/IndexSPTAG.cpp
View File

@ -200,11 +200,17 @@ CPUSPTAGRNG::Query(const DatasetPtr& dataset_ptr, const Config& config) {
int64_t
CPUSPTAGRNG::Count() {
if (!index_ptr_) {
KNOWHERE_THROW_MSG("index not initialize");
}
return index_ptr_->GetNumSamples();
}
int64_t
CPUSPTAGRNG::Dim() {
if (!index_ptr_) {
KNOWHERE_THROW_MSG("index not initialize");
}
return index_ptr_->GetFeatureDim();
}

16
core/src/index/knowhere/knowhere/index/vector_offset_index/IndexIVF_NM.cpp
View File

@ -335,5 +335,21 @@ IVF_NM::SealImpl() {
#endif
}
int64_t
IVF_NM::Count() {
if (!index_) {
KNOWHERE_THROW_MSG("index not initialize");
}
return index_->ntotal;
}
int64_t
IVF_NM::Dim() {
if (!index_) {
KNOWHERE_THROW_MSG("index not initialize");
}
return index_->d;
}
} // namespace knowhere
} // namespace milvus

8
core/src/index/knowhere/knowhere/index/vector_offset_index/IndexIVF_NM.h
View File

@ -59,14 +59,10 @@ class IVF_NM : public VecIndex, public OffsetBaseIndex {
#endif
int64_t
Count() override {
return index_->ntotal;
}
Count() override;
int64_t
Dim() override {
return index_->d;
}
Dim() override;
#if 0
DatasetPtr

6
core/src/index/knowhere/knowhere/index/vector_offset_index/IndexNSG_NM.cpp
View File

@ -151,11 +151,17 @@ NSG_NM::Train(const DatasetPtr& dataset_ptr, const Config& config) {
int64_t
NSG_NM::Count() {
if (!index_) {
KNOWHERE_THROW_MSG("index not initialize");
}
return index_->ntotal;
}
int64_t
NSG_NM::Dim() {
if (!index_) {
KNOWHERE_THROW_MSG("index not initialize");
}
return index_->dimension;
}

11
core/src/index/thirdparty/hnswlib/hnswalg_nm.h vendored
View File

@ -46,9 +46,6 @@ namespace hnswlib {
dist_func_param_ = s->get_dist_func_param();
M_ = M;
maxM_ = M_;
cmli_cnt_ = 0;
cmli_cnt2_ = 0;
cmli_time_ = 0.0;
maxM0_ = M_ * 2;
ef_construction_ = std::max(ef_construction,M_);
ef_ = 10;
@ -124,9 +121,6 @@ namespace hnswlib {
size_t M_;
size_t maxM_;
size_t maxM0_;
size_t cmli_cnt_;
size_t cmli_cnt2_;
double cmli_time_;
size_t ef_construction_;
double mult_, revSize_;
@ -497,13 +491,8 @@ namespace hnswlib {
dist_func_param_), data[j]);
}
if (candidates.size() >= Mcurmax) cmli_cnt2_ ++;
size_t indx = 0;
auto t0 = std::chrono::high_resolution_clock::now();
getNeighborsByHeuristic2(candidates, Mcurmax, data, indx, pdata);
auto t1 = std::chrono::high_resolution_clock::now();
cmli_time_ += (double)std::chrono::duration_cast<std::chrono::milliseconds>( t1 - t0 ).count();
cmli_cnt_ ++;
// while (candidates.size() > 0) {
// data[indx] = candidates.top().second;