admin/milvus
1
0
Fork 0
mirror of https://gitee.com/milvus-io/milvus.git synced 2026-01-04 09:52:30 +08:00
Code Issues Packages Projects Releases Wiki Activity

refactor(db): replace faiss with serializer wrapper

Browse Source 
Former-commit-id: 62b5a7d8353c2b0ef4017f43d03d1c6944685400
This commit is contained in:
Xu Peng 2019-04-30 15:58:00 +08:00
parent 713d1ec6b7
commit 204c68879c
2 changed files with 10 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
cpp/src/db/MemManager.cpp
View File

@ -1,14 +1,12 @@
#include <faiss/AutoTune.h>
#include <iostream>
#include <sstream>
#include <thread>
#include <wrapper/Index.h>
#include <cache/CpuCacheMgr.h>
#include <easylogging++.h>
#include "MemManager.h"
#include "Meta.h"
#include "FaissSerializer.h"
namespace zilliz {
@ -21,43 +19,36 @@ MemVectors::MemVectors(const std::shared_ptr<meta::Meta>& meta_ptr,
options_(options),
schema_(schema),
_pIdGenerator(new SimpleIDGenerator()),
pIndex_(faiss::index_factory(schema_.dimension, "IDMap,Flat")) {
pSerializer_(new FaissSerializer(schema_.dimension, schema_.location)) {
}
void MemVectors::add(size_t n_, const float* vectors_, IDNumbers& vector_ids_) {
_pIdGenerator->getNextIDNumbers(n_, vector_ids_);
pIndex_->add_with_ids(n_, vectors_, &vector_ids_[0]);
pSerializer_->AddWithIds(n_, vectors_, vector_ids_.data());
for(auto i=0 ; i<n_; i++) {
vector_ids_.push_back(i);
}
}
size_t MemVectors::total() const {
return pIndex_->ntotal;
return pSerializer_->Count();
}
size_t MemVectors::approximate_size() const {
return total() * schema_.dimension;
return pSerializer_->Size();
}
Status MemVectors::serialize(std::string& group_id) {
/* std::stringstream ss; */
/* ss << "/tmp/test/" << _pIdGenerator->getNextIDNumber(); */
/* faiss::write_index(pIndex_, ss.str().c_str()); */
/* std::cout << pIndex_->ntotal << std::endl; */
/* std::cout << _file_location << std::endl; */
/* faiss::write_index(pIndex_, _file_location.c_str()); */
group_id = schema_.group_id;
auto rows = approximate_size();
write_index(pIndex_.get(), schema_.location.c_str());
pSerializer_->Serialize();
schema_.rows = rows;
schema_.file_type = (rows >= options_.index_trigger_size) ?
meta::GroupFileSchema::TO_INDEX : meta::GroupFileSchema::RAW;
auto status = pMeta_->update_group_file(schema_);
zilliz::vecwise::cache::CpuCacheMgr::GetInstance(
)->InsertItem(schema_.location, std::make_shared<Index>(pIndex_));
pSerializer_->Cache();
return status;
}

8
cpp/src/db/MemManager.h
View File

@ -10,10 +10,6 @@
#include "Status.h"
#include "Meta.h"
namespace faiss {
class Index;
}
namespace zilliz {
namespace vecwise {
@ -23,6 +19,8 @@ namespace meta {
class Meta;
}
class Serializer;
class MemVectors {
public:
explicit MemVectors(const std::shared_ptr<meta::Meta>&,
@ -49,7 +47,7 @@ private:
Options options_;
meta::GroupFileSchema schema_;
IDGenerator* _pIdGenerator;
std::shared_ptr<faiss::Index> pIndex_;
std::shared_ptr<Serializer> pSerializer_;
}; // MemVectors