diff --git a/core/src/db/insert/VectorSource.cpp b/core/src/db/insert/VectorSource.cpp index 2a88e1651c..ad659c19e7 100644 --- a/core/src/db/insert/VectorSource.cpp +++ b/core/src/db/insert/VectorSource.cpp @@ -18,6 +18,7 @@ #include "db/engine/ExecutionEngine.h" #include "metrics/Metrics.h" #include "utils/Log.h" +#include "utils/TimeRecorder.h" namespace milvus { namespace engine { @@ -36,9 +37,8 @@ VectorSource::VectorSource(milvus::engine::VectorsData vectors, } Status -VectorSource::Add(/*const ExecutionEnginePtr& execution_engine,*/ const segment::SegmentWriterPtr& segment_writer_ptr, - const meta::SegmentSchema& table_file_schema, const size_t& num_vectors_to_add, - size_t& num_vectors_added) { +VectorSource::Add(const segment::SegmentWriterPtr& segment_writer_ptr, const meta::SegmentSchema& table_file_schema, + const size_t& num_vectors_to_add, size_t& num_vectors_added) { uint64_t n = vectors_.vector_count_; server::CollectAddMetrics metrics(n, table_file_schema.dimension_); @@ -61,35 +61,17 @@ VectorSource::Add(/*const ExecutionEnginePtr& execution_engine,*/ const segment: Status status; if (!vectors_.float_data_.empty()) { - /* - status = execution_engine->AddWithIds( - num_vectors_added, vectors_.float_data_.data() + current_num_vectors_added * table_file_schema.dimension_, - vector_ids_to_add.data()); - */ - std::vector vectors; + LOG_ENGINE_DEBUG_ << LogOut("[%s][%ld]", "insert", 0) << "Insert float data into segment"; auto size = num_vectors_added * table_file_schema.dimension_ * sizeof(float); - vectors.resize(size); - memcpy(vectors.data(), vectors_.float_data_.data() + current_num_vectors_added * table_file_schema.dimension_, - size); - LOG_ENGINE_DEBUG_ << LogOut("[%s][%ld]", "insert", 0) << "Insert into segment"; - status = segment_writer_ptr->AddVectors(table_file_schema.file_id_, vectors, vector_ids_to_add); - + float* ptr = vectors_.float_data_.data() + current_num_vectors_added * table_file_schema.dimension_; + status = segment_writer_ptr->AddVectors(table_file_schema.file_id_, (uint8_t*)ptr, size, vector_ids_to_add); } else if (!vectors_.binary_data_.empty()) { - /* - status = execution_engine->AddWithIds( - num_vectors_added, - vectors_.binary_data_.data() + current_num_vectors_added * SingleVectorSize(table_file_schema.dimension_), - vector_ids_to_add.data()); - */ + LOG_ENGINE_DEBUG_ << LogOut("[%s][%ld]", "insert", 0) << "Insert binary data into segment"; std::vector vectors; auto size = num_vectors_added * SingleVectorSize(table_file_schema.dimension_) * sizeof(uint8_t); - vectors.resize(size); - memcpy( - vectors.data(), - vectors_.binary_data_.data() + current_num_vectors_added * SingleVectorSize(table_file_schema.dimension_), - size); - LOG_ENGINE_DEBUG_ << LogOut("[%s][%ld]", "insert", 0) << "Insert into segment"; - status = segment_writer_ptr->AddVectors(table_file_schema.file_id_, vectors, vector_ids_to_add); + uint8_t* ptr = + vectors_.binary_data_.data() + current_num_vectors_added * SingleVectorSize(table_file_schema.dimension_); + status = segment_writer_ptr->AddVectors(table_file_schema.file_id_, ptr, size, vector_ids_to_add); } // Clear vector data @@ -175,6 +157,7 @@ VectorSource::SingleVectorSize(uint16_t dimension) { return 0; } + size_t VectorSource::SingleEntitySize(uint16_t dimension) { // TODO(yukun) add entity type and size compute diff --git a/core/src/db/insert/VectorSource.h b/core/src/db/insert/VectorSource.h index 6b9f89e428..46aa249b33 100644 --- a/core/src/db/insert/VectorSource.h +++ b/core/src/db/insert/VectorSource.h @@ -36,8 +36,8 @@ class VectorSource { const std::unordered_map>& attr_data); Status - Add(/*const ExecutionEnginePtr& execution_engine,*/ const segment::SegmentWriterPtr& segment_writer_ptr, - const meta::SegmentSchema& table_file_schema, const size_t& num_vectors_to_add, size_t& num_vectors_added); + Add(const segment::SegmentWriterPtr& segment_writer_ptr, const meta::SegmentSchema& table_file_schema, + const size_t& num_vectors_to_add, size_t& num_vectors_added); Status AddEntities(const segment::SegmentWriterPtr& segment_writer_ptr, const meta::SegmentSchema& collection_file_schema, diff --git a/core/src/db/merge/MergeLayeredStrategy.cpp b/core/src/db/merge/MergeLayeredStrategy.cpp index 2a4e20f249..58dbc3ee5d 100644 --- a/core/src/db/merge/MergeLayeredStrategy.cpp +++ b/core/src/db/merge/MergeLayeredStrategy.cpp @@ -20,7 +20,7 @@ namespace milvus { namespace engine { -const int64_t FORCE_MERGE_THREASHOLD = 10; // force merge files older this time(in second) +const int64_t FORCE_MERGE_THREASHOLD = 30; // force merge files older this time(in second) Status MergeLayeredStrategy::RegroupFiles(meta::FilesHolder& files_holder, MergeFilesGroups& files_groups) { diff --git a/core/src/segment/SegmentWriter.cpp b/core/src/segment/SegmentWriter.cpp index 91416d0ebe..fd491fdc4d 100644 --- a/core/src/segment/SegmentWriter.cpp +++ b/core/src/segment/SegmentWriter.cpp @@ -52,6 +52,16 @@ SegmentWriter::AddVectors(const std::string& name, const std::vector& d return Status::OK(); } +Status +SegmentWriter::AddVectors(const std::string& name, const uint8_t* data, uint64_t size, + const std::vector& uids) { + segment_ptr_->vectors_ptr_->AddData(data, size); + segment_ptr_->vectors_ptr_->AddUids(uids); + segment_ptr_->vectors_ptr_->SetName(name); + + return Status::OK(); +} + Status SegmentWriter::AddAttrs(const std::string& name, const std::unordered_map& attr_nbytes, const std::unordered_map>& attr_data, diff --git a/core/src/segment/SegmentWriter.h b/core/src/segment/SegmentWriter.h index 5c45e782c1..d3be407075 100644 --- a/core/src/segment/SegmentWriter.h +++ b/core/src/segment/SegmentWriter.h @@ -36,6 +36,9 @@ class SegmentWriter { Status AddVectors(const std::string& name, const std::vector& data, const std::vector& uids); + Status + AddVectors(const std::string& name, const uint8_t* data, uint64_t size, const std::vector& uids); + Status AddAttrs(const std::string& name, const std::unordered_map& attr_nbytes, const std::unordered_map>& attr_data, const std::vector& uids); diff --git a/core/src/segment/Vectors.cpp b/core/src/segment/Vectors.cpp index d0a7622e2e..d34a10d196 100644 --- a/core/src/segment/Vectors.cpp +++ b/core/src/segment/Vectors.cpp @@ -34,6 +34,13 @@ Vectors::AddData(const std::vector& data) { data_.insert(data_.end(), std::make_move_iterator(data.begin()), std::make_move_iterator(data.end())); } +void +Vectors::AddData(const uint8_t* data, uint64_t size) { + int64_t old_size = data_.size(); + data_.resize(data_.size() + size); + memcpy(data_.data() + old_size, data, size); +} + void Vectors::AddUids(const std::vector& uids) { uids_.reserve(uids_.size() + uids.size()); diff --git a/core/src/segment/Vectors.h b/core/src/segment/Vectors.h index b5594cc957..cc7369398e 100644 --- a/core/src/segment/Vectors.h +++ b/core/src/segment/Vectors.h @@ -33,6 +33,9 @@ class Vectors { void AddData(const std::vector& data); + void + AddData(const uint8_t* data, uint64_t size); + void AddUids(const std::vector& uids);