From 28a0f1de01c2b0dd425fe39a9db53cf2b453c416 Mon Sep 17 00:00:00 2001 From: "yudong.cai" Date: Mon, 9 Mar 2020 16:19:51 +0800 Subject: [PATCH] #1548 move store/Directory.cpp to storage Signed-off-by: yudong.cai --- core/src/CMakeLists.txt | 3 -- core/src/codecs/DeletedDocsFormat.h | 6 +-- core/src/codecs/IdBloomFilterFormat.h | 8 +-- core/src/codecs/VectorsFormat.h | 10 ++-- .../default/DefaultDeletedDocsFormat.cpp | 9 ++-- .../codecs/default/DefaultDeletedDocsFormat.h | 4 +- .../default/DefaultIdBloomFilterFormat.cpp | 12 ++--- .../default/DefaultIdBloomFilterFormat.h | 8 +-- .../codecs/default/DefaultVectorsFormat.cpp | 16 +++--- .../src/codecs/default/DefaultVectorsFormat.h | 9 ++-- core/src/segment/SegmentReader.cpp | 14 ++--- core/src/segment/SegmentReader.h | 4 +- core/src/segment/SegmentWriter.cpp | 20 ++++---- core/src/segment/SegmentWriter.h | 4 +- core/src/storage/Operation.h | 51 +++++++++++++++++++ .../disk/DiskOperation.cpp} | 25 +++++---- .../disk/DiskOperation.h} | 22 ++++---- core/unittest/CMakeLists.txt | 3 -- 18 files changed, 138 insertions(+), 90 deletions(-) create mode 100644 core/src/storage/Operation.h rename core/src/{store/Directory.cpp => storage/disk/DiskOperation.cpp} (82%) rename core/src/{store/Directory.h => storage/disk/DiskOperation.h} (80%) diff --git a/core/src/CMakeLists.txt b/core/src/CMakeLists.txt index ef1af69eb9..173e1aa293 100644 --- a/core/src/CMakeLists.txt +++ b/core/src/CMakeLists.txt @@ -125,8 +125,6 @@ aux_source_directory(${MILVUS_ENGINE_SRC}/codecs/default codecs_default_files) aux_source_directory(${MILVUS_ENGINE_SRC}/segment segment_files) -aux_source_directory(${MILVUS_ENGINE_SRC}/store store_files) - set(engine_files ${CMAKE_CURRENT_SOURCE_DIR}/main.cpp ${cache_files} @@ -143,7 +141,6 @@ set(engine_files ${codecs_files} ${codecs_default_files} ${segment_files} - ${store_files} ) if (MILVUS_WITH_PROMETHEUS) diff --git a/core/src/codecs/DeletedDocsFormat.h b/core/src/codecs/DeletedDocsFormat.h index d5f263a238..1369664629 100644 --- a/core/src/codecs/DeletedDocsFormat.h +++ b/core/src/codecs/DeletedDocsFormat.h @@ -20,7 +20,7 @@ #include #include "segment/DeletedDocs.h" -#include "store/Directory.h" +#include "src/storage/disk/DiskOperation.h" namespace milvus { namespace codec { @@ -28,10 +28,10 @@ namespace codec { class DeletedDocsFormat { public: virtual void - read(const store::DirectoryPtr& directory_ptr, segment::DeletedDocsPtr& deleted_docs) = 0; + read(const storage::OperationPtr& directory_ptr, segment::DeletedDocsPtr& deleted_docs) = 0; virtual void - write(const store::DirectoryPtr& directory_ptr, const segment::DeletedDocsPtr& deleted_docs) = 0; + write(const storage::OperationPtr& directory_ptr, const segment::DeletedDocsPtr& deleted_docs) = 0; }; using DeletedDocsFormatPtr = std::shared_ptr; diff --git a/core/src/codecs/IdBloomFilterFormat.h b/core/src/codecs/IdBloomFilterFormat.h index 11cafac773..17736bfaa4 100644 --- a/core/src/codecs/IdBloomFilterFormat.h +++ b/core/src/codecs/IdBloomFilterFormat.h @@ -20,7 +20,7 @@ #include #include "segment/IdBloomFilter.h" -#include "store/Directory.h" +#include "src/storage/disk/DiskOperation.h" namespace milvus { namespace codec { @@ -28,13 +28,13 @@ namespace codec { class IdBloomFilterFormat { public: virtual void - read(const store::DirectoryPtr& directory_ptr, segment::IdBloomFilterPtr& id_bloom_filter_ptr) = 0; + read(const storage::OperationPtr& directory_ptr, segment::IdBloomFilterPtr& id_bloom_filter_ptr) = 0; virtual void - write(const store::DirectoryPtr& directory_ptr, const segment::IdBloomFilterPtr& id_bloom_filter_ptr) = 0; + write(const storage::OperationPtr& directory_ptr, const segment::IdBloomFilterPtr& id_bloom_filter_ptr) = 0; virtual void - create(const store::DirectoryPtr& directory_ptr, segment::IdBloomFilterPtr& id_bloom_filter_ptr) = 0; + create(const storage::OperationPtr& directory_ptr, segment::IdBloomFilterPtr& id_bloom_filter_ptr) = 0; }; using IdBloomFilterFormatPtr = std::shared_ptr; diff --git a/core/src/codecs/VectorsFormat.h b/core/src/codecs/VectorsFormat.h index cfb7504fd6..2fa77d628d 100644 --- a/core/src/codecs/VectorsFormat.h +++ b/core/src/codecs/VectorsFormat.h @@ -21,7 +21,7 @@ #include #include "segment/Vectors.h" -#include "store/Directory.h" +#include "src/storage/disk/DiskOperation.h" namespace milvus { namespace codec { @@ -29,16 +29,16 @@ namespace codec { class VectorsFormat { public: virtual void - read(const store::DirectoryPtr& directory_ptr, segment::VectorsPtr& vectors_read) = 0; + read(const storage::OperationPtr& directory_ptr, segment::VectorsPtr& vectors_read) = 0; virtual void - write(const store::DirectoryPtr& directory_ptr, const segment::VectorsPtr& vectors) = 0; + write(const storage::OperationPtr& directory_ptr, const segment::VectorsPtr& vectors) = 0; virtual void - read_uids(const store::DirectoryPtr& directory_ptr, std::vector& uids) = 0; + read_uids(const storage::OperationPtr& directory_ptr, std::vector& uids) = 0; virtual void - read_vectors(const store::DirectoryPtr& directory_ptr, off_t offset, size_t num_bytes, + read_vectors(const storage::OperationPtr& directory_ptr, off_t offset, size_t num_bytes, std::vector& raw_vectors) = 0; }; diff --git a/core/src/codecs/default/DefaultDeletedDocsFormat.cpp b/core/src/codecs/default/DefaultDeletedDocsFormat.cpp index 0b0039cd4f..1027e477a1 100644 --- a/core/src/codecs/default/DefaultDeletedDocsFormat.cpp +++ b/core/src/codecs/default/DefaultDeletedDocsFormat.cpp @@ -35,10 +35,10 @@ namespace milvus { namespace codec { void -DefaultDeletedDocsFormat::read(const store::DirectoryPtr& directory_ptr, segment::DeletedDocsPtr& deleted_docs) { +DefaultDeletedDocsFormat::read(const storage::OperationPtr& directory_ptr, segment::DeletedDocsPtr& deleted_docs) { const std::lock_guard lock(mutex_); - std::string dir_path = directory_ptr->GetDirPath(); + std::string dir_path = directory_ptr->GetDirectory(); const std::string del_file_path = dir_path + "/" + deleted_docs_filename_; int del_fd = open(del_file_path.c_str(), O_RDONLY, 00664); @@ -75,10 +75,11 @@ DefaultDeletedDocsFormat::read(const store::DirectoryPtr& directory_ptr, segment } void -DefaultDeletedDocsFormat::write(const store::DirectoryPtr& directory_ptr, const segment::DeletedDocsPtr& deleted_docs) { +DefaultDeletedDocsFormat::write(const storage::OperationPtr& directory_ptr, + const segment::DeletedDocsPtr& deleted_docs) { const std::lock_guard lock(mutex_); - std::string dir_path = directory_ptr->GetDirPath(); + std::string dir_path = directory_ptr->GetDirectory(); const std::string del_file_path = dir_path + "/" + deleted_docs_filename_; // Create a temporary file from the existing file diff --git a/core/src/codecs/default/DefaultDeletedDocsFormat.h b/core/src/codecs/default/DefaultDeletedDocsFormat.h index dd58297f35..f1e13ed574 100644 --- a/core/src/codecs/default/DefaultDeletedDocsFormat.h +++ b/core/src/codecs/default/DefaultDeletedDocsFormat.h @@ -30,10 +30,10 @@ class DefaultDeletedDocsFormat : public DeletedDocsFormat { DefaultDeletedDocsFormat() = default; void - read(const store::DirectoryPtr& directory_ptr, segment::DeletedDocsPtr& deleted_docs) override; + read(const storage::OperationPtr& directory_ptr, segment::DeletedDocsPtr& deleted_docs) override; void - write(const store::DirectoryPtr& directory_ptr, const segment::DeletedDocsPtr& deleted_docs) override; + write(const storage::OperationPtr& directory_ptr, const segment::DeletedDocsPtr& deleted_docs) override; // No copy and move DefaultDeletedDocsFormat(const DefaultDeletedDocsFormat&) = delete; diff --git a/core/src/codecs/default/DefaultIdBloomFilterFormat.cpp b/core/src/codecs/default/DefaultIdBloomFilterFormat.cpp index 7cbabcfb74..5eb759fd50 100644 --- a/core/src/codecs/default/DefaultIdBloomFilterFormat.cpp +++ b/core/src/codecs/default/DefaultIdBloomFilterFormat.cpp @@ -30,11 +30,11 @@ constexpr unsigned int bloom_filter_capacity = 500000; constexpr double bloom_filter_error_rate = 0.01; void -DefaultIdBloomFilterFormat::read(const store::DirectoryPtr& directory_ptr, +DefaultIdBloomFilterFormat::read(const storage::OperationPtr& directory_ptr, segment::IdBloomFilterPtr& id_bloom_filter_ptr) { const std::lock_guard lock(mutex_); - std::string dir_path = directory_ptr->GetDirPath(); + std::string dir_path = directory_ptr->GetDirectory(); const std::string bloom_filter_file_path = dir_path + "/" + bloom_filter_filename_; scaling_bloom_t* bloom_filter = new_scaling_bloom_from_file(bloom_filter_capacity, bloom_filter_error_rate, bloom_filter_file_path.c_str()); @@ -48,11 +48,11 @@ DefaultIdBloomFilterFormat::read(const store::DirectoryPtr& directory_ptr, } void -DefaultIdBloomFilterFormat::write(const store::DirectoryPtr& directory_ptr, +DefaultIdBloomFilterFormat::write(const storage::OperationPtr& directory_ptr, const segment::IdBloomFilterPtr& id_bloom_filter_ptr) { const std::lock_guard lock(mutex_); - std::string dir_path = directory_ptr->GetDirPath(); + std::string dir_path = directory_ptr->GetDirectory(); const std::string bloom_filter_file_path = dir_path + "/" + bloom_filter_filename_; if (scaling_bloom_flush(id_bloom_filter_ptr->GetBloomFilter()) == -1) { std::string err_msg = @@ -63,9 +63,9 @@ DefaultIdBloomFilterFormat::write(const store::DirectoryPtr& directory_ptr, } void -DefaultIdBloomFilterFormat::create(const store::DirectoryPtr& directory_ptr, +DefaultIdBloomFilterFormat::create(const storage::OperationPtr& directory_ptr, segment::IdBloomFilterPtr& id_bloom_filter_ptr) { - std::string dir_path = directory_ptr->GetDirPath(); + std::string dir_path = directory_ptr->GetDirectory(); const std::string bloom_filter_file_path = dir_path + "/" + bloom_filter_filename_; scaling_bloom_t* bloom_filter = new_scaling_bloom(bloom_filter_capacity, bloom_filter_error_rate, bloom_filter_file_path.c_str()); diff --git a/core/src/codecs/default/DefaultIdBloomFilterFormat.h b/core/src/codecs/default/DefaultIdBloomFilterFormat.h index 7a5c4d29fd..ceb669ffff 100644 --- a/core/src/codecs/default/DefaultIdBloomFilterFormat.h +++ b/core/src/codecs/default/DefaultIdBloomFilterFormat.h @@ -22,7 +22,7 @@ #include "codecs/IdBloomFilterFormat.h" #include "segment/IdBloomFilter.h" -#include "store/Directory.h" +#include "src/storage/disk/DiskOperation.h" namespace milvus { namespace codec { @@ -32,13 +32,13 @@ class DefaultIdBloomFilterFormat : public IdBloomFilterFormat { DefaultIdBloomFilterFormat() = default; void - read(const store::DirectoryPtr& directory_ptr, segment::IdBloomFilterPtr& id_bloom_filter_ptr) override; + read(const storage::OperationPtr& directory_ptr, segment::IdBloomFilterPtr& id_bloom_filter_ptr) override; void - write(const store::DirectoryPtr& directory_ptr, const segment::IdBloomFilterPtr& id_bloom_filter_ptr) override; + write(const storage::OperationPtr& directory_ptr, const segment::IdBloomFilterPtr& id_bloom_filter_ptr) override; void - create(const store::DirectoryPtr& directory_ptr, segment::IdBloomFilterPtr& id_bloom_filter_ptr) override; + create(const storage::OperationPtr& directory_ptr, segment::IdBloomFilterPtr& id_bloom_filter_ptr) override; // No copy and move DefaultIdBloomFilterFormat(const DefaultIdBloomFilterFormat&) = delete; diff --git a/core/src/codecs/default/DefaultVectorsFormat.cpp b/core/src/codecs/default/DefaultVectorsFormat.cpp index f32f0ae5ba..36936a07c0 100644 --- a/core/src/codecs/default/DefaultVectorsFormat.cpp +++ b/core/src/codecs/default/DefaultVectorsFormat.cpp @@ -102,10 +102,10 @@ DefaultVectorsFormat::read_uids_internal(const std::string& file_path, std::vect } void -DefaultVectorsFormat::read(const store::DirectoryPtr& directory_ptr, segment::VectorsPtr& vectors_read) { +DefaultVectorsFormat::read(const storage::OperationPtr& directory_ptr, segment::VectorsPtr& vectors_read) { const std::lock_guard lock(mutex_); - std::string dir_path = directory_ptr->GetDirPath(); + std::string dir_path = directory_ptr->GetDirectory(); if (!boost::filesystem::is_directory(dir_path)) { std::string err_msg = "Directory: " + dir_path + "does not exist"; ENGINE_LOG_ERROR << err_msg; @@ -134,10 +134,10 @@ DefaultVectorsFormat::read(const store::DirectoryPtr& directory_ptr, segment::Ve } void -DefaultVectorsFormat::write(const store::DirectoryPtr& directory_ptr, const segment::VectorsPtr& vectors) { +DefaultVectorsFormat::write(const storage::OperationPtr& directory_ptr, const segment::VectorsPtr& vectors) { const std::lock_guard lock(mutex_); - std::string dir_path = directory_ptr->GetDirPath(); + std::string dir_path = directory_ptr->GetDirectory(); const std::string rv_file_path = dir_path + "/" + vectors->GetName() + raw_vector_extension_; const std::string uid_file_path = dir_path + "/" + vectors->GetName() + user_id_extension_; @@ -197,10 +197,10 @@ DefaultVectorsFormat::write(const store::DirectoryPtr& directory_ptr, const segm } void -DefaultVectorsFormat::read_uids(const store::DirectoryPtr& directory_ptr, std::vector& uids) { +DefaultVectorsFormat::read_uids(const storage::OperationPtr& directory_ptr, std::vector& uids) { const std::lock_guard lock(mutex_); - std::string dir_path = directory_ptr->GetDirPath(); + std::string dir_path = directory_ptr->GetDirectory(); if (!boost::filesystem::is_directory(dir_path)) { std::string err_msg = "Directory: " + dir_path + "does not exist"; ENGINE_LOG_ERROR << err_msg; @@ -221,11 +221,11 @@ DefaultVectorsFormat::read_uids(const store::DirectoryPtr& directory_ptr, std::v } void -DefaultVectorsFormat::read_vectors(const store::DirectoryPtr& directory_ptr, off_t offset, size_t num_bytes, +DefaultVectorsFormat::read_vectors(const storage::OperationPtr& directory_ptr, off_t offset, size_t num_bytes, std::vector& raw_vectors) { const std::lock_guard lock(mutex_); - std::string dir_path = directory_ptr->GetDirPath(); + std::string dir_path = directory_ptr->GetDirectory(); if (!boost::filesystem::is_directory(dir_path)) { std::string err_msg = "Directory: " + dir_path + "does not exist"; ENGINE_LOG_ERROR << err_msg; diff --git a/core/src/codecs/default/DefaultVectorsFormat.h b/core/src/codecs/default/DefaultVectorsFormat.h index 54c9b5278d..1fd2081fb9 100644 --- a/core/src/codecs/default/DefaultVectorsFormat.h +++ b/core/src/codecs/default/DefaultVectorsFormat.h @@ -32,16 +32,17 @@ class DefaultVectorsFormat : public VectorsFormat { DefaultVectorsFormat() = default; void - read(const store::DirectoryPtr&, segment::VectorsPtr&) override; + read(const storage::OperationPtr& directory_ptr, segment::VectorsPtr& vectors_read) override; void - write(const store::DirectoryPtr&, const segment::VectorsPtr&) override; + write(const storage::OperationPtr& directory_ptr, const segment::VectorsPtr& vectors) override; void - read_vectors(const store::DirectoryPtr&, off_t, size_t, std::vector&) override; + read_uids(const storage::OperationPtr& directory_ptr, std::vector& uids) override; void - read_uids(const store::DirectoryPtr&, std::vector&) override; + read_vectors(const storage::OperationPtr& directory_ptr, off_t offset, size_t num_bytes, + std::vector& raw_vectors) override; // No copy and move DefaultVectorsFormat(const DefaultVectorsFormat&) = delete; diff --git a/core/src/segment/SegmentReader.cpp b/core/src/segment/SegmentReader.cpp index c0983a89be..6bbf497ba1 100644 --- a/core/src/segment/SegmentReader.cpp +++ b/core/src/segment/SegmentReader.cpp @@ -21,14 +21,14 @@ #include "Vectors.h" #include "codecs/default/DefaultCodec.h" -#include "store/Directory.h" +#include "src/storage/disk/DiskOperation.h" #include "utils/Log.h" namespace milvus { namespace segment { SegmentReader::SegmentReader(const std::string& directory) { - directory_ptr_ = std::make_shared(directory); + directory_ptr_ = std::make_shared(directory); segment_ptr_ = std::make_shared(); } @@ -43,7 +43,7 @@ SegmentReader::Load() { // TODO(zhiru) codec::DefaultCodec default_codec; try { - directory_ptr_->Create(); + directory_ptr_->CreateDirectory(); default_codec.GetVectorsFormat()->read(directory_ptr_, segment_ptr_->vectors_ptr_); default_codec.GetDeletedDocsFormat()->read(directory_ptr_, segment_ptr_->deleted_docs_ptr_); } catch (std::exception& e) { @@ -56,7 +56,7 @@ Status SegmentReader::LoadVectors(off_t offset, size_t num_bytes, std::vector& raw_vectors) { codec::DefaultCodec default_codec; try { - directory_ptr_->Create(); + directory_ptr_->CreateDirectory(); default_codec.GetVectorsFormat()->read_vectors(directory_ptr_, offset, num_bytes, raw_vectors); } catch (std::exception& e) { std::string err_msg = "Failed to load raw vectors: " + std::string(e.what()); @@ -70,7 +70,7 @@ Status SegmentReader::LoadUids(std::vector& uids) { codec::DefaultCodec default_codec; try { - directory_ptr_->Create(); + directory_ptr_->CreateDirectory(); default_codec.GetVectorsFormat()->read_uids(directory_ptr_, uids); } catch (std::exception& e) { std::string err_msg = "Failed to load uids: " + std::string(e.what()); @@ -90,7 +90,7 @@ Status SegmentReader::LoadBloomFilter(segment::IdBloomFilterPtr& id_bloom_filter_ptr) { codec::DefaultCodec default_codec; try { - directory_ptr_->Create(); + directory_ptr_->CreateDirectory(); default_codec.GetIdBloomFilterFormat()->read(directory_ptr_, id_bloom_filter_ptr); } catch (std::exception& e) { std::string err_msg = "Failed to load bloom filter: " + std::string(e.what()); @@ -104,7 +104,7 @@ Status SegmentReader::LoadDeletedDocs(segment::DeletedDocsPtr& deleted_docs_ptr) { codec::DefaultCodec default_codec; try { - directory_ptr_->Create(); + directory_ptr_->CreateDirectory(); default_codec.GetDeletedDocsFormat()->read(directory_ptr_, deleted_docs_ptr); } catch (std::exception& e) { std::string err_msg = "Failed to load deleted docs: " + std::string(e.what()); diff --git a/core/src/segment/SegmentReader.h b/core/src/segment/SegmentReader.h index 95531d5685..7a41a7939a 100644 --- a/core/src/segment/SegmentReader.h +++ b/core/src/segment/SegmentReader.h @@ -22,7 +22,7 @@ #include #include "segment/Types.h" -#include "store/Directory.h" +#include "src/storage/disk/DiskOperation.h" #include "utils/Status.h" namespace milvus { @@ -55,7 +55,7 @@ class SegmentReader { GetSegment(SegmentPtr& segment_ptr); private: - store::DirectoryPtr directory_ptr_; + storage::OperationPtr directory_ptr_; SegmentPtr segment_ptr_; }; diff --git a/core/src/segment/SegmentWriter.cpp b/core/src/segment/SegmentWriter.cpp index c80776e2f5..fa210ae36e 100644 --- a/core/src/segment/SegmentWriter.cpp +++ b/core/src/segment/SegmentWriter.cpp @@ -23,14 +23,14 @@ #include "SegmentReader.h" #include "Vectors.h" #include "codecs/default/DefaultCodec.h" -#include "store/Directory.h" +#include "src/storage/disk/DiskOperation.h" #include "utils/Log.h" namespace milvus { namespace segment { SegmentWriter::SegmentWriter(const std::string& directory) { - directory_ptr_ = std::make_shared(directory); + directory_ptr_ = std::make_shared(directory); segment_ptr_ = std::make_shared(); } @@ -84,7 +84,7 @@ Status SegmentWriter::WriteVectors() { codec::DefaultCodec default_codec; try { - directory_ptr_->Create(); + directory_ptr_->CreateDirectory(); default_codec.GetVectorsFormat()->write(directory_ptr_, segment_ptr_->vectors_ptr_); } catch (std::exception& e) { std::string err_msg = "Failed to write vectors: " + std::string(e.what()); @@ -98,7 +98,7 @@ Status SegmentWriter::WriteBloomFilter() { codec::DefaultCodec default_codec; try { - directory_ptr_->Create(); + directory_ptr_->CreateDirectory(); auto start = std::chrono::high_resolution_clock::now(); @@ -138,7 +138,7 @@ Status SegmentWriter::WriteDeletedDocs() { codec::DefaultCodec default_codec; try { - directory_ptr_->Create(); + directory_ptr_->CreateDirectory(); DeletedDocsPtr deleted_docs_ptr = std::make_shared(); default_codec.GetDeletedDocsFormat()->write(directory_ptr_, deleted_docs_ptr); } catch (std::exception& e) { @@ -153,7 +153,7 @@ Status SegmentWriter::WriteDeletedDocs(const DeletedDocsPtr& deleted_docs) { codec::DefaultCodec default_codec; try { - directory_ptr_->Create(); + directory_ptr_->CreateDirectory(); default_codec.GetDeletedDocsFormat()->write(directory_ptr_, deleted_docs); } catch (std::exception& e) { std::string err_msg = "Failed to write deleted docs: " + std::string(e.what()); @@ -167,7 +167,7 @@ Status SegmentWriter::WriteBloomFilter(const IdBloomFilterPtr& id_bloom_filter_ptr) { codec::DefaultCodec default_codec; try { - directory_ptr_->Create(); + directory_ptr_->CreateDirectory(); default_codec.GetIdBloomFilterFormat()->write(directory_ptr_, id_bloom_filter_ptr); } catch (std::exception& e) { std::string err_msg = "Failed to write bloom filter: " + std::string(e.what()); @@ -191,11 +191,11 @@ SegmentWriter::GetSegment(SegmentPtr& segment_ptr) { Status SegmentWriter::Merge(const std::string& dir_to_merge, const std::string& name) { - if (dir_to_merge == directory_ptr_->GetDirPath()) { + if (dir_to_merge == directory_ptr_->GetDirectory()) { return Status(DB_ERROR, "Cannot Merge Self"); } - ENGINE_LOG_DEBUG << "Merging from " << dir_to_merge << " to " << directory_ptr_->GetDirPath(); + ENGINE_LOG_DEBUG << "Merging from " << dir_to_merge << " to " << directory_ptr_->GetDirectory(); auto start = std::chrono::high_resolution_clock::now(); @@ -234,7 +234,7 @@ SegmentWriter::Merge(const std::string& dir_to_merge, const std::string& name) { ENGINE_LOG_DEBUG << "Adding " << segment_to_merge->vectors_ptr_->GetCount() << " vectors and uids took " << diff.count() << " s"; - ENGINE_LOG_DEBUG << "Merging completed from " << dir_to_merge << " to " << directory_ptr_->GetDirPath(); + ENGINE_LOG_DEBUG << "Merging completed from " << dir_to_merge << " to " << directory_ptr_->GetDirectory(); return Status::OK(); } diff --git a/core/src/segment/SegmentWriter.h b/core/src/segment/SegmentWriter.h index f7b6a7398c..a430cda9b8 100644 --- a/core/src/segment/SegmentWriter.h +++ b/core/src/segment/SegmentWriter.h @@ -22,7 +22,7 @@ #include #include "segment/Types.h" -#include "store/Directory.h" +#include "src/storage/disk/DiskOperation.h" #include "utils/Status.h" namespace milvus { @@ -70,7 +70,7 @@ class SegmentWriter { WriteDeletedDocs(); private: - store::DirectoryPtr directory_ptr_; + storage::OperationPtr directory_ptr_; SegmentPtr segment_ptr_; }; diff --git a/core/src/storage/Operation.h b/core/src/storage/Operation.h new file mode 100644 index 0000000000..a29069c7da --- /dev/null +++ b/core/src/storage/Operation.h @@ -0,0 +1,51 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include + +namespace milvus { +namespace storage { + +class Operation { + public: + virtual void + CreateDirectory() = 0; + + virtual const std::string& + GetDirectory() const = 0; + + virtual void + ListDirectory(std::vector& file_paths) = 0; + + virtual bool + DeleteFile(const std::string& file_path) = 0; + + // TODO(zhiru): + // open(), sync(), close() + // function that opens a stream for reading file + // function that creates a new, empty file and returns an stream for appending data to this file + // function that creates a new, empty, temporary file and returns an stream for appending data to this file +}; + +using OperationPtr = std::shared_ptr; + +} // namespace storage +} // namespace milvus diff --git a/core/src/store/Directory.cpp b/core/src/storage/disk/DiskOperation.cpp similarity index 82% rename from core/src/store/Directory.cpp rename to core/src/storage/disk/DiskOperation.cpp index 1d41da004f..90a36c2bb2 100644 --- a/core/src/store/Directory.cpp +++ b/core/src/storage/disk/DiskOperation.cpp @@ -15,21 +15,20 @@ // specific language governing permissions and limitations // under the License. -#include "store/Directory.h" - #include +#include "storage/disk/DiskOperation.h" #include "utils/Exception.h" #include "utils/Log.h" namespace milvus { -namespace store { +namespace storage { -Directory::Directory(const std::string& dir_path) : dir_path_(dir_path) { +DiskOperation::DiskOperation(const std::string& dir_path) : dir_path_(dir_path) { } void -Directory::Create() { +DiskOperation::CreateDirectory() { if (!boost::filesystem::is_directory(dir_path_)) { auto ret = boost::filesystem::create_directory(dir_path_); if (!ret) { @@ -40,8 +39,13 @@ Directory::Create() { } } +const std::string& +DiskOperation::GetDirectory() const { + return dir_path_; +} + void -Directory::ListAll(std::vector& file_paths) { +DiskOperation::ListDirectory(std::vector& file_paths) { boost::filesystem::path target_path(dir_path_); typedef boost::filesystem::directory_iterator d_it; d_it it_end; @@ -54,14 +58,9 @@ Directory::ListAll(std::vector& file_paths) { } bool -Directory::DeleteFile(const std::string& file_path) { +DiskOperation::DeleteFile(const std::string& file_path) { return boost::filesystem::remove(file_path); } -const std::string& -Directory::GetDirPath() const { - return dir_path_; -} - -} // namespace store +} // namespace storage } // namespace milvus diff --git a/core/src/store/Directory.h b/core/src/storage/disk/DiskOperation.h similarity index 80% rename from core/src/store/Directory.h rename to core/src/storage/disk/DiskOperation.h index de1dc276ac..50c1a4ba3d 100644 --- a/core/src/store/Directory.h +++ b/core/src/storage/disk/DiskOperation.h @@ -21,25 +21,27 @@ #include #include +#include "storage/Operation.h" + namespace milvus { -namespace store { +namespace storage { -class Directory { +class DiskOperation : public Operation { public: - explicit Directory(const std::string& dir_path); + explicit DiskOperation(const std::string& dir_path); void - Create(); + CreateDirectory(); + + const std::string& + GetDirectory() const; void - ListAll(std::vector& file_paths); + ListDirectory(std::vector& file_paths); bool DeleteFile(const std::string& file_path); - const std::string& - GetDirPath() const; - // TODO(zhiru): // open(), sync(), close() // function that opens a stream for reading file @@ -50,7 +52,7 @@ class Directory { const std::string dir_path_; }; -using DirectoryPtr = std::shared_ptr; +using DiskOperationPtr = std::shared_ptr; -} // namespace store +} // namespace storage } // namespace milvus diff --git a/core/unittest/CMakeLists.txt b/core/unittest/CMakeLists.txt index cdc2f033f9..6674fdf032 100644 --- a/core/unittest/CMakeLists.txt +++ b/core/unittest/CMakeLists.txt @@ -111,8 +111,6 @@ aux_source_directory(${MILVUS_ENGINE_SRC}/codecs/default codecs_default_files) aux_source_directory(${MILVUS_ENGINE_SRC}/segment segment_files) -aux_source_directory(${MILVUS_ENGINE_SRC}/store store_files) - set(entry_file ${CMAKE_CURRENT_SOURCE_DIR}/main.cpp) @@ -146,7 +144,6 @@ set(common_files ${codecs_files} ${codecs_default_files} ${segment_files} - ${store_files} ) set(unittest_libs