admin/milvus
1
0
Fork 0
mirror of https://gitee.com/milvus-io/milvus.git synced 2025-12-28 22:45:26 +08:00
Code Issues Packages Projects Releases Wiki Activity

#1548 move store/Directory.cpp to storage

Browse Source 
Signed-off-by: yudong.cai <yudong.cai@zilliz.com>
This commit is contained in:
yudong.cai 2020-03-09 16:19:51 +08:00
parent 4b9dc0263b
commit 28a0f1de01
18 changed files with 138 additions and 90 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
core/src/CMakeLists.txt
View File

@ -125,8 +125,6 @@ aux_source_directory(${MILVUS_ENGINE_SRC}/codecs/default codecs_default_files)
aux_source_directory(${MILVUS_ENGINE_SRC}/segment segment_files)
aux_source_directory(${MILVUS_ENGINE_SRC}/store store_files)
set(engine_files
${CMAKE_CURRENT_SOURCE_DIR}/main.cpp
${cache_files}
@ -143,7 +141,6 @@ set(engine_files
${codecs_files}
${codecs_default_files}
${segment_files}
${store_files}
)
if (MILVUS_WITH_PROMETHEUS)

6
core/src/codecs/DeletedDocsFormat.h
View File

@ -20,7 +20,7 @@
#include <memory>
#include "segment/DeletedDocs.h"
#include "store/Directory.h"
#include "src/storage/disk/DiskOperation.h"
namespace milvus {
namespace codec {
@ -28,10 +28,10 @@ namespace codec {
class DeletedDocsFormat {
public:
virtual void
read(const store::DirectoryPtr& directory_ptr, segment::DeletedDocsPtr& deleted_docs) = 0;
read(const storage::OperationPtr& directory_ptr, segment::DeletedDocsPtr& deleted_docs) = 0;
virtual void
write(const store::DirectoryPtr& directory_ptr, const segment::DeletedDocsPtr& deleted_docs) = 0;
write(const storage::OperationPtr& directory_ptr, const segment::DeletedDocsPtr& deleted_docs) = 0;
};
using DeletedDocsFormatPtr = std::shared_ptr<DeletedDocsFormat>;

8
core/src/codecs/IdBloomFilterFormat.h
View File

@ -20,7 +20,7 @@
#include <memory>
#include "segment/IdBloomFilter.h"
#include "store/Directory.h"
#include "src/storage/disk/DiskOperation.h"
namespace milvus {
namespace codec {
@ -28,13 +28,13 @@ namespace codec {
class IdBloomFilterFormat {
public:
virtual void
read(const store::DirectoryPtr& directory_ptr, segment::IdBloomFilterPtr& id_bloom_filter_ptr) = 0;
read(const storage::OperationPtr& directory_ptr, segment::IdBloomFilterPtr& id_bloom_filter_ptr) = 0;
virtual void
write(const store::DirectoryPtr& directory_ptr, const segment::IdBloomFilterPtr& id_bloom_filter_ptr) = 0;
write(const storage::OperationPtr& directory_ptr, const segment::IdBloomFilterPtr& id_bloom_filter_ptr) = 0;
virtual void
create(const store::DirectoryPtr& directory_ptr, segment::IdBloomFilterPtr& id_bloom_filter_ptr) = 0;
create(const storage::OperationPtr& directory_ptr, segment::IdBloomFilterPtr& id_bloom_filter_ptr) = 0;
};
using IdBloomFilterFormatPtr = std::shared_ptr<IdBloomFilterFormat>;

10
core/src/codecs/VectorsFormat.h
View File

@ -21,7 +21,7 @@
#include <vector>
#include "segment/Vectors.h"
#include "store/Directory.h"
#include "src/storage/disk/DiskOperation.h"
namespace milvus {
namespace codec {
@ -29,16 +29,16 @@ namespace codec {
class VectorsFormat {
public:
virtual void
read(const store::DirectoryPtr& directory_ptr, segment::VectorsPtr& vectors_read) = 0;
read(const storage::OperationPtr& directory_ptr, segment::VectorsPtr& vectors_read) = 0;
virtual void
write(const store::DirectoryPtr& directory_ptr, const segment::VectorsPtr& vectors) = 0;
write(const storage::OperationPtr& directory_ptr, const segment::VectorsPtr& vectors) = 0;
virtual void
read_uids(const store::DirectoryPtr& directory_ptr, std::vector<segment::doc_id_t>& uids) = 0;
read_uids(const storage::OperationPtr& directory_ptr, std::vector<segment::doc_id_t>& uids) = 0;
virtual void
read_vectors(const store::DirectoryPtr& directory_ptr, off_t offset, size_t num_bytes,
read_vectors(const storage::OperationPtr& directory_ptr, off_t offset, size_t num_bytes,
std::vector<uint8_t>& raw_vectors) = 0;
};

9
core/src/codecs/default/DefaultDeletedDocsFormat.cpp
View File

@ -35,10 +35,10 @@ namespace milvus {
namespace codec {
void
DefaultDeletedDocsFormat::read(const store::DirectoryPtr& directory_ptr, segment::DeletedDocsPtr& deleted_docs) {
DefaultDeletedDocsFormat::read(const storage::OperationPtr& directory_ptr, segment::DeletedDocsPtr& deleted_docs) {
const std::lock_guard<std::mutex> lock(mutex_);
std::string dir_path = directory_ptr->GetDirPath();
std::string dir_path = directory_ptr->GetDirectory();
const std::string del_file_path = dir_path + "/" + deleted_docs_filename_;
int del_fd = open(del_file_path.c_str(), O_RDONLY, 00664);
@ -75,10 +75,11 @@ DefaultDeletedDocsFormat::read(const store::DirectoryPtr& directory_ptr, segment
}
void
DefaultDeletedDocsFormat::write(const store::DirectoryPtr& directory_ptr, const segment::DeletedDocsPtr& deleted_docs) {
DefaultDeletedDocsFormat::write(const storage::OperationPtr& directory_ptr,
const segment::DeletedDocsPtr& deleted_docs) {
const std::lock_guard<std::mutex> lock(mutex_);
std::string dir_path = directory_ptr->GetDirPath();
std::string dir_path = directory_ptr->GetDirectory();
const std::string del_file_path = dir_path + "/" + deleted_docs_filename_;
// Create a temporary file from the existing file

4
core/src/codecs/default/DefaultDeletedDocsFormat.h
View File

@ -30,10 +30,10 @@ class DefaultDeletedDocsFormat : public DeletedDocsFormat {
DefaultDeletedDocsFormat() = default;
void
read(const store::DirectoryPtr& directory_ptr, segment::DeletedDocsPtr& deleted_docs) override;
read(const storage::OperationPtr& directory_ptr, segment::DeletedDocsPtr& deleted_docs) override;
void
write(const store::DirectoryPtr& directory_ptr, const segment::DeletedDocsPtr& deleted_docs) override;
write(const storage::OperationPtr& directory_ptr, const segment::DeletedDocsPtr& deleted_docs) override;
// No copy and move
DefaultDeletedDocsFormat(const DefaultDeletedDocsFormat&) = delete;

12
core/src/codecs/default/DefaultIdBloomFilterFormat.cpp
View File

@ -30,11 +30,11 @@ constexpr unsigned int bloom_filter_capacity = 500000;
constexpr double bloom_filter_error_rate = 0.01;
void
DefaultIdBloomFilterFormat::read(const store::DirectoryPtr& directory_ptr,
DefaultIdBloomFilterFormat::read(const storage::OperationPtr& directory_ptr,
segment::IdBloomFilterPtr& id_bloom_filter_ptr) {
const std::lock_guard<std::mutex> lock(mutex_);
std::string dir_path = directory_ptr->GetDirPath();
std::string dir_path = directory_ptr->GetDirectory();
const std::string bloom_filter_file_path = dir_path + "/" + bloom_filter_filename_;
scaling_bloom_t* bloom_filter =
new_scaling_bloom_from_file(bloom_filter_capacity, bloom_filter_error_rate, bloom_filter_file_path.c_str());
@ -48,11 +48,11 @@ DefaultIdBloomFilterFormat::read(const store::DirectoryPtr& directory_ptr,
}
void
DefaultIdBloomFilterFormat::write(const store::DirectoryPtr& directory_ptr,
DefaultIdBloomFilterFormat::write(const storage::OperationPtr& directory_ptr,
const segment::IdBloomFilterPtr& id_bloom_filter_ptr) {
const std::lock_guard<std::mutex> lock(mutex_);
std::string dir_path = directory_ptr->GetDirPath();
std::string dir_path = directory_ptr->GetDirectory();
const std::string bloom_filter_file_path = dir_path + "/" + bloom_filter_filename_;
if (scaling_bloom_flush(id_bloom_filter_ptr->GetBloomFilter()) == -1) {
std::string err_msg =
@ -63,9 +63,9 @@ DefaultIdBloomFilterFormat::write(const store::DirectoryPtr& directory_ptr,
}
void
DefaultIdBloomFilterFormat::create(const store::DirectoryPtr& directory_ptr,
DefaultIdBloomFilterFormat::create(const storage::OperationPtr& directory_ptr,
segment::IdBloomFilterPtr& id_bloom_filter_ptr) {
std::string dir_path = directory_ptr->GetDirPath();
std::string dir_path = directory_ptr->GetDirectory();
const std::string bloom_filter_file_path = dir_path + "/" + bloom_filter_filename_;
scaling_bloom_t* bloom_filter =
new_scaling_bloom(bloom_filter_capacity, bloom_filter_error_rate, bloom_filter_file_path.c_str());

8
core/src/codecs/default/DefaultIdBloomFilterFormat.h
View File

@ -22,7 +22,7 @@
#include "codecs/IdBloomFilterFormat.h"
#include "segment/IdBloomFilter.h"
#include "store/Directory.h"
#include "src/storage/disk/DiskOperation.h"
namespace milvus {
namespace codec {
@ -32,13 +32,13 @@ class DefaultIdBloomFilterFormat : public IdBloomFilterFormat {
DefaultIdBloomFilterFormat() = default;
void
read(const store::DirectoryPtr& directory_ptr, segment::IdBloomFilterPtr& id_bloom_filter_ptr) override;
read(const storage::OperationPtr& directory_ptr, segment::IdBloomFilterPtr& id_bloom_filter_ptr) override;
void
write(const store::DirectoryPtr& directory_ptr, const segment::IdBloomFilterPtr& id_bloom_filter_ptr) override;
write(const storage::OperationPtr& directory_ptr, const segment::IdBloomFilterPtr& id_bloom_filter_ptr) override;
void
create(const store::DirectoryPtr& directory_ptr, segment::IdBloomFilterPtr& id_bloom_filter_ptr) override;
create(const storage::OperationPtr& directory_ptr, segment::IdBloomFilterPtr& id_bloom_filter_ptr) override;
// No copy and move
DefaultIdBloomFilterFormat(const DefaultIdBloomFilterFormat&) = delete;

16
core/src/codecs/default/DefaultVectorsFormat.cpp
View File

@ -102,10 +102,10 @@ DefaultVectorsFormat::read_uids_internal(const std::string& file_path, std::vect
}
void
DefaultVectorsFormat::read(const store::DirectoryPtr& directory_ptr, segment::VectorsPtr& vectors_read) {
DefaultVectorsFormat::read(const storage::OperationPtr& directory_ptr, segment::VectorsPtr& vectors_read) {
const std::lock_guard<std::mutex> lock(mutex_);
std::string dir_path = directory_ptr->GetDirPath();
std::string dir_path = directory_ptr->GetDirectory();
if (!boost::filesystem::is_directory(dir_path)) {
std::string err_msg = "Directory: " + dir_path + "does not exist";
ENGINE_LOG_ERROR << err_msg;
@ -134,10 +134,10 @@ DefaultVectorsFormat::read(const store::DirectoryPtr& directory_ptr, segment::Ve
}
void
DefaultVectorsFormat::write(const store::DirectoryPtr& directory_ptr, const segment::VectorsPtr& vectors) {
DefaultVectorsFormat::write(const storage::OperationPtr& directory_ptr, const segment::VectorsPtr& vectors) {
const std::lock_guard<std::mutex> lock(mutex_);
std::string dir_path = directory_ptr->GetDirPath();
std::string dir_path = directory_ptr->GetDirectory();
const std::string rv_file_path = dir_path + "/" + vectors->GetName() + raw_vector_extension_;
const std::string uid_file_path = dir_path + "/" + vectors->GetName() + user_id_extension_;
@ -197,10 +197,10 @@ DefaultVectorsFormat::write(const store::DirectoryPtr& directory_ptr, const segm
}
void
DefaultVectorsFormat::read_uids(const store::DirectoryPtr& directory_ptr, std::vector<segment::doc_id_t>& uids) {
DefaultVectorsFormat::read_uids(const storage::OperationPtr& directory_ptr, std::vector<segment::doc_id_t>& uids) {
const std::lock_guard<std::mutex> lock(mutex_);
std::string dir_path = directory_ptr->GetDirPath();
std::string dir_path = directory_ptr->GetDirectory();
if (!boost::filesystem::is_directory(dir_path)) {
std::string err_msg = "Directory: " + dir_path + "does not exist";
ENGINE_LOG_ERROR << err_msg;
@ -221,11 +221,11 @@ DefaultVectorsFormat::read_uids(const store::DirectoryPtr& directory_ptr, std::v
}
void
DefaultVectorsFormat::read_vectors(const store::DirectoryPtr& directory_ptr, off_t offset, size_t num_bytes,
DefaultVectorsFormat::read_vectors(const storage::OperationPtr& directory_ptr, off_t offset, size_t num_bytes,
std::vector<uint8_t>& raw_vectors) {
const std::lock_guard<std::mutex> lock(mutex_);
std::string dir_path = directory_ptr->GetDirPath();
std::string dir_path = directory_ptr->GetDirectory();
if (!boost::filesystem::is_directory(dir_path)) {
std::string err_msg = "Directory: " + dir_path + "does not exist";
ENGINE_LOG_ERROR << err_msg;

9
core/src/codecs/default/DefaultVectorsFormat.h
View File

@ -32,16 +32,17 @@ class DefaultVectorsFormat : public VectorsFormat {
DefaultVectorsFormat() = default;
void
read(const store::DirectoryPtr&, segment::VectorsPtr&) override;
read(const storage::OperationPtr& directory_ptr, segment::VectorsPtr& vectors_read) override;
void
write(const store::DirectoryPtr&, const segment::VectorsPtr&) override;
write(const storage::OperationPtr& directory_ptr, const segment::VectorsPtr& vectors) override;
void
read_vectors(const store::DirectoryPtr&, off_t, size_t, std::vector<uint8_t>&) override;
read_uids(const storage::OperationPtr& directory_ptr, std::vector<segment::doc_id_t>& uids) override;
void
read_uids(const store::DirectoryPtr&, std::vector<segment::doc_id_t>&) override;
read_vectors(const storage::OperationPtr& directory_ptr, off_t offset, size_t num_bytes,
std::vector<uint8_t>& raw_vectors) override;
// No copy and move
DefaultVectorsFormat(const DefaultVectorsFormat&) = delete;

14
core/src/segment/SegmentReader.cpp
View File

@ -21,14 +21,14 @@
#include "Vectors.h"
#include "codecs/default/DefaultCodec.h"
#include "store/Directory.h"
#include "src/storage/disk/DiskOperation.h"
#include "utils/Log.h"
namespace milvus {
namespace segment {
SegmentReader::SegmentReader(const std::string& directory) {
directory_ptr_ = std::make_shared<store::Directory>(directory);
directory_ptr_ = std::make_shared<storage::DiskOperation>(directory);
segment_ptr_ = std::make_shared<Segment>();
}
@ -43,7 +43,7 @@ SegmentReader::Load() {
// TODO(zhiru)
codec::DefaultCodec default_codec;
try {
directory_ptr_->Create();
directory_ptr_->CreateDirectory();
default_codec.GetVectorsFormat()->read(directory_ptr_, segment_ptr_->vectors_ptr_);
default_codec.GetDeletedDocsFormat()->read(directory_ptr_, segment_ptr_->deleted_docs_ptr_);
} catch (std::exception& e) {
@ -56,7 +56,7 @@ Status
SegmentReader::LoadVectors(off_t offset, size_t num_bytes, std::vector<uint8_t>& raw_vectors) {
codec::DefaultCodec default_codec;
try {
directory_ptr_->Create();
directory_ptr_->CreateDirectory();
default_codec.GetVectorsFormat()->read_vectors(directory_ptr_, offset, num_bytes, raw_vectors);
} catch (std::exception& e) {
std::string err_msg = "Failed to load raw vectors: " + std::string(e.what());
@ -70,7 +70,7 @@ Status
SegmentReader::LoadUids(std::vector<doc_id_t>& uids) {
codec::DefaultCodec default_codec;
try {
directory_ptr_->Create();
directory_ptr_->CreateDirectory();
default_codec.GetVectorsFormat()->read_uids(directory_ptr_, uids);
} catch (std::exception& e) {
std::string err_msg = "Failed to load uids: " + std::string(e.what());
@ -90,7 +90,7 @@ Status
SegmentReader::LoadBloomFilter(segment::IdBloomFilterPtr& id_bloom_filter_ptr) {
codec::DefaultCodec default_codec;
try {
directory_ptr_->Create();
directory_ptr_->CreateDirectory();
default_codec.GetIdBloomFilterFormat()->read(directory_ptr_, id_bloom_filter_ptr);
} catch (std::exception& e) {
std::string err_msg = "Failed to load bloom filter: " + std::string(e.what());
@ -104,7 +104,7 @@ Status
SegmentReader::LoadDeletedDocs(segment::DeletedDocsPtr& deleted_docs_ptr) {
codec::DefaultCodec default_codec;
try {
directory_ptr_->Create();
directory_ptr_->CreateDirectory();
default_codec.GetDeletedDocsFormat()->read(directory_ptr_, deleted_docs_ptr);
} catch (std::exception& e) {
std::string err_msg = "Failed to load deleted docs: " + std::string(e.what());

4
core/src/segment/SegmentReader.h
View File

@ -22,7 +22,7 @@
#include <vector>
#include "segment/Types.h"
#include "store/Directory.h"
#include "src/storage/disk/DiskOperation.h"
#include "utils/Status.h"
namespace milvus {
@ -55,7 +55,7 @@ class SegmentReader {
GetSegment(SegmentPtr& segment_ptr);
private:
store::DirectoryPtr directory_ptr_;
storage::OperationPtr directory_ptr_;
SegmentPtr segment_ptr_;
};

20
core/src/segment/SegmentWriter.cpp
View File

@ -23,14 +23,14 @@
#include "SegmentReader.h"
#include "Vectors.h"
#include "codecs/default/DefaultCodec.h"
#include "store/Directory.h"
#include "src/storage/disk/DiskOperation.h"
#include "utils/Log.h"
namespace milvus {
namespace segment {
SegmentWriter::SegmentWriter(const std::string& directory) {
directory_ptr_ = std::make_shared<store::Directory>(directory);
directory_ptr_ = std::make_shared<storage::DiskOperation>(directory);
segment_ptr_ = std::make_shared<Segment>();
}
@ -84,7 +84,7 @@ Status
SegmentWriter::WriteVectors() {
codec::DefaultCodec default_codec;
try {
directory_ptr_->Create();
directory_ptr_->CreateDirectory();
default_codec.GetVectorsFormat()->write(directory_ptr_, segment_ptr_->vectors_ptr_);
} catch (std::exception& e) {
std::string err_msg = "Failed to write vectors: " + std::string(e.what());
@ -98,7 +98,7 @@ Status
SegmentWriter::WriteBloomFilter() {
codec::DefaultCodec default_codec;
try {
directory_ptr_->Create();
directory_ptr_->CreateDirectory();
auto start = std::chrono::high_resolution_clock::now();
@ -138,7 +138,7 @@ Status
SegmentWriter::WriteDeletedDocs() {
codec::DefaultCodec default_codec;
try {
directory_ptr_->Create();
directory_ptr_->CreateDirectory();
DeletedDocsPtr deleted_docs_ptr = std::make_shared<DeletedDocs>();
default_codec.GetDeletedDocsFormat()->write(directory_ptr_, deleted_docs_ptr);
} catch (std::exception& e) {
@ -153,7 +153,7 @@ Status
SegmentWriter::WriteDeletedDocs(const DeletedDocsPtr& deleted_docs) {
codec::DefaultCodec default_codec;
try {
directory_ptr_->Create();
directory_ptr_->CreateDirectory();
default_codec.GetDeletedDocsFormat()->write(directory_ptr_, deleted_docs);
} catch (std::exception& e) {
std::string err_msg = "Failed to write deleted docs: " + std::string(e.what());
@ -167,7 +167,7 @@ Status
SegmentWriter::WriteBloomFilter(const IdBloomFilterPtr& id_bloom_filter_ptr) {
codec::DefaultCodec default_codec;
try {
directory_ptr_->Create();
directory_ptr_->CreateDirectory();
default_codec.GetIdBloomFilterFormat()->write(directory_ptr_, id_bloom_filter_ptr);
} catch (std::exception& e) {
std::string err_msg = "Failed to write bloom filter: " + std::string(e.what());
@ -191,11 +191,11 @@ SegmentWriter::GetSegment(SegmentPtr& segment_ptr) {
Status
SegmentWriter::Merge(const std::string& dir_to_merge, const std::string& name) {
if (dir_to_merge == directory_ptr_->GetDirPath()) {
if (dir_to_merge == directory_ptr_->GetDirectory()) {
return Status(DB_ERROR, "Cannot Merge Self");
}
ENGINE_LOG_DEBUG << "Merging from " << dir_to_merge << " to " << directory_ptr_->GetDirPath();
ENGINE_LOG_DEBUG << "Merging from " << dir_to_merge << " to " << directory_ptr_->GetDirectory();
auto start = std::chrono::high_resolution_clock::now();
@ -234,7 +234,7 @@ SegmentWriter::Merge(const std::string& dir_to_merge, const std::string& name) {
ENGINE_LOG_DEBUG << "Adding " << segment_to_merge->vectors_ptr_->GetCount() << " vectors and uids took "
<< diff.count() << " s";
ENGINE_LOG_DEBUG << "Merging completed from " << dir_to_merge << " to " << directory_ptr_->GetDirPath();
ENGINE_LOG_DEBUG << "Merging completed from " << dir_to_merge << " to " << directory_ptr_->GetDirectory();
return Status::OK();
}

4
core/src/segment/SegmentWriter.h
View File

@ -22,7 +22,7 @@
#include <vector>
#include "segment/Types.h"
#include "store/Directory.h"
#include "src/storage/disk/DiskOperation.h"
#include "utils/Status.h"
namespace milvus {
@ -70,7 +70,7 @@ class SegmentWriter {
WriteDeletedDocs();
private:
store::DirectoryPtr directory_ptr_;
storage::OperationPtr directory_ptr_;
SegmentPtr segment_ptr_;
};

51
core/src/storage/Operation.h Normal file
View File

@ -0,0 +1,51 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#pragma once
#include <memory>
#include <string>
#include <vector>
namespace milvus {
namespace storage {
class Operation {
public:
virtual void
CreateDirectory() = 0;
virtual const std::string&
GetDirectory() const = 0;
virtual void
ListDirectory(std::vector<std::string>& file_paths) = 0;
virtual bool
DeleteFile(const std::string& file_path) = 0;
// TODO(zhiru):
// open(), sync(), close()
// function that opens a stream for reading file
// function that creates a new, empty file and returns an stream for appending data to this file
// function that creates a new, empty, temporary file and returns an stream for appending data to this file
};
using OperationPtr = std::shared_ptr<Operation>;
} // namespace storage
} // namespace milvus

25
core/src/store/Directory.cpp → core/src/storage/disk/DiskOperation.cpp
View File

@ -15,21 +15,20 @@
// specific language governing permissions and limitations
// under the License.
#include "store/Directory.h"
#include <boost/filesystem.hpp>
#include "storage/disk/DiskOperation.h"
#include "utils/Exception.h"
#include "utils/Log.h"
namespace milvus {
namespace store {
namespace storage {
Directory::Directory(const std::string& dir_path) : dir_path_(dir_path) {
DiskOperation::DiskOperation(const std::string& dir_path) : dir_path_(dir_path) {
}
void
Directory::Create() {
DiskOperation::CreateDirectory() {
if (!boost::filesystem::is_directory(dir_path_)) {
auto ret = boost::filesystem::create_directory(dir_path_);
if (!ret) {
@ -40,8 +39,13 @@ Directory::Create() {
}
}
const std::string&
DiskOperation::GetDirectory() const {
return dir_path_;
}
void
Directory::ListAll(std::vector<std::string>& file_paths) {
DiskOperation::ListDirectory(std::vector<std::string>& file_paths) {
boost::filesystem::path target_path(dir_path_);
typedef boost::filesystem::directory_iterator d_it;
d_it it_end;
@ -54,14 +58,9 @@ Directory::ListAll(std::vector<std::string>& file_paths) {
}
bool
Directory::DeleteFile(const std::string& file_path) {
DiskOperation::DeleteFile(const std::string& file_path) {
return boost::filesystem::remove(file_path);
}
const std::string&
Directory::GetDirPath() const {
return dir_path_;
}
} // namespace store
} // namespace storage
} // namespace milvus

22
core/src/store/Directory.h → core/src/storage/disk/DiskOperation.h
View File

@ -21,25 +21,27 @@
#include <string>
#include <vector>
#include "storage/Operation.h"
namespace milvus {
namespace store {
namespace storage {
class Directory {
class DiskOperation : public Operation {
public:
explicit Directory(const std::string& dir_path);
explicit DiskOperation(const std::string& dir_path);
void
Create();
CreateDirectory();
const std::string&
GetDirectory() const;
void
ListAll(std::vector<std::string>& file_paths);
ListDirectory(std::vector<std::string>& file_paths);
bool
DeleteFile(const std::string& file_path);
const std::string&
GetDirPath() const;
// TODO(zhiru):
// open(), sync(), close()
// function that opens a stream for reading file
@ -50,7 +52,7 @@ class Directory {
const std::string dir_path_;
};
using DirectoryPtr = std::shared_ptr<Directory>;
using DiskOperationPtr = std::shared_ptr<DiskOperation>;
} // namespace store
} // namespace storage
} // namespace milvus

3
core/unittest/CMakeLists.txt
View File

@ -111,8 +111,6 @@ aux_source_directory(${MILVUS_ENGINE_SRC}/codecs/default codecs_default_files)
aux_source_directory(${MILVUS_ENGINE_SRC}/segment segment_files)
aux_source_directory(${MILVUS_ENGINE_SRC}/store store_files)
set(entry_file
${CMAKE_CURRENT_SOURCE_DIR}/main.cpp)
@ -146,7 +144,6 @@ set(common_files
${codecs_files}
${codecs_default_files}
${segment_files}
${store_files}
)
set(unittest_libs