admin/milvus
1
0
Fork 0
mirror of https://gitee.com/milvus-io/milvus.git synced 2025-12-31 07:55:38 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(db): add simple delete partitions

Browse Source 
Former-commit-id: 61aeee06c7d928939e6d3d28013630464cf0e046
This commit is contained in:
Xu Peng 2019-05-19 21:50:24 +08:00
parent 5f4e16f3d7
commit 5a406d6ea0
9 changed files with 79 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
cpp/src/db/DB.h
View File

@ -23,6 +23,8 @@ public:
virtual Status add_group(meta::GroupSchema& group_info_) = 0;
virtual Status get_group(meta::GroupSchema& group_info_) = 0;
virtual Status delete_vectors(const std::string& group_id,
const meta::DatesT& dates) = 0;
virtual Status has_group(const std::string& group_id_, bool& has_or_not_) = 0;
virtual Status get_group_files(const std::string& group_id_,
const int date_delta_,

6
cpp/src/db/DBImpl.cpp
View File

@ -44,6 +44,12 @@ Status DBImpl<EngineT>::get_group(meta::GroupSchema& group_info) {
return _pMeta->get_group(group_info);
}
template<typename EngineT>
Status DBImpl<EngineT>::delete_vectors(const std::string& group_id,
const meta::DatesT& dates) {
return _pMeta->delete_group_partitions(group_id, dates);
}
template<typename EngineT>
Status DBImpl<EngineT>::has_group(const std::string& group_id_, bool& has_or_not_) {
return _pMeta->has_group(group_id_, has_or_not_);

1
cpp/src/db/DBImpl.h
View File

@ -35,6 +35,7 @@ public:
virtual Status add_group(meta::GroupSchema& group_info) override;
virtual Status get_group(meta::GroupSchema& group_info) override;
virtual Status delete_vectors(const std::string& group_id, const meta::DatesT& dates) override;
virtual Status has_group(const std::string& group_id_, bool& has_or_not_) override;
virtual Status get_group_files(const std::string& group_id_,

38
cpp/src/db/DBMetaImpl.cpp
View File

@ -104,6 +104,44 @@ Status DBMetaImpl::initialize() {
return Status::OK();
}
// PXU TODO: Temp solution. Will fix later
Status DBMetaImpl::delete_group_partitions(const std::string& group_id,
const meta::DatesT& dates) {
if (dates.size() == 0) {
return Status::OK();
}
GroupSchema group_info;
group_info.group_id = group_id;
auto status = get_group(group_info);
if (!status.ok()) {
return status;
}
auto yesterday = GetDate(-2);
for (auto& date : dates) {
if (date >= yesterday) {
return Status::Error("Could not delete partitions with 2 days");
}
}
try {
ConnectorPtr->update_all(
set(
c(&GroupFileSchema::file_type) = (int)GroupFileSchema::TO_DELETE
),
where(
c(&GroupFileSchema::group_id) == group_id and
in(&GroupFileSchema::date, dates)
));
} catch (std::exception & e) {
LOG(DEBUG) << e.what();
throw e;
}
return Status::OK();
}
Status DBMetaImpl::add_group(GroupSchema& group_info) {
if (group_info.group_id == "") {
std::stringstream ss;

2
cpp/src/db/DBMetaImpl.h
View File

@ -24,6 +24,8 @@ public:
virtual Status has_group(const std::string& group_id_, bool& has_or_not_) override;
virtual Status add_group_file(GroupFileSchema& group_file_info) override;
virtual Status delete_group_partitions(const std::string& group_id,
const meta::DatesT& dates) override;
virtual Status has_group_file(const std::string& group_id_,
const std::string& file_id_,

4
cpp/src/db/LocalMetaImpl.h
View File

@ -22,7 +22,9 @@ public:
virtual Status get_group(GroupSchema& group_info_) override;
virtual Status has_group(const std::string& group_id_, bool& has_or_not_) override;
virtual Status add_group_file(GroupFileSchema& group_file_info) = 0;
virtual Status add_group_file(GroupFileSchema& group_file_info) override;
/* virtual Status delete_group_partitions(const std::string& group_id, */
/* const meta::DatesT& dates) override; */
virtual Status has_group_file(const std::string& group_id_,
const std::string& file_id_,

21
cpp/src/db/Meta.cpp
View File

@ -11,13 +11,28 @@ namespace vecwise {
namespace engine {
namespace meta {
DateT Meta::GetDate(const std::time_t& t) {
DateT Meta::GetDate(const std::time_t& t, int day_delta) {
tm *ltm = std::localtime(&t);
if (day_delta > 0) {
do {
++ltm->tm_mday;
--day_delta;
} while(day_delta > 0);
mktime(ltm);
} else if (day_delta < 0) {
do {
--ltm->tm_mday;
++day_delta;
} while(day_delta < 0);
mktime(ltm);
} else {
ltm->tm_mday;
}
return ltm->tm_year*10000 + ltm->tm_mon*100 + ltm->tm_mday;
}
DateT Meta::GetDate() {
return GetDate(std::time(nullptr));
DateT Meta::GetDate(int day_delta) {
return GetDate(std::time(nullptr), day_delta);
}
} // namespace meta

6
cpp/src/db/Meta.h
View File

@ -67,6 +67,8 @@ public:
virtual Status has_group(const std::string& group_id_, bool& has_or_not_) = 0;
virtual Status add_group_file(GroupFileSchema& group_file_info) = 0;
virtual Status delete_group_partitions(const std::string& group_id,
const meta::DatesT& dates) = 0;
virtual Status has_group_file(const std::string& group_id_,
const std::string& file_id_,
@ -98,8 +100,8 @@ public:
virtual Status count(const std::string& group_id, long& result) = 0;
static DateT GetDate(const std::time_t& t);
static DateT GetDate();
static DateT GetDate(const std::time_t& t, int day_delta);
static DateT GetDate(int day_delta = 0);
}; // MetaData

5
cpp/unittest/db/meta_tests.cpp
View File

@ -59,6 +59,11 @@ TEST_F(MetaTest, GROUP_FILE_TEST) {
ASSERT_TRUE(status.ok());
ASSERT_EQ(group_file.file_type, new_file_type);
meta::DatesT dates;
dates.push_back(meta::Meta::GetDate());
status = impl_->delete_group_partitions(group_file.group_id, dates);
ASSERT_FALSE(status.ok());
/* group_file.file_type = meta::GroupFileSchema::NEW; */
/* status = impl_->get_group_file(group_file.group_id, group_file.file_id, group_file); */
/* ASSERT_TRUE(status.ok()); */