admin/milvus
1
0
Fork 0
mirror of https://gitee.com/milvus-io/milvus.git synced 2026-01-01 00:15:30 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'branch-0.3.1-yuncong' into 'branch-0.3.1-yuncong'

Browse Source 
MS-217 Fix SQ8 row count bug

See merge request megasearch/milvus!198

Former-commit-id: 43f141e0b099dfe34cbbe3b84b8a77befa6e443b
This commit is contained in:
peng.xu 2019-07-14 11:45:31 +08:00
commit 489d8039dc
6 changed files with 23 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
cpp/CHANGELOG.md
View File

@ -9,8 +9,9 @@ Please mark all change in change log and use the ticket from JIRA.
- MS-148 - Disable cleanup if mode is read only
- MS-149 - Fixed searching only one index file issue in distributed mode
- MS-153 - fix c_str error when connecting to MySQL
- MS-157 - fix changelog
- MS-153 - Fix c_str error when connecting to MySQL
- MS-157 - Fix changelog
- MS-217 - Fix SQ8 row count bug
## Improvement
- MS-156 - Add unittest for merge result functions

6
cpp/conf/server_config.template
View File

@ -14,8 +14,8 @@ db_config:
db_backend_url: sqlite://:@:/
index_building_threshold: 1024 # index building trigger threshold, default: 1024, unit: MB
archive_disk_threshold: 512 # triger archive action if storage size exceed this value, unit: GB
archive_days_threshold: 30 # files older than x days will be archived, unit: day
archive_disk_threshold: 0 # triger archive action if storage size exceed this value, 0 means no limit, unit: GB
archive_days_threshold: 0 # files older than x days will be archived, 0 means no limit, unit: day
metric_config:
is_startup: off # if monitoring start: on, off
@ -37,4 +37,4 @@ engine_config:
nprobe: 10
nlist: 16384
use_blas_threshold: 20
metric_type: L2 #L2 or Inner Product
metric_type: L2 # compare vectors by euclidean distance(L2) or inner product(IP), optional: L2 or IP

2
cpp/src/db/DBImpl.cpp
View File

@ -487,7 +487,7 @@ Status DBImpl::BuildIndex(const meta::TableFileSchema& file) {
//step 6: update meta
table_file.file_type_ = meta::TableFileSchema::INDEX;
table_file.size_ = index->PhysicalSize();
table_file.size_ = index->Size();
auto to_remove = file;
to_remove.file_type_ = meta::TableFileSchema::TO_DELETE;

22
cpp/src/db/DBMetaImpl.cpp
View File

@ -674,16 +674,22 @@ Status DBMetaImpl::Archive() {
Status DBMetaImpl::Size(uint64_t &result) {
result = 0;
try {
auto selected = ConnectorPtr->select(columns(sum(&TableFileSchema::size_)),
where(
c(&TableFileSchema::file_type_) != (int) TableFileSchema::TO_DELETE
));
auto files = ConnectorPtr->select(columns(&TableFileSchema::size_,
&TableFileSchema::file_type_,
&TableFileSchema::engine_type_),
where(
c(&TableFileSchema::file_type_) != (int) TableFileSchema::TO_DELETE
));
for (auto &sub_query : selected) {
if (!std::get<0>(sub_query)) {
continue;
for (auto &file : files) {
auto file_size = std::get<0>(file);
auto file_type = std::get<1>(file);
auto engine_type = std::get<2>(file);
if(file_type == (int)TableFileSchema::INDEX && engine_type == (int)EngineType::FAISS_IVFSQ8) {
result += (uint64_t)file_size/4;//hardcode for sq8
} else {
result += (uint64_t)file_size;
}
result += (uint64_t) (*std::get<0>(sub_query));
}
} catch (std::exception &e) {
return HandleException("Encounter exception when calculte db size", e);

2
cpp/src/db/FaissExecutionEngine.cpp
View File

@ -110,7 +110,7 @@ Status FaissExecutionEngine::Merge(const std::string& location) {
if (location == location_) {
return Status::Error("Cannot Merge Self");
}
ENGINE_LOG_DEBUG << "Merge index file: " << location << " to: " << location_;
ENGINE_LOG_DEBUG << "Merge raw file: " << location << " to: " << location_;
auto to_merge = zilliz::milvus::cache::CpuCacheMgr::GetInstance()->GetIndex(location);
if (!to_merge) {

2
cpp/src/db/Options.h
View File

@ -22,7 +22,7 @@ static constexpr uint64_t ONE_GB = ONE_KB*ONE_MB;
static const std::string ARCHIVE_CONF_DISK = "disk";
static const std::string ARCHIVE_CONF_DAYS = "days";
static const std::string ARCHIVE_CONF_DEFAULT = ARCHIVE_CONF_DISK + ":512";
static const std::string ARCHIVE_CONF_DEFAULT = "";
struct ArchiveConf {
using CriteriaT = std::map<std::string, int>;