From 9a17db48bc691dca98160ba206c54540c2ccbc31 Mon Sep 17 00:00:00 2001 From: jielinxu <52057195+jielinxu@users.noreply.github.com> Date: Thu, 28 Nov 2019 17:15:21 +0800 Subject: [PATCH 01/44] Add docker compilation images --- install.md | 88 +++++++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 81 insertions(+), 7 deletions(-) diff --git a/install.md b/install.md index f53586af36..854326f664 100644 --- a/install.md +++ b/install.md @@ -1,6 +1,11 @@ # Install Milvus from Source Code -## Software requirements +- [Build from source](https://github.com/jielinxu/milvus/edit/0.6.0/install.md#build-from-source) +- [Compile Milvus on Docker](https://github.com/jielinxu/milvus/edit/0.6.0/install.md#compile-milvus-on-docker) + +## Build from source + +### Requirements - Ubuntu 18.04 or higher @@ -8,21 +13,21 @@ - CMake 3.12 or higher -##### For GPU version, you will also need: +##### For GPU-enabled version, you will also need: - CUDA 10.0 or higher - NVIDIA driver 418 or higher -## Compilation +### Compilation -### Step 1 Install dependencies +#### Step 1 Install dependencies ```shell $ cd [Milvus root path]/core $ ./ubuntu_build_deps.sh ``` -### Step 2 Build +#### Step 2 Build ```shell $ cd [Milvus root path]/core @@ -31,7 +36,7 @@ or $ ./build.sh -t Release ``` -By default, it will build CPU version. To build GPU version, add `-g` option +By default, it will build CPU-only version. To build GPU version, add `-g` option ```shell $ ./build.sh -g ``` @@ -43,7 +48,7 @@ $./build.sh -h When the build is completed, all the stuff that you need in order to run Milvus will be installed under `[Milvus root path]/core/milvus`. -## Launch Milvus server +### Launch Milvus server ```shell $ cd [Milvus root path]/core/milvus @@ -68,6 +73,75 @@ To stop Milvus server, run: $ ./stop_server.sh ``` +## Compile Milvus on Docker + +With this dockerfile, you should be able to compile CPU-only or GPU-enabled Milvus on any Linux platform that run Docker. To build a GPU-enabled Milvus, you neeed to install [NVIDIA Docker](https://github.com/NVIDIA/nvidia-docker/) first. + +### Step 1 Pull Milvus Docker images + +Pull CPU-only image: + +```shell +$ Docker pull milvusdb/milvus-cpu-build-env:v0.6.0-ubuntu18.04 +``` + +Pull GPU-enabled image: + +```shell +$ Docker pull milvusdb/milvus-gpu-build-env:v0.6.0-ubuntu18.04 +``` +### Step 2 Start the Docker container + +Start a CPU-only container: + +```shell +$ Docker run -it -p 19530:19530 -d milvusdb/milvus-cpu-build-env:v0.6.0-ubuntu18.04 +``` + +Start a GPU container: + +```shell +$ Docker run — runtime=nvidia -it -p 19530:19530 -d milvusdb/milvus-gpu-build-env:v0.6.0-ubuntu18.04 +``` +To enter the container: + +```shell +$ Docker exec -it [container_id] bash +``` +### Step 3 Download Milvus source code and compile it in the container + +Download Milvus source code: + +```shell +$ cd /home +$ wget https://github.com/milvus-io/milvus/archive/0.6.0.zip +``` + +Unzip the source package: + +```shell +$ Apt-get update +$ Apt-get install unzip +$ unzip ./0.6.0.zip +``` + +The source code is extracted into a folder called `milvus-0.6.0`. + +If you are using a CPU-only image, compile it like this: +```shell +$ ./build.sh -t Release +``` + +For GPU-enabled image, you need to add a `-g` parameter: +```shell +$ ./build.sh -g -t Release +``` + +Then start Milvus server： +```shell +$ ./start_server.sh +``` + ## Troubleshooting 1. If you encounter the following error when compiling: `protocol https not supported or disabled in libcurl`. From 534d0558291497fc7f09bf5d5ca5725ce4fcf7fc Mon Sep 17 00:00:00 2001 From: jielinxu <52057195+jielinxu@users.noreply.github.com> Date: Thu, 28 Nov 2019 17:18:01 +0800 Subject: [PATCH 02/44] Update install.md --- install.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/install.md b/install.md index 854326f664..4bdb4a6127 100644 --- a/install.md +++ b/install.md @@ -1,7 +1,7 @@ # Install Milvus from Source Code -- [Build from source](https://github.com/jielinxu/milvus/edit/0.6.0/install.md#build-from-source) -- [Compile Milvus on Docker](https://github.com/jielinxu/milvus/edit/0.6.0/install.md#compile-milvus-on-docker) +- [Build from source](#build-from-source) +- [Compile Milvus on Docker](#compile-milvus-on-docker) ## Build from source From add279dadc1d628c0eea24218b6e40802c54434a Mon Sep 17 00:00:00 2001 From: jielinxu <52057195+jielinxu@users.noreply.github.com> Date: Thu, 28 Nov 2019 17:33:52 +0800 Subject: [PATCH 03/44] Update install.md --- install.md | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/install.md b/install.md index 4bdb4a6127..b7293c788c 100644 --- a/install.md +++ b/install.md @@ -75,9 +75,9 @@ $ ./stop_server.sh ## Compile Milvus on Docker -With this dockerfile, you should be able to compile CPU-only or GPU-enabled Milvus on any Linux platform that run Docker. To build a GPU-enabled Milvus, you neeed to install [NVIDIA Docker](https://github.com/NVIDIA/nvidia-docker/) first. +With this dockerfile, you should be able to compile Milvus on any Linux platform that run Docker. To build a GPU supported Milvus, you neeed to install [NVIDIA Docker](https://github.com/NVIDIA/nvidia-docker/) first. -### Step 1 Pull Milvus Docker images +#### Step 1 Pull Milvus Docker images Pull CPU-only image: @@ -90,7 +90,7 @@ Pull GPU-enabled image: ```shell $ Docker pull milvusdb/milvus-gpu-build-env:v0.6.0-ubuntu18.04 ``` -### Step 2 Start the Docker container +#### Step 2 Start the Docker container Start a CPU-only container: @@ -108,7 +108,7 @@ To enter the container: ```shell $ Docker exec -it [container_id] bash ``` -### Step 3 Download Milvus source code and compile it in the container +#### Step 3 Download Milvus source code Download Milvus source code: @@ -125,14 +125,19 @@ $ Apt-get install unzip $ unzip ./0.6.0.zip ``` -The source code is extracted into a folder called `milvus-0.6.0`. +The source code is extracted into a folder called `milvus-0.6.0`. To enter its core directory: + +```shell +$ cd ./milvus-0.6.0/core +``` +#### Step 4 Compile Milvus in the container If you are using a CPU-only image, compile it like this: ```shell $ ./build.sh -t Release ``` -For GPU-enabled image, you need to add a `-g` parameter: +If you are using a GPU-enabled image, you need to add a `-g` parameter: ```shell $ ./build.sh -g -t Release ``` From c438fa77be4aeab69ef76051799a8710e8b9ae58 Mon Sep 17 00:00:00 2001 From: jielinxu <52057195+jielinxu@users.noreply.github.com> Date: Thu, 28 Nov 2019 17:34:45 +0800 Subject: [PATCH 04/44] Update install.md --- install.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/install.md b/install.md index b7293c788c..0e4701f6cf 100644 --- a/install.md +++ b/install.md @@ -77,7 +77,7 @@ $ ./stop_server.sh With this dockerfile, you should be able to compile Milvus on any Linux platform that run Docker. To build a GPU supported Milvus, you neeed to install [NVIDIA Docker](https://github.com/NVIDIA/nvidia-docker/) first. -#### Step 1 Pull Milvus Docker images +### Step 1 Pull Milvus Docker images Pull CPU-only image: @@ -90,7 +90,7 @@ Pull GPU-enabled image: ```shell $ Docker pull milvusdb/milvus-gpu-build-env:v0.6.0-ubuntu18.04 ``` -#### Step 2 Start the Docker container +### Step 2 Start the Docker container Start a CPU-only container: @@ -108,7 +108,7 @@ To enter the container: ```shell $ Docker exec -it [container_id] bash ``` -#### Step 3 Download Milvus source code +### Step 3 Download Milvus source code Download Milvus source code: @@ -130,7 +130,7 @@ The source code is extracted into a folder called `milvus-0.6.0`. To enter its c ```shell $ cd ./milvus-0.6.0/core ``` -#### Step 4 Compile Milvus in the container +### Step 4 Compile Milvus in the container If you are using a CPU-only image, compile it like this: ```shell From 627576d575f24c29e08f72c751fb580374c99ea5 Mon Sep 17 00:00:00 2001 From: jielinxu <52057195+jielinxu@users.noreply.github.com> Date: Thu, 28 Nov 2019 17:36:21 +0800 Subject: [PATCH 05/44] Update install.md --- install.md | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/install.md b/install.md index 0e4701f6cf..48ad3d7577 100644 --- a/install.md +++ b/install.md @@ -152,9 +152,10 @@ $ ./start_server.sh `protocol https not supported or disabled in libcurl`. First, make sure you have `libcurl4-openssl-dev` installed in your system. Then try reinstall CMake from source with `--system-curl` option: -```shell -$ ./bootstrap --system-curl -$ make -$ sudo make install -``` + + ```shell + $ ./bootstrap --system-curl + $ make + $ sudo make install + ``` From c7ca9a770b380e28642bfcdce538cde385b31d78 Mon Sep 17 00:00:00 2001 From: jielinxu <52057195+jielinxu@users.noreply.github.com> Date: Thu, 28 Nov 2019 18:20:57 +0800 Subject: [PATCH 06/44] Update install.md --- install.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/install.md b/install.md index 48ad3d7577..c226d67c7d 100644 --- a/install.md +++ b/install.md @@ -82,31 +82,31 @@ With this dockerfile, you should be able to compile Milvus on any Linux platform Pull CPU-only image: ```shell -$ Docker pull milvusdb/milvus-cpu-build-env:v0.6.0-ubuntu18.04 +$ docker pull milvusdb/milvus-cpu-build-env:v0.6.0-ubuntu18.04 ``` Pull GPU-enabled image: ```shell -$ Docker pull milvusdb/milvus-gpu-build-env:v0.6.0-ubuntu18.04 +$ docker pull milvusdb/milvus-gpu-build-env:v0.6.0-ubuntu18.04 ``` ### Step 2 Start the Docker container Start a CPU-only container: ```shell -$ Docker run -it -p 19530:19530 -d milvusdb/milvus-cpu-build-env:v0.6.0-ubuntu18.04 +$ docker run -it -p 19530:19530 -d milvusdb/milvus-cpu-build-env:v0.6.0-ubuntu18.04 ``` Start a GPU container: ```shell -$ Docker run — runtime=nvidia -it -p 19530:19530 -d milvusdb/milvus-gpu-build-env:v0.6.0-ubuntu18.04 +$ docker run — runtime=nvidia -it -p 19530:19530 -d milvusdb/milvus-gpu-build-env:v0.6.0-ubuntu18.04 ``` To enter the container: ```shell -$ Docker exec -it [container_id] bash +$ docker exec -it [container_id] bash ``` ### Step 3 Download Milvus source code From 1e4442e33ce4f37234f2e8002bb4404aaf01d5e0 Mon Sep 17 00:00:00 2001 From: jielinxu <52057195+jielinxu@users.noreply.github.com> Date: Thu, 28 Nov 2019 18:28:57 +0800 Subject: [PATCH 07/44] Update install.md --- install.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/install.md b/install.md index c226d67c7d..fd11121f39 100644 --- a/install.md +++ b/install.md @@ -120,8 +120,8 @@ $ wget https://github.com/milvus-io/milvus/archive/0.6.0.zip Unzip the source package: ```shell -$ Apt-get update -$ Apt-get install unzip +$ apt-get update +$ apt-get install unzip $ unzip ./0.6.0.zip ``` From 930097128b8067e925d7cd84d1f48b16aafec869 Mon Sep 17 00:00:00 2001 From: groot Date: Fri, 29 Nov 2019 10:30:10 +0800 Subject: [PATCH 08/44] #596 Frequently insert operation cost too much disk space --- CHANGELOG.md | 1 + core/src/db/DBImpl.cpp | 126 ++++++--------------------- core/src/db/DBImpl.h | 30 ++----- core/src/db/IndexFailedChecker.cpp | 109 +++++++++++++++++++++++ core/src/db/IndexFailedChecker.h | 56 ++++++++++++ core/src/db/OngoingFileChecker.cpp | 98 +++++++++++++++++++++ core/src/db/OngoingFileChecker.h | 61 +++++++++++++ core/src/db/meta/Meta.h | 4 +- core/src/db/meta/MetaTypes.h | 4 + core/src/db/meta/MySQLMetaImpl.cpp | 73 ++++++++-------- core/src/db/meta/MySQLMetaImpl.h | 4 +- core/src/db/meta/SqliteMetaImpl.cpp | 29 +++++- core/src/db/meta/SqliteMetaImpl.h | 4 +- core/unittest/db/test_meta.cpp | 3 +- core/unittest/db/test_meta_mysql.cpp | 3 +- 15 files changed, 434 insertions(+), 171 deletions(-) create mode 100644 core/src/db/IndexFailedChecker.cpp create mode 100644 core/src/db/IndexFailedChecker.h create mode 100644 core/src/db/OngoingFileChecker.cpp create mode 100644 core/src/db/OngoingFileChecker.h diff --git a/CHANGELOG.md b/CHANGELOG.md index fab5bf3bd0..7c44e9cbe8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -35,6 +35,7 @@ Please mark all change in change log and use the ticket from JIRA. - \#545 - Avoid dead circle of build index thread when error occurs - \#552 - Server down during building index_type: IVF_PQ using GPU-edition - \#561 - Milvus server should report exception/error message or terminate on mysql metadata backend error +- \#596 - Frequently insert operation cost too much disk space - \#599 - Build index log is incorrect ## Feature diff --git a/core/src/db/DBImpl.cpp b/core/src/db/DBImpl.cpp index 8e08a850f8..c9ebed6378 100644 --- a/core/src/db/DBImpl.cpp +++ b/core/src/db/DBImpl.cpp @@ -52,8 +52,6 @@ constexpr uint64_t METRIC_ACTION_INTERVAL = 1; constexpr uint64_t COMPACT_ACTION_INTERVAL = 1; constexpr uint64_t INDEX_ACTION_INTERVAL = 1; -constexpr uint64_t INDEX_FAILED_RETRY_TIME = 1; - static const Status SHUTDOWN_ERROR = Status(DB_ERROR, "Milsvus server is shutdown!"); void @@ -370,7 +368,7 @@ DBImpl::CreateIndex(const std::string& table_id, const TableIndex& index) { WaitMergeFileFinish(); // step 4: wait and build index - status = CleanFailedIndexFileOfTable(table_id); + status = index_failed_checker_.CleanFailedIndexFileOfTable(table_id); status = BuildTableIndexRecursively(table_id, index); return status; @@ -504,7 +502,9 @@ DBImpl::QueryAsync(const std::string& table_id, const meta::TableFilesSchema& fi TimeRecorder rc(""); - // step 1: get files to search + // step 1: construct search job + auto status = ongoing_files_checker_.MarkOngoingFiles(files); + ENGINE_LOG_DEBUG << "Engine query begin, index file count: " << files.size(); scheduler::SearchJobPtr job = std::make_shared(k, nq, nprobe, vectors); for (auto& file : files) { @@ -512,9 +512,11 @@ DBImpl::QueryAsync(const std::string& table_id, const meta::TableFilesSchema& fi job->AddIndexFile(file_ptr); } - // step 2: put search task to scheduler + // step 2: put search job to scheduler and wait result scheduler::JobMgrInst::GetInstance()->Put(job); job->WaitResult(); + + status = ongoing_files_checker_.UnmarkOngoingFiles(files); if (!job->GetStatus().ok()) { return job->GetStatus(); } @@ -751,13 +753,15 @@ DBImpl::BackgroundMergeFiles(const std::string& table_id) { } for (auto& kv : raw_files) { - auto files = kv.second; + meta::TableFilesSchema& files = kv.second; if (files.size() < options_.merge_trigger_number_) { ENGINE_LOG_TRACE << "Files number not greater equal than merge trigger number, skip merge action"; continue; } + status = ongoing_files_checker_.MarkOngoingFiles(files); MergeFiles(table_id, kv.first, kv.second); + status = ongoing_files_checker_.UnmarkOngoingFiles(files); if (shutting_down_.load(std::memory_order_acquire)) { ENGINE_LOG_DEBUG << "Server will shutdown, skip merge action for table: " << table_id; @@ -787,17 +791,18 @@ DBImpl::BackgroundCompaction(std::set table_ids) { meta_ptr_->Archive(); + meta::Table2FileIDs ignore_files = ongoing_files_checker_.GetOngoingFiles(); { uint64_t ttl = 10 * meta::SECOND; // default: file data will be erase from cache after few seconds - meta_ptr_->CleanUpCacheWithTTL(ttl); + meta_ptr_->CleanUpCacheWithTTL(ttl, ignore_files); } { - uint64_t ttl = 5 * meta::M_SEC; // default: file will be deleted after few minutes + uint64_t ttl = 20 * meta::SECOND; // default: file will be deleted after few seconds if (options_.mode_ == DBOptions::MODE::CLUSTER_WRITABLE) { - ttl = meta::D_SEC; + ttl = meta::H_SEC; } - meta_ptr_->CleanUpFilesWithTTL(ttl); + meta_ptr_->CleanUpFilesWithTTL(ttl, ignore_files); } // ENGINE_LOG_TRACE << " Background compaction thread exit"; @@ -838,9 +843,11 @@ DBImpl::BackgroundBuildIndex() { std::unique_lock lock(build_index_mutex_); meta::TableFilesSchema to_index_files; meta_ptr_->FilesToIndex(to_index_files); - Status status = IgnoreFailedIndexFiles(to_index_files); + Status status = index_failed_checker_.IgnoreFailedIndexFiles(to_index_files); if (!to_index_files.empty()) { + status = ongoing_files_checker_.MarkOngoingFiles(to_index_files); + // step 2: put build index task to scheduler std::map job2file_map; for (auto& file : to_index_files) { @@ -859,13 +866,15 @@ DBImpl::BackgroundBuildIndex() { Status status = job->GetStatus(); ENGINE_LOG_ERROR << "Building index job " << job->id() << " failed: " << status.ToString(); - MarkFailedIndexFile(file_schema); + index_failed_checker_.MarkFailedIndexFile(file_schema); } else { - MarkSucceedIndexFile(file_schema); + index_failed_checker_.MarkSucceedIndexFile(file_schema); ENGINE_LOG_DEBUG << "Building index job " << job->id() << " succeed."; } } + status = ongoing_files_checker_.UnmarkOngoingFiles(to_index_files); + ENGINE_LOG_DEBUG << "Background build index thread finished"; } @@ -894,6 +903,8 @@ DBImpl::GetFilesToBuildIndex(const std::string& table_id, const std::vector Status DBImpl::GetFilesToSearch(const std::string& table_id, const std::vector& file_ids, const meta::DatesT& dates, meta::TableFilesSchema& files) { + ENGINE_LOG_DEBUG << "Collect files from table: " << table_id; + meta::DatePartionedTableFilesSchema date_files; auto status = meta_ptr_->FilesToSearch(table_id, file_ids, dates, date_files); if (!status.ok()) { @@ -934,7 +945,7 @@ DBImpl::DropTableRecursively(const std::string& table_id, const meta::DatesT& da if (dates.empty()) { status = mem_mgr_->EraseMemVector(table_id); // not allow insert status = meta_ptr_->DropTable(table_id); // soft delete table - CleanFailedIndexFileOfTable(table_id); + index_failed_checker_.CleanFailedIndexFileOfTable(table_id); // scheduler will determine when to delete table files auto nres = scheduler::ResMgrInst::GetInstance()->GetNumOfComputeResource(); @@ -1014,7 +1025,7 @@ DBImpl::BuildTableIndexRecursively(const std::string& table_id, const TableIndex GetFilesToBuildIndex(table_id, file_types, table_files); times++; - IgnoreFailedIndexFiles(table_files); + index_failed_checker_.IgnoreFailedIndexFiles(table_files); } // build index for partition @@ -1029,7 +1040,7 @@ DBImpl::BuildTableIndexRecursively(const std::string& table_id, const TableIndex // failed to build index for some files, return error std::vector failed_files; - GetFailedIndexFileOfTable(table_id, failed_files); + index_failed_checker_.GetFailedIndexFileOfTable(table_id, failed_files); if (!failed_files.empty()) { std::string msg = "Failed to build index for " + std::to_string(failed_files.size()) + ((failed_files.size() == 1) ? " file" : " files"); @@ -1047,7 +1058,7 @@ DBImpl::BuildTableIndexRecursively(const std::string& table_id, const TableIndex Status DBImpl::DropTableIndexRecursively(const std::string& table_id) { ENGINE_LOG_DEBUG << "Drop index for table: " << table_id; - CleanFailedIndexFileOfTable(table_id); + index_failed_checker_.CleanFailedIndexFileOfTable(table_id); auto status = meta_ptr_->DropTableIndex(table_id); if (!status.ok()) { return status; @@ -1090,86 +1101,5 @@ DBImpl::GetTableRowCountRecursively(const std::string& table_id, uint64_t& row_c return Status::OK(); } -Status -DBImpl::CleanFailedIndexFileOfTable(const std::string& table_id) { - std::lock_guard lck(index_failed_mutex_); - index_failed_files_.erase(table_id); // rebuild failed index files for this table - - return Status::OK(); -} - -Status -DBImpl::GetFailedIndexFileOfTable(const std::string& table_id, std::vector& failed_files) { - failed_files.clear(); - std::lock_guard lck(index_failed_mutex_); - auto iter = index_failed_files_.find(table_id); - if (iter != index_failed_files_.end()) { - FileID2FailedTimes& failed_map = iter->second; - for (auto it_file = failed_map.begin(); it_file != failed_map.end(); ++it_file) { - failed_files.push_back(it_file->first); - } - } - - return Status::OK(); -} - -Status -DBImpl::MarkFailedIndexFile(const meta::TableFileSchema& file) { - std::lock_guard lck(index_failed_mutex_); - - auto iter = index_failed_files_.find(file.table_id_); - if (iter == index_failed_files_.end()) { - FileID2FailedTimes failed_files; - failed_files.insert(std::make_pair(file.file_id_, 1)); - index_failed_files_.insert(std::make_pair(file.table_id_, failed_files)); - } else { - auto it_failed_files = iter->second.find(file.file_id_); - if (it_failed_files != iter->second.end()) { - it_failed_files->second++; - } else { - iter->second.insert(std::make_pair(file.file_id_, 1)); - } - } - - return Status::OK(); -} - -Status -DBImpl::MarkSucceedIndexFile(const meta::TableFileSchema& file) { - std::lock_guard lck(index_failed_mutex_); - - auto iter = index_failed_files_.find(file.table_id_); - if (iter != index_failed_files_.end()) { - iter->second.erase(file.file_id_); - } - - return Status::OK(); -} - -Status -DBImpl::IgnoreFailedIndexFiles(meta::TableFilesSchema& table_files) { - std::lock_guard lck(index_failed_mutex_); - - // there could be some failed files belong to different table. - // some files may has failed for several times, no need to build index for these files. - // thus we can avoid dead circle for build index operation - for (auto it_file = table_files.begin(); it_file != table_files.end();) { - auto it_failed_files = index_failed_files_.find((*it_file).table_id_); - if (it_failed_files != index_failed_files_.end()) { - auto it_failed_file = it_failed_files->second.find((*it_file).file_id_); - if (it_failed_file != it_failed_files->second.end()) { - if (it_failed_file->second >= INDEX_FAILED_RETRY_TIME) { - it_file = table_files.erase(it_file); - continue; - } - } - } - - ++it_file; - } - - return Status::OK(); -} - } // namespace engine } // namespace milvus diff --git a/core/src/db/DBImpl.h b/core/src/db/DBImpl.h index 3baac92c0a..5091160d63 100644 --- a/core/src/db/DBImpl.h +++ b/core/src/db/DBImpl.h @@ -18,8 +18,10 @@ #pragma once #include "DB.h" -#include "Types.h" -#include "src/db/insert/MemManager.h" +#include "db/IndexFailedChecker.h" +#include "db/OngoingFileChecker.h" +#include "db/Types.h" +#include "db/insert/MemManager.h" #include "utils/ThreadPool.h" #include @@ -178,21 +180,6 @@ class DBImpl : public DB { Status GetTableRowCountRecursively(const std::string& table_id, uint64_t& row_count); - Status - CleanFailedIndexFileOfTable(const std::string& table_id); - - Status - GetFailedIndexFileOfTable(const std::string& table_id, std::vector& failed_files); - - Status - MarkFailedIndexFile(const meta::TableFileSchema& file); - - Status - MarkSucceedIndexFile(const meta::TableFileSchema& file); - - Status - IgnoreFailedIndexFiles(meta::TableFilesSchema& table_files); - private: const DBOptions options_; @@ -214,11 +201,10 @@ class DBImpl : public DB { std::list> index_thread_results_; std::mutex build_index_mutex_; - std::mutex index_failed_mutex_; - using FileID2FailedTimes = std::map; - using Table2FailedFiles = std::map; - Table2FailedFiles index_failed_files_; // file id mapping to failed times -}; // DBImpl + + IndexFailedChecker index_failed_checker_; + OngoingFileChecker ongoing_files_checker_; +}; // DBImpl } // namespace engine } // namespace milvus diff --git a/core/src/db/IndexFailedChecker.cpp b/core/src/db/IndexFailedChecker.cpp new file mode 100644 index 0000000000..f3667996a5 --- /dev/null +++ b/core/src/db/IndexFailedChecker.cpp @@ -0,0 +1,109 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "db/IndexFailedChecker.h" + +#include + +namespace milvus { +namespace engine { + +constexpr uint64_t INDEX_FAILED_RETRY_TIME = 1; + +Status +IndexFailedChecker::CleanFailedIndexFileOfTable(const std::string& table_id) { + std::lock_guard lck(mutex_); + index_failed_files_.erase(table_id); // rebuild failed index files for this table + + return Status::OK(); +} + +Status +IndexFailedChecker::GetFailedIndexFileOfTable(const std::string& table_id, std::vector& failed_files) { + failed_files.clear(); + std::lock_guard lck(mutex_); + auto iter = index_failed_files_.find(table_id); + if (iter != index_failed_files_.end()) { + FileID2FailedTimes& failed_map = iter->second; + for (auto it_file = failed_map.begin(); it_file != failed_map.end(); ++it_file) { + failed_files.push_back(it_file->first); + } + } + + return Status::OK(); +} + +Status +IndexFailedChecker::MarkFailedIndexFile(const meta::TableFileSchema& file) { + std::lock_guard lck(mutex_); + + auto iter = index_failed_files_.find(file.table_id_); + if (iter == index_failed_files_.end()) { + FileID2FailedTimes failed_files; + failed_files.insert(std::make_pair(file.file_id_, 1)); + index_failed_files_.insert(std::make_pair(file.table_id_, failed_files)); + } else { + auto it_failed_files = iter->second.find(file.file_id_); + if (it_failed_files != iter->second.end()) { + it_failed_files->second++; + } else { + iter->second.insert(std::make_pair(file.file_id_, 1)); + } + } + + return Status::OK(); +} + +Status +IndexFailedChecker::MarkSucceedIndexFile(const meta::TableFileSchema& file) { + std::lock_guard lck(mutex_); + + auto iter = index_failed_files_.find(file.table_id_); + if (iter != index_failed_files_.end()) { + iter->second.erase(file.file_id_); + } + + return Status::OK(); +} + +Status +IndexFailedChecker::IgnoreFailedIndexFiles(meta::TableFilesSchema& table_files) { + std::lock_guard lck(mutex_); + + // there could be some failed files belong to different table. + // some files may has failed for several times, no need to build index for these files. + // thus we can avoid dead circle for build index operation + for (auto it_file = table_files.begin(); it_file != table_files.end();) { + auto it_failed_files = index_failed_files_.find((*it_file).table_id_); + if (it_failed_files != index_failed_files_.end()) { + auto it_failed_file = it_failed_files->second.find((*it_file).file_id_); + if (it_failed_file != it_failed_files->second.end()) { + if (it_failed_file->second >= INDEX_FAILED_RETRY_TIME) { + it_file = table_files.erase(it_file); + continue; + } + } + } + + ++it_file; + } + + return Status::OK(); +} + +} // namespace engine +} // namespace milvus diff --git a/core/src/db/IndexFailedChecker.h b/core/src/db/IndexFailedChecker.h new file mode 100644 index 0000000000..cdcc34a76d --- /dev/null +++ b/core/src/db/IndexFailedChecker.h @@ -0,0 +1,56 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include "meta/Meta.h" +#include "utils/Status.h" + +#include +#include +#include +#include + +namespace milvus { +namespace engine { + +class IndexFailedChecker { + public: + Status + CleanFailedIndexFileOfTable(const std::string& table_id); + + Status + GetFailedIndexFileOfTable(const std::string& table_id, std::vector& failed_files); + + Status + MarkFailedIndexFile(const meta::TableFileSchema& file); + + Status + MarkSucceedIndexFile(const meta::TableFileSchema& file); + + Status + IgnoreFailedIndexFiles(meta::TableFilesSchema& table_files); + + private: + std::mutex mutex_; + using FileID2FailedTimes = std::map; + using Table2FailedFiles = std::map; + Table2FailedFiles index_failed_files_; // file id mapping to failed times +}; + +} // namespace engine +} // namespace milvus diff --git a/core/src/db/OngoingFileChecker.cpp b/core/src/db/OngoingFileChecker.cpp new file mode 100644 index 0000000000..6bf966ba73 --- /dev/null +++ b/core/src/db/OngoingFileChecker.cpp @@ -0,0 +1,98 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "db/OngoingFileChecker.h" + +#include + +namespace milvus { +namespace engine { + +Status +OngoingFileChecker::MarkOngoingFile(const meta::TableFileSchema& table_file) { + std::lock_guard lck(mutex_); + return MarkOngoingFileNoLock(table_file); +} + +Status +OngoingFileChecker::MarkOngoingFiles(const meta::TableFilesSchema& table_files) { + std::lock_guard lck(mutex_); + + for (auto& table_file : table_files) { + MarkOngoingFileNoLock(table_file); + } + + return Status::OK(); +} + +Status +OngoingFileChecker::UnmarkOngoingFile(const meta::TableFileSchema& table_file) { + std::lock_guard lck(mutex_); + return UnmarkOngoingFileNoLock(table_file); +} + +Status +OngoingFileChecker::UnmarkOngoingFiles(const meta::TableFilesSchema& table_files) { + std::lock_guard lck(mutex_); + + for (auto& table_file : table_files) { + UnmarkOngoingFileNoLock(table_file); + } + + return Status::OK(); +} + +meta::Table2FileIDs +OngoingFileChecker::GetOngoingFiles() { + // return copy + // don't return reference(avoid multi-threads conflict) + return ongoing_files_; +} + +Status +OngoingFileChecker::MarkOngoingFileNoLock(const meta::TableFileSchema& table_file) { + if (table_file.table_id_.empty() || table_file.file_id_.empty()) { + return Status(DB_ERROR, "Invalid table files"); + } + + auto iter = ongoing_files_.find(table_file.table_id_); + if (iter == ongoing_files_.end()) { + meta::FileIDArray file_ids = {table_file.file_id_}; + ongoing_files_.insert(std::make_pair(table_file.table_id_, file_ids)); + } else { + iter->second.insert(table_file.file_id_); + } + + return Status::OK(); +} + +Status +OngoingFileChecker::UnmarkOngoingFileNoLock(const meta::TableFileSchema& table_file) { + if (table_file.table_id_.empty() || table_file.file_id_.empty()) { + return Status(DB_ERROR, "Invalid table files"); + } + + auto iter = ongoing_files_.find(table_file.table_id_); + if (iter != ongoing_files_.end()) { + iter->second.erase(table_file.file_id_); + } + + return Status::OK(); +} + +} // namespace engine +} // namespace milvus diff --git a/core/src/db/OngoingFileChecker.h b/core/src/db/OngoingFileChecker.h new file mode 100644 index 0000000000..8f21a45045 --- /dev/null +++ b/core/src/db/OngoingFileChecker.h @@ -0,0 +1,61 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include "meta/Meta.h" +#include "utils/Status.h" + +#include +#include +#include +#include + +namespace milvus { +namespace engine { + +class OngoingFileChecker { + public: + Status + MarkOngoingFile(const meta::TableFileSchema& table_file); + + Status + MarkOngoingFiles(const meta::TableFilesSchema& table_files); + + Status + UnmarkOngoingFile(const meta::TableFileSchema& table_file); + + Status + UnmarkOngoingFiles(const meta::TableFilesSchema& table_files); + + meta::Table2FileIDs + GetOngoingFiles(); + + private: + Status + MarkOngoingFileNoLock(const meta::TableFileSchema& table_file); + + Status + UnmarkOngoingFileNoLock(const meta::TableFileSchema& table_file); + + private: + std::mutex mutex_; + meta::Table2FileIDs ongoing_files_; +}; + +} // namespace engine +} // namespace milvus diff --git a/core/src/db/meta/Meta.h b/core/src/db/meta/Meta.h index bf46f02fea..f620087ad5 100644 --- a/core/src/db/meta/Meta.h +++ b/core/src/db/meta/Meta.h @@ -121,10 +121,10 @@ class Meta { CleanUpShadowFiles() = 0; virtual Status - CleanUpCacheWithTTL(uint64_t seconds) = 0; + CleanUpCacheWithTTL(uint64_t seconds, const Table2FileIDs& ignore_files) = 0; virtual Status - CleanUpFilesWithTTL(uint64_t seconds) = 0; + CleanUpFilesWithTTL(uint64_t seconds, const Table2FileIDs& ignore_files) = 0; virtual Status DropAll() = 0; diff --git a/core/src/db/meta/MetaTypes.h b/core/src/db/meta/MetaTypes.h index d98b74be7d..3e28658983 100644 --- a/core/src/db/meta/MetaTypes.h +++ b/core/src/db/meta/MetaTypes.h @@ -23,6 +23,7 @@ #include #include +#include #include #include @@ -97,6 +98,9 @@ using TableFileSchemaPtr = std::shared_ptr; using TableFilesSchema = std::vector; using DatePartionedTableFilesSchema = std::map; +using FileIDArray = std::set; +using Table2FileIDs = std::map; + } // namespace meta } // namespace engine } // namespace milvus diff --git a/core/src/db/meta/MySQLMetaImpl.cpp b/core/src/db/meta/MySQLMetaImpl.cpp index 7b53e6361a..d9ca892341 100644 --- a/core/src/db/meta/MySQLMetaImpl.cpp +++ b/core/src/db/meta/MySQLMetaImpl.cpp @@ -1639,7 +1639,8 @@ MySQLMetaImpl::FilesByType(const std::string& table_id, const std::vector& case (int)TableFileSchema::BACKUP: msg = msg + " backup files:" + std::to_string(backup_count); break; - default:break; + default: + break; } } ENGINE_LOG_DEBUG << msg; @@ -1782,7 +1783,7 @@ MySQLMetaImpl::CleanUpShadowFiles() { } Status -MySQLMetaImpl::CleanUpCacheWithTTL(uint64_t seconds) { +MySQLMetaImpl::CleanUpCacheWithTTL(uint64_t seconds, const Table2FileIDs& ignore_files) { auto now = utils::GetMicroSecTimeStamp(); // erase deleted/backup files from cache @@ -1795,14 +1796,13 @@ MySQLMetaImpl::CleanUpCacheWithTTL(uint64_t seconds) { return Status(DB_ERROR, "Failed to connect to meta server(mysql)"); } - mysqlpp::Query cleanUpFilesWithTTLQuery = connectionPtr->query(); - cleanUpFilesWithTTLQuery << "SELECT id, table_id, file_id, date" - << " FROM " << META_TABLEFILES << " WHERE file_type IN (" - << std::to_string(TableFileSchema::TO_DELETE) << "," - << std::to_string(TableFileSchema::BACKUP) << ")" - << " AND updated_time < " << std::to_string(now - seconds * US_PS) << ";"; + mysqlpp::Query query = connectionPtr->query(); + query << "SELECT id, table_id, file_id, date" + << " FROM " << META_TABLEFILES << " WHERE file_type IN (" << std::to_string(TableFileSchema::TO_DELETE) + << "," << std::to_string(TableFileSchema::BACKUP) << ")" + << " AND updated_time < " << std::to_string(now - seconds * US_PS) << ";"; - mysqlpp::StoreQueryResult res = cleanUpFilesWithTTLQuery.store(); + mysqlpp::StoreQueryResult res = query.store(); TableFileSchema table_file; std::vector idsToDelete; @@ -1824,7 +1824,7 @@ MySQLMetaImpl::CleanUpCacheWithTTL(uint64_t seconds) { } Status -MySQLMetaImpl::CleanUpFilesWithTTL(uint64_t seconds) { +MySQLMetaImpl::CleanUpFilesWithTTL(uint64_t seconds, const Table2FileIDs& ignore_files) { auto now = utils::GetMicroSecTimeStamp(); std::set table_ids; @@ -1839,15 +1839,14 @@ MySQLMetaImpl::CleanUpFilesWithTTL(uint64_t seconds) { return Status(DB_ERROR, "Failed to connect to meta server(mysql)"); } - mysqlpp::Query cleanUpFilesWithTTLQuery = connectionPtr->query(); - cleanUpFilesWithTTLQuery << "SELECT id, table_id, file_id, date" - << " FROM " << META_TABLEFILES - << " WHERE file_type = " << std::to_string(TableFileSchema::TO_DELETE) - << " AND updated_time < " << std::to_string(now - seconds * US_PS) << ";"; + mysqlpp::Query query = connectionPtr->query(); + query << "SELECT id, table_id, file_id, date" + << " FROM " << META_TABLEFILES << " WHERE file_type = " << std::to_string(TableFileSchema::TO_DELETE) + << " AND updated_time < " << std::to_string(now - seconds * US_PS) << ";"; - ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUpFilesWithTTL: " << cleanUpFilesWithTTLQuery.str(); + ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUpFilesWithTTL: " << query.str(); - mysqlpp::StoreQueryResult res = cleanUpFilesWithTTLQuery.store(); + mysqlpp::StoreQueryResult res = query.store(); TableFileSchema table_file; std::vector idsToDelete; @@ -1874,13 +1873,12 @@ MySQLMetaImpl::CleanUpFilesWithTTL(uint64_t seconds) { std::string idsToDeleteStr = idsToDeleteSS.str(); idsToDeleteStr = idsToDeleteStr.substr(0, idsToDeleteStr.size() - 4); // remove the last " OR " - cleanUpFilesWithTTLQuery << "DELETE FROM " << META_TABLEFILES << " WHERE " << idsToDeleteStr << ";"; + query << "DELETE FROM " << META_TABLEFILES << " WHERE " << idsToDeleteStr << ";"; - ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUpFilesWithTTL: " << cleanUpFilesWithTTLQuery.str(); + ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUpFilesWithTTL: " << query.str(); - if (!cleanUpFilesWithTTLQuery.exec()) { - return HandleException("QUERY ERROR WHEN CLEANING UP FILES WITH TTL", - cleanUpFilesWithTTLQuery.error()); + if (!query.exec()) { + return HandleException("QUERY ERROR WHEN CLEANING UP FILES WITH TTL", query.error()); } } @@ -1903,14 +1901,13 @@ MySQLMetaImpl::CleanUpFilesWithTTL(uint64_t seconds) { return Status(DB_ERROR, "Failed to connect to meta server(mysql)"); } - mysqlpp::Query cleanUpFilesWithTTLQuery = connectionPtr->query(); - cleanUpFilesWithTTLQuery << "SELECT id, table_id" - << " FROM " << META_TABLES - << " WHERE state = " << std::to_string(TableSchema::TO_DELETE) << ";"; + mysqlpp::Query query = connectionPtr->query(); + query << "SELECT id, table_id" + << " FROM " << META_TABLES << " WHERE state = " << std::to_string(TableSchema::TO_DELETE) << ";"; - ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUpFilesWithTTL: " << cleanUpFilesWithTTLQuery.str(); + ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUpFilesWithTTL: " << query.str(); - mysqlpp::StoreQueryResult res = cleanUpFilesWithTTLQuery.store(); + mysqlpp::StoreQueryResult res = query.store(); int64_t remove_tables = 0; if (!res.empty()) { @@ -1926,13 +1923,12 @@ MySQLMetaImpl::CleanUpFilesWithTTL(uint64_t seconds) { } std::string idsToDeleteStr = idsToDeleteSS.str(); idsToDeleteStr = idsToDeleteStr.substr(0, idsToDeleteStr.size() - 4); // remove the last " OR " - cleanUpFilesWithTTLQuery << "DELETE FROM " << META_TABLES << " WHERE " << idsToDeleteStr << ";"; + query << "DELETE FROM " << META_TABLES << " WHERE " << idsToDeleteStr << ";"; - ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUpFilesWithTTL: " << cleanUpFilesWithTTLQuery.str(); + ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUpFilesWithTTL: " << query.str(); - if (!cleanUpFilesWithTTLQuery.exec()) { - return HandleException("QUERY ERROR WHEN CLEANING UP TABLES WITH TTL", - cleanUpFilesWithTTLQuery.error()); + if (!query.exec()) { + return HandleException("QUERY ERROR WHEN CLEANING UP TABLES WITH TTL", query.error()); } } @@ -1957,14 +1953,13 @@ MySQLMetaImpl::CleanUpFilesWithTTL(uint64_t seconds) { } for (auto& table_id : table_ids) { - mysqlpp::Query cleanUpFilesWithTTLQuery = connectionPtr->query(); - cleanUpFilesWithTTLQuery << "SELECT file_id" - << " FROM " << META_TABLEFILES << " WHERE table_id = " << mysqlpp::quote - << table_id << ";"; + mysqlpp::Query query = connectionPtr->query(); + query << "SELECT file_id" + << " FROM " << META_TABLEFILES << " WHERE table_id = " << mysqlpp::quote << table_id << ";"; - ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUpFilesWithTTL: " << cleanUpFilesWithTTLQuery.str(); + ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUpFilesWithTTL: " << query.str(); - mysqlpp::StoreQueryResult res = cleanUpFilesWithTTLQuery.store(); + mysqlpp::StoreQueryResult res = query.store(); if (res.empty()) { utils::DeleteTablePath(options_, table_id); diff --git a/core/src/db/meta/MySQLMetaImpl.h b/core/src/db/meta/MySQLMetaImpl.h index e7697316af..d13ed7e043 100644 --- a/core/src/db/meta/MySQLMetaImpl.h +++ b/core/src/db/meta/MySQLMetaImpl.h @@ -120,10 +120,10 @@ class MySQLMetaImpl : public Meta { CleanUpShadowFiles() override; Status - CleanUpCacheWithTTL(uint64_t seconds) override; + CleanUpCacheWithTTL(uint64_t seconds, const Table2FileIDs& ignore_files) override; Status - CleanUpFilesWithTTL(uint64_t seconds) override; + CleanUpFilesWithTTL(uint64_t seconds, const Table2FileIDs& ignore_files) override; Status DropAll() override; diff --git a/core/src/db/meta/SqliteMetaImpl.cpp b/core/src/db/meta/SqliteMetaImpl.cpp index 07f890d50a..e751b14555 100644 --- a/core/src/db/meta/SqliteMetaImpl.cpp +++ b/core/src/db/meta/SqliteMetaImpl.cpp @@ -1294,7 +1294,7 @@ SqliteMetaImpl::CleanUpShadowFiles() { } Status -SqliteMetaImpl::CleanUpCacheWithTTL(uint64_t seconds) { +SqliteMetaImpl::CleanUpCacheWithTTL(uint64_t seconds, const Table2FileIDs& ignore_files) { auto now = utils::GetMicroSecTimeStamp(); // erase deleted/backup files from cache @@ -1309,6 +1309,7 @@ SqliteMetaImpl::CleanUpCacheWithTTL(uint64_t seconds) { (int)TableFileSchema::BACKUP, }; + // collect files to be erased auto files = ConnectorPtr->select(columns(&TableFileSchema::id_, &TableFileSchema::table_id_, &TableFileSchema::file_id_, @@ -1326,6 +1327,15 @@ SqliteMetaImpl::CleanUpCacheWithTTL(uint64_t seconds) { table_file.file_id_ = std::get<2>(file); table_file.date_ = std::get<3>(file); + // check if the file can be deleted + auto iter = ignore_files.find(table_file.table_id_); + if (iter != ignore_files.end()) { + if (iter->second.find(table_file.file_id_) != iter->second.end()) { + continue; // ignore this file, don't delete it + } + } + + // erase file data from cache utils::GetTableFilePath(options_, table_file); server::CommonUtil::EraseFromCache(table_file.location_); } @@ -1338,7 +1348,7 @@ SqliteMetaImpl::CleanUpCacheWithTTL(uint64_t seconds) { } Status -SqliteMetaImpl::CleanUpFilesWithTTL(uint64_t seconds) { +SqliteMetaImpl::CleanUpFilesWithTTL(uint64_t seconds, const Table2FileIDs& ignore_files) { auto now = utils::GetMicroSecTimeStamp(); std::set table_ids; @@ -1349,6 +1359,7 @@ SqliteMetaImpl::CleanUpFilesWithTTL(uint64_t seconds) { // multi-threads call sqlite update may get exception('bad logic', etc), so we add a lock here std::lock_guard meta_lock(meta_mutex_); + // collect files to be deleted auto files = ConnectorPtr->select(columns(&TableFileSchema::id_, &TableFileSchema::table_id_, &TableFileSchema::file_id_, @@ -1368,10 +1379,20 @@ SqliteMetaImpl::CleanUpFilesWithTTL(uint64_t seconds) { table_file.file_id_ = std::get<2>(file); table_file.date_ = std::get<3>(file); - utils::DeleteTableFilePath(options_, table_file); - ENGINE_LOG_DEBUG << "Removing file id:" << table_file.file_id_ << " location:" << table_file.location_; + // check if the file can be deleted + auto iter = ignore_files.find(table_file.table_id_); + if (iter != ignore_files.end()) { + if (iter->second.find(table_file.file_id_) != iter->second.end()) { + continue; // ignore this file, don't delete it + } + } + + // delete file from meta ConnectorPtr->remove(table_file.id_); + // delete file from disk storage + utils::DeleteTableFilePath(options_, table_file); + ENGINE_LOG_DEBUG << "Removing file id:" << table_file.file_id_ << " location:" << table_file.location_; table_ids.insert(table_file.table_id_); } return true; diff --git a/core/src/db/meta/SqliteMetaImpl.h b/core/src/db/meta/SqliteMetaImpl.h index 5581efe361..4ef4542022 100644 --- a/core/src/db/meta/SqliteMetaImpl.h +++ b/core/src/db/meta/SqliteMetaImpl.h @@ -120,10 +120,10 @@ class SqliteMetaImpl : public Meta { CleanUpShadowFiles() override; Status - CleanUpCacheWithTTL(uint64_t seconds) override; + CleanUpCacheWithTTL(uint64_t seconds, const Table2FileIDs& ignore_files) override; Status - CleanUpFilesWithTTL(uint64_t seconds) override; + CleanUpFilesWithTTL(uint64_t seconds, const Table2FileIDs& ignore_files) override; Status DropAll() override; diff --git a/core/unittest/db/test_meta.cpp b/core/unittest/db/test_meta.cpp index b89c73c296..c2a9f8e8bc 100644 --- a/core/unittest/db/test_meta.cpp +++ b/core/unittest/db/test_meta.cpp @@ -335,7 +335,8 @@ TEST_F(MetaTest, TABLE_FILES_TEST) { status = impl_->DropTable(table_id); ASSERT_TRUE(status.ok()); - status = impl_->CleanUpFilesWithTTL(1UL); + milvus::engine::meta::Table2FileIDs ignore_files; + status = impl_->CleanUpFilesWithTTL(1UL, ignore_files); ASSERT_TRUE(status.ok()); } diff --git a/core/unittest/db/test_meta_mysql.cpp b/core/unittest/db/test_meta_mysql.cpp index 9a52a01b7b..cdd8aa32a8 100644 --- a/core/unittest/db/test_meta_mysql.cpp +++ b/core/unittest/db/test_meta_mysql.cpp @@ -349,7 +349,8 @@ TEST_F(MySqlMetaTest, TABLE_FILES_TEST) { status = impl_->DropTable(table_id); ASSERT_TRUE(status.ok()); - status = impl_->CleanUpFilesWithTTL(0UL); + milvus::engine::meta::Table2FileIDs ignore_files; + status = impl_->CleanUpFilesWithTTL(0UL, ignore_files); ASSERT_TRUE(status.ok()); } From 50cd6dd8d291eb9fd1bec2c35dbd2e9b9713b2a3 Mon Sep 17 00:00:00 2001 From: groot Date: Fri, 29 Nov 2019 10:47:23 +0800 Subject: [PATCH 09/44] add unittest case --- core/src/db/IndexFailedChecker.cpp | 3 ++ core/src/db/OngoingFileChecker.cpp | 3 ++ core/unittest/db/test_misc.cpp | 64 ++++++++++++++++++++++++++++++ 3 files changed, 70 insertions(+) diff --git a/core/src/db/IndexFailedChecker.cpp b/core/src/db/IndexFailedChecker.cpp index f3667996a5..d1bb753294 100644 --- a/core/src/db/IndexFailedChecker.cpp +++ b/core/src/db/IndexFailedChecker.cpp @@ -75,6 +75,9 @@ IndexFailedChecker::MarkSucceedIndexFile(const meta::TableFileSchema& file) { auto iter = index_failed_files_.find(file.table_id_); if (iter != index_failed_files_.end()) { iter->second.erase(file.file_id_); + if (iter->second.empty()) { + index_failed_files_.erase(file.table_id_); + } } return Status::OK(); diff --git a/core/src/db/OngoingFileChecker.cpp b/core/src/db/OngoingFileChecker.cpp index 6bf966ba73..3462621d3c 100644 --- a/core/src/db/OngoingFileChecker.cpp +++ b/core/src/db/OngoingFileChecker.cpp @@ -89,6 +89,9 @@ OngoingFileChecker::UnmarkOngoingFileNoLock(const meta::TableFileSchema& table_f auto iter = ongoing_files_.find(table_file.table_id_); if (iter != ongoing_files_.end()) { iter->second.erase(table_file.file_id_); + if (iter->second.empty()) { + ongoing_files_.erase(table_file.table_id_); + } } return Status::OK(); diff --git a/core/unittest/db/test_misc.cpp b/core/unittest/db/test_misc.cpp index c044dde8da..1ed02afb53 100644 --- a/core/unittest/db/test_misc.cpp +++ b/core/unittest/db/test_misc.cpp @@ -15,6 +15,8 @@ // specific language governing permissions and limitations // under the License. +#include "db/IndexFailedChecker.h" +#include "db/OngoingFileChecker.h" #include "db/Options.h" #include "db/Utils.h" #include "db/engine/EngineFactory.h" @@ -119,3 +121,65 @@ TEST(DBMiscTest, UTILS_TEST) { status = milvus::engine::utils::DeleteTableFilePath(options, file); ASSERT_TRUE(status.ok()); } + +TEST(DBMiscTest, CHECKER_TEST) { + { + milvus::engine::IndexFailedChecker checker; + milvus::engine::meta::TableFileSchema schema; + schema.table_id_ = "aaa"; + schema.file_id_ = "5000"; + checker.MarkFailedIndexFile(schema); + schema.table_id_ = "bbb"; + schema.file_id_ = "5001"; + checker.MarkFailedIndexFile(schema); + + std::vector failed_files; + checker.GetFailedIndexFileOfTable("aaa", failed_files); + ASSERT_EQ(failed_files.size(), 1UL); + + schema.table_id_ = "bbb"; + schema.file_id_ = "5002"; + checker.MarkFailedIndexFile(schema); + checker.MarkFailedIndexFile(schema); + + milvus::engine::meta::TableFilesSchema table_files = {schema}; + checker.IgnoreFailedIndexFiles(table_files); + ASSERT_TRUE(table_files.empty()); + + checker.GetFailedIndexFileOfTable("bbb", failed_files); + ASSERT_EQ(failed_files.size(), 2UL); + + checker.MarkSucceedIndexFile(schema); + checker.GetFailedIndexFileOfTable("bbb", failed_files); + ASSERT_EQ(failed_files.size(), 1UL); + } + + { + milvus::engine::OngoingFileChecker checker; + milvus::engine::meta::TableFileSchema schema; + schema.table_id_ = "aaa"; + schema.file_id_ = "5000"; + checker.MarkOngoingFile(schema); + + auto ongoing_files = checker.GetOngoingFiles(); + ASSERT_EQ(ongoing_files.size(), 1UL); + + schema.table_id_ = "bbb"; + schema.file_id_ = "5001"; + milvus::engine::meta::TableFilesSchema table_files = {schema}; + checker.MarkOngoingFiles(table_files); + + ongoing_files = checker.GetOngoingFiles(); + ASSERT_EQ(ongoing_files.size(), 2UL); + + checker.UnmarkOngoingFile(schema); + ongoing_files = checker.GetOngoingFiles(); + ASSERT_EQ(ongoing_files.size(), 1UL); + + schema.table_id_ = "aaa"; + schema.file_id_ = "5000"; + checker.UnmarkOngoingFile(schema); + ongoing_files = checker.GetOngoingFiles(); + ASSERT_EQ(ongoing_files.size(), 0UL); + } +} From cc5c0bf2f21f6d8b362b6105bd96130c44ce1d15 Mon Sep 17 00:00:00 2001 From: groot Date: Fri, 29 Nov 2019 10:55:59 +0800 Subject: [PATCH 10/44] mysql ongoing files --- core/src/db/meta/MySQLMetaImpl.cpp | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/core/src/db/meta/MySQLMetaImpl.cpp b/core/src/db/meta/MySQLMetaImpl.cpp index d9ca892341..559e0d5a11 100644 --- a/core/src/db/meta/MySQLMetaImpl.cpp +++ b/core/src/db/meta/MySQLMetaImpl.cpp @@ -1813,6 +1813,15 @@ MySQLMetaImpl::CleanUpCacheWithTTL(uint64_t seconds, const Table2FileIDs& ignore resRow["file_id"].to_string(table_file.file_id_); table_file.date_ = resRow["date"]; + // check if the file can be deleted + auto iter = ignore_files.find(table_file.table_id_); + if (iter != ignore_files.end()) { + if (iter->second.find(table_file.file_id_) != iter->second.end()) { + continue; // ignore this file, don't delete it + } + } + + // erase file data from cache utils::GetTableFilePath(options_, table_file); server::CommonUtil::EraseFromCache(table_file.location_); } @@ -1857,14 +1866,23 @@ MySQLMetaImpl::CleanUpFilesWithTTL(uint64_t seconds, const Table2FileIDs& ignore resRow["file_id"].to_string(table_file.file_id_); table_file.date_ = resRow["date"]; - utils::DeleteTableFilePath(options_, table_file); + // check if the file can be deleted + auto iter = ignore_files.find(table_file.table_id_); + if (iter != ignore_files.end()) { + if (iter->second.find(table_file.file_id_) != iter->second.end()) { + continue; // ignore this file, don't delete it + } + } + // delete file from disk storage + utils::DeleteTableFilePath(options_, table_file); ENGINE_LOG_DEBUG << "Removing file id:" << table_file.id_ << " location:" << table_file.location_; idsToDelete.emplace_back(std::to_string(table_file.id_)); table_ids.insert(table_file.table_id_); } + // delete file from meta if (!idsToDelete.empty()) { std::stringstream idsToDeleteSS; for (auto& id : idsToDelete) { From 095b1243e6cf0870583d1c434167bed3004d23d3 Mon Sep 17 00:00:00 2001 From: groot Date: Fri, 29 Nov 2019 11:37:31 +0800 Subject: [PATCH 11/44] add more log --- core/src/db/DBImpl.cpp | 5 +++-- core/src/db/meta/MySQLMetaImpl.cpp | 8 ++++++-- core/src/db/meta/SqliteMetaImpl.cpp | 4 ++++ 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/core/src/db/DBImpl.cpp b/core/src/db/DBImpl.cpp index c9ebed6378..e96b921fb7 100644 --- a/core/src/db/DBImpl.cpp +++ b/core/src/db/DBImpl.cpp @@ -849,15 +849,16 @@ DBImpl::BackgroundBuildIndex() { status = ongoing_files_checker_.MarkOngoingFiles(to_index_files); // step 2: put build index task to scheduler - std::map job2file_map; + std::vector> job2file_map; for (auto& file : to_index_files) { scheduler::BuildIndexJobPtr job = std::make_shared(meta_ptr_, options_); scheduler::TableFileSchemaPtr file_ptr = std::make_shared(file); job->AddToIndexFiles(file_ptr); scheduler::JobMgrInst::GetInstance()->Put(job); - job2file_map.insert(std::make_pair(job, file_ptr)); + job2file_map.push_back(std::make_pair(job, file_ptr)); } + // step 3: wait build index finished and mark failed files for (auto iter = job2file_map.begin(); iter != job2file_map.end(); ++iter) { scheduler::BuildIndexJobPtr job = iter->first; meta::TableFileSchema& file_schema = *(iter->second.get()); diff --git a/core/src/db/meta/MySQLMetaImpl.cpp b/core/src/db/meta/MySQLMetaImpl.cpp index 559e0d5a11..4e2f3b47e1 100644 --- a/core/src/db/meta/MySQLMetaImpl.cpp +++ b/core/src/db/meta/MySQLMetaImpl.cpp @@ -1817,7 +1817,9 @@ MySQLMetaImpl::CleanUpCacheWithTTL(uint64_t seconds, const Table2FileIDs& ignore auto iter = ignore_files.find(table_file.table_id_); if (iter != ignore_files.end()) { if (iter->second.find(table_file.file_id_) != iter->second.end()) { - continue; // ignore this file, don't delete it + ENGINE_LOG_DEBUG << "File:" << table_file.file_id_ + << " currently is in use, not able to erase from cache now"; + continue; // ignore this file, don't delete it } } @@ -1870,7 +1872,9 @@ MySQLMetaImpl::CleanUpFilesWithTTL(uint64_t seconds, const Table2FileIDs& ignore auto iter = ignore_files.find(table_file.table_id_); if (iter != ignore_files.end()) { if (iter->second.find(table_file.file_id_) != iter->second.end()) { - continue; // ignore this file, don't delete it + ENGINE_LOG_DEBUG << "File:" << table_file.file_id_ + << " currently is in use, not able to delete now"; + continue; // ignore this file, don't delete it } } diff --git a/core/src/db/meta/SqliteMetaImpl.cpp b/core/src/db/meta/SqliteMetaImpl.cpp index e751b14555..b2f4102dde 100644 --- a/core/src/db/meta/SqliteMetaImpl.cpp +++ b/core/src/db/meta/SqliteMetaImpl.cpp @@ -1331,6 +1331,8 @@ SqliteMetaImpl::CleanUpCacheWithTTL(uint64_t seconds, const Table2FileIDs& ignor auto iter = ignore_files.find(table_file.table_id_); if (iter != ignore_files.end()) { if (iter->second.find(table_file.file_id_) != iter->second.end()) { + ENGINE_LOG_DEBUG << "File:" << table_file.file_id_ + << " currently is in use, not able to erase from cache now"; continue; // ignore this file, don't delete it } } @@ -1383,6 +1385,8 @@ SqliteMetaImpl::CleanUpFilesWithTTL(uint64_t seconds, const Table2FileIDs& ignor auto iter = ignore_files.find(table_file.table_id_); if (iter != ignore_files.end()) { if (iter->second.find(table_file.file_id_) != iter->second.end()) { + ENGINE_LOG_DEBUG << "File:" << table_file.file_id_ + << " currently is in use, not able to delete now"; continue; // ignore this file, don't delete it } } From 85211d537f7b8a579c4ce3b3450536f35b351f10 Mon Sep 17 00:00:00 2001 From: groot Date: Fri, 29 Nov 2019 12:38:18 +0800 Subject: [PATCH 12/44] reconstruct code --- core/src/db/DBImpl.cpp | 9 +++---- core/src/db/IndexFailedChecker.cpp | 4 +-- core/src/db/IndexFailedChecker.h | 4 +-- core/src/db/OngoingFileChecker.cpp | 39 ++++++++++++++++++++++------ core/src/db/OngoingFileChecker.h | 8 +++--- core/src/db/meta/Meta.h | 11 ++++++-- core/src/db/meta/MetaTypes.h | 4 +-- core/src/db/meta/MySQLMetaImpl.cpp | 28 ++++++++------------ core/src/db/meta/MySQLMetaImpl.h | 4 +-- core/src/db/meta/SqliteMetaImpl.cpp | 28 ++++++++------------ core/src/db/meta/SqliteMetaImpl.h | 4 +-- core/unittest/db/test_meta.cpp | 3 +-- core/unittest/db/test_meta_mysql.cpp | 3 +-- core/unittest/db/test_misc.cpp | 13 ++++------ 14 files changed, 86 insertions(+), 76 deletions(-) diff --git a/core/src/db/DBImpl.cpp b/core/src/db/DBImpl.cpp index e96b921fb7..c88936a08e 100644 --- a/core/src/db/DBImpl.cpp +++ b/core/src/db/DBImpl.cpp @@ -791,18 +791,17 @@ DBImpl::BackgroundCompaction(std::set table_ids) { meta_ptr_->Archive(); - meta::Table2FileIDs ignore_files = ongoing_files_checker_.GetOngoingFiles(); { - uint64_t ttl = 10 * meta::SECOND; // default: file data will be erase from cache after few seconds - meta_ptr_->CleanUpCacheWithTTL(ttl, ignore_files); + uint64_t ttl = 1 * meta::SECOND; // default: file data will be erase from cache after few seconds + meta_ptr_->CleanUpCacheWithTTL(ttl, &ongoing_files_checker_); } { - uint64_t ttl = 20 * meta::SECOND; // default: file will be deleted after few seconds + uint64_t ttl = 1 * meta::SECOND; // default: file will be deleted after few seconds if (options_.mode_ == DBOptions::MODE::CLUSTER_WRITABLE) { ttl = meta::H_SEC; } - meta_ptr_->CleanUpFilesWithTTL(ttl, ignore_files); + meta_ptr_->CleanUpFilesWithTTL(ttl, &ongoing_files_checker_); } // ENGINE_LOG_TRACE << " Background compaction thread exit"; diff --git a/core/src/db/IndexFailedChecker.cpp b/core/src/db/IndexFailedChecker.cpp index d1bb753294..bdd152dc4b 100644 --- a/core/src/db/IndexFailedChecker.cpp +++ b/core/src/db/IndexFailedChecker.cpp @@ -38,7 +38,7 @@ IndexFailedChecker::GetFailedIndexFileOfTable(const std::string& table_id, std:: std::lock_guard lck(mutex_); auto iter = index_failed_files_.find(table_id); if (iter != index_failed_files_.end()) { - FileID2FailedTimes& failed_map = iter->second; + meta::File2RefCount& failed_map = iter->second; for (auto it_file = failed_map.begin(); it_file != failed_map.end(); ++it_file) { failed_files.push_back(it_file->first); } @@ -53,7 +53,7 @@ IndexFailedChecker::MarkFailedIndexFile(const meta::TableFileSchema& file) { auto iter = index_failed_files_.find(file.table_id_); if (iter == index_failed_files_.end()) { - FileID2FailedTimes failed_files; + meta::File2RefCount failed_files; failed_files.insert(std::make_pair(file.file_id_, 1)); index_failed_files_.insert(std::make_pair(file.table_id_, failed_files)); } else { diff --git a/core/src/db/IndexFailedChecker.h b/core/src/db/IndexFailedChecker.h index cdcc34a76d..edd6a114bf 100644 --- a/core/src/db/IndexFailedChecker.h +++ b/core/src/db/IndexFailedChecker.h @@ -47,9 +47,7 @@ class IndexFailedChecker { private: std::mutex mutex_; - using FileID2FailedTimes = std::map; - using Table2FailedFiles = std::map; - Table2FailedFiles index_failed_files_; // file id mapping to failed times + meta::Table2Files index_failed_files_; // table id mapping to (file id mapping to failed times) }; } // namespace engine diff --git a/core/src/db/OngoingFileChecker.cpp b/core/src/db/OngoingFileChecker.cpp index 3462621d3c..daaf6a2a4f 100644 --- a/core/src/db/OngoingFileChecker.cpp +++ b/core/src/db/OngoingFileChecker.cpp @@ -16,6 +16,7 @@ // under the License. #include "db/OngoingFileChecker.h" +#include "utils/Log.h" #include @@ -56,11 +57,21 @@ OngoingFileChecker::UnmarkOngoingFiles(const meta::TableFilesSchema& table_files return Status::OK(); } -meta::Table2FileIDs -OngoingFileChecker::GetOngoingFiles() { - // return copy - // don't return reference(avoid multi-threads conflict) - return ongoing_files_; +bool +OngoingFileChecker::IsIgnored(const meta::TableFileSchema& schema) { + std::lock_guard lck(mutex_); + + auto iter = ongoing_files_.find(schema.table_id_); + if (iter == ongoing_files_.end()) { + return false; + } else { + auto it_file = iter->second.find(schema.file_id_); + if (it_file == iter->second.end()) { + return false; + } else { + return (it_file->second > 0); + } + } } Status @@ -71,12 +82,21 @@ OngoingFileChecker::MarkOngoingFileNoLock(const meta::TableFileSchema& table_fil auto iter = ongoing_files_.find(table_file.table_id_); if (iter == ongoing_files_.end()) { - meta::FileIDArray file_ids = {table_file.file_id_}; - ongoing_files_.insert(std::make_pair(table_file.table_id_, file_ids)); + meta::File2RefCount files_refcount; + files_refcount.insert(std::make_pair(table_file.file_id_, 1)); + ongoing_files_.insert(std::make_pair(table_file.table_id_, files_refcount)); } else { - iter->second.insert(table_file.file_id_); + auto it_file = iter->second.find(table_file.file_id_); + if (it_file == iter->second.end()) { + iter->second[table_file.file_id_] = 1; + } else { + it_file->second++; + } } + ENGINE_LOG_DEBUG << "Mark ongoing file:" << table_file.file_id_ + << " refcount:" << ongoing_files_[table_file.table_id_][table_file.file_id_]; + return Status::OK(); } @@ -94,6 +114,9 @@ OngoingFileChecker::UnmarkOngoingFileNoLock(const meta::TableFileSchema& table_f } } + ENGINE_LOG_DEBUG << "Mark ongoing file:" << table_file.file_id_ + << " refcount:" << ongoing_files_[table_file.table_id_][table_file.file_id_]; + return Status::OK(); } diff --git a/core/src/db/OngoingFileChecker.h b/core/src/db/OngoingFileChecker.h index 8f21a45045..c7995e425a 100644 --- a/core/src/db/OngoingFileChecker.h +++ b/core/src/db/OngoingFileChecker.h @@ -28,7 +28,7 @@ namespace milvus { namespace engine { -class OngoingFileChecker { +class OngoingFileChecker : public meta::Meta::CleanUpFilter { public: Status MarkOngoingFile(const meta::TableFileSchema& table_file); @@ -42,8 +42,8 @@ class OngoingFileChecker { Status UnmarkOngoingFiles(const meta::TableFilesSchema& table_files); - meta::Table2FileIDs - GetOngoingFiles(); + bool + IsIgnored(const meta::TableFileSchema& schema) override; private: Status @@ -54,7 +54,7 @@ class OngoingFileChecker { private: std::mutex mutex_; - meta::Table2FileIDs ongoing_files_; + meta::Table2Files ongoing_files_; // table id mapping to (file id mapping to ongoing ref-count) }; } // namespace engine diff --git a/core/src/db/meta/Meta.h b/core/src/db/meta/Meta.h index f620087ad5..8dabb58203 100644 --- a/core/src/db/meta/Meta.h +++ b/core/src/db/meta/Meta.h @@ -35,6 +35,13 @@ static const char* META_TABLES = "Tables"; static const char* META_TABLEFILES = "TableFiles"; class Meta { + public: + class CleanUpFilter { + public: + virtual bool + IsIgnored(const TableFileSchema& schema) = 0; + }; + public: virtual ~Meta() = default; @@ -121,10 +128,10 @@ class Meta { CleanUpShadowFiles() = 0; virtual Status - CleanUpCacheWithTTL(uint64_t seconds, const Table2FileIDs& ignore_files) = 0; + CleanUpCacheWithTTL(uint64_t seconds, CleanUpFilter* filter = nullptr) = 0; virtual Status - CleanUpFilesWithTTL(uint64_t seconds, const Table2FileIDs& ignore_files) = 0; + CleanUpFilesWithTTL(uint64_t seconds, CleanUpFilter* filter = nullptr) = 0; virtual Status DropAll() = 0; diff --git a/core/src/db/meta/MetaTypes.h b/core/src/db/meta/MetaTypes.h index 3e28658983..f8c082032e 100644 --- a/core/src/db/meta/MetaTypes.h +++ b/core/src/db/meta/MetaTypes.h @@ -98,8 +98,8 @@ using TableFileSchemaPtr = std::shared_ptr; using TableFilesSchema = std::vector; using DatePartionedTableFilesSchema = std::map; -using FileIDArray = std::set; -using Table2FileIDs = std::map; +using File2RefCount = std::map; +using Table2Files = std::map; } // namespace meta } // namespace engine diff --git a/core/src/db/meta/MySQLMetaImpl.cpp b/core/src/db/meta/MySQLMetaImpl.cpp index 4e2f3b47e1..1fab7a0f77 100644 --- a/core/src/db/meta/MySQLMetaImpl.cpp +++ b/core/src/db/meta/MySQLMetaImpl.cpp @@ -1783,7 +1783,7 @@ MySQLMetaImpl::CleanUpShadowFiles() { } Status -MySQLMetaImpl::CleanUpCacheWithTTL(uint64_t seconds, const Table2FileIDs& ignore_files) { +MySQLMetaImpl::CleanUpCacheWithTTL(uint64_t seconds, CleanUpFilter* filter) { auto now = utils::GetMicroSecTimeStamp(); // erase deleted/backup files from cache @@ -1813,14 +1813,11 @@ MySQLMetaImpl::CleanUpCacheWithTTL(uint64_t seconds, const Table2FileIDs& ignore resRow["file_id"].to_string(table_file.file_id_); table_file.date_ = resRow["date"]; - // check if the file can be deleted - auto iter = ignore_files.find(table_file.table_id_); - if (iter != ignore_files.end()) { - if (iter->second.find(table_file.file_id_) != iter->second.end()) { - ENGINE_LOG_DEBUG << "File:" << table_file.file_id_ - << " currently is in use, not able to erase from cache now"; - continue; // ignore this file, don't delete it - } + // check if the file can be erased + if (filter && filter->IsIgnored(table_file)) { + ENGINE_LOG_DEBUG << "File:" << table_file.file_id_ + << " currently is in use, not able to erase from cache now"; + continue; // ignore this file, don't erase it } // erase file data from cache @@ -1835,7 +1832,7 @@ MySQLMetaImpl::CleanUpCacheWithTTL(uint64_t seconds, const Table2FileIDs& ignore } Status -MySQLMetaImpl::CleanUpFilesWithTTL(uint64_t seconds, const Table2FileIDs& ignore_files) { +MySQLMetaImpl::CleanUpFilesWithTTL(uint64_t seconds, CleanUpFilter* filter) { auto now = utils::GetMicroSecTimeStamp(); std::set table_ids; @@ -1869,13 +1866,10 @@ MySQLMetaImpl::CleanUpFilesWithTTL(uint64_t seconds, const Table2FileIDs& ignore table_file.date_ = resRow["date"]; // check if the file can be deleted - auto iter = ignore_files.find(table_file.table_id_); - if (iter != ignore_files.end()) { - if (iter->second.find(table_file.file_id_) != iter->second.end()) { - ENGINE_LOG_DEBUG << "File:" << table_file.file_id_ - << " currently is in use, not able to delete now"; - continue; // ignore this file, don't delete it - } + if (filter && filter->IsIgnored(table_file)) { + ENGINE_LOG_DEBUG << "File:" << table_file.file_id_ + << " currently is in use, not able to delete now"; + continue; // ignore this file, don't delete it } // delete file from disk storage diff --git a/core/src/db/meta/MySQLMetaImpl.h b/core/src/db/meta/MySQLMetaImpl.h index d13ed7e043..d2a94f6cd3 100644 --- a/core/src/db/meta/MySQLMetaImpl.h +++ b/core/src/db/meta/MySQLMetaImpl.h @@ -120,10 +120,10 @@ class MySQLMetaImpl : public Meta { CleanUpShadowFiles() override; Status - CleanUpCacheWithTTL(uint64_t seconds, const Table2FileIDs& ignore_files) override; + CleanUpCacheWithTTL(uint64_t seconds, CleanUpFilter* filter = nullptr) override; Status - CleanUpFilesWithTTL(uint64_t seconds, const Table2FileIDs& ignore_files) override; + CleanUpFilesWithTTL(uint64_t seconds, CleanUpFilter* filter = nullptr) override; Status DropAll() override; diff --git a/core/src/db/meta/SqliteMetaImpl.cpp b/core/src/db/meta/SqliteMetaImpl.cpp index b2f4102dde..9b63f122da 100644 --- a/core/src/db/meta/SqliteMetaImpl.cpp +++ b/core/src/db/meta/SqliteMetaImpl.cpp @@ -1294,7 +1294,7 @@ SqliteMetaImpl::CleanUpShadowFiles() { } Status -SqliteMetaImpl::CleanUpCacheWithTTL(uint64_t seconds, const Table2FileIDs& ignore_files) { +SqliteMetaImpl::CleanUpCacheWithTTL(uint64_t seconds, CleanUpFilter* filter) { auto now = utils::GetMicroSecTimeStamp(); // erase deleted/backup files from cache @@ -1327,14 +1327,11 @@ SqliteMetaImpl::CleanUpCacheWithTTL(uint64_t seconds, const Table2FileIDs& ignor table_file.file_id_ = std::get<2>(file); table_file.date_ = std::get<3>(file); - // check if the file can be deleted - auto iter = ignore_files.find(table_file.table_id_); - if (iter != ignore_files.end()) { - if (iter->second.find(table_file.file_id_) != iter->second.end()) { - ENGINE_LOG_DEBUG << "File:" << table_file.file_id_ - << " currently is in use, not able to erase from cache now"; - continue; // ignore this file, don't delete it - } + // check if the file can be erased + if (filter && filter->IsIgnored(table_file)) { + ENGINE_LOG_DEBUG << "File:" << table_file.file_id_ + << " currently is in use, not able to erase from cache now"; + continue; // ignore this file, don't erase it } // erase file data from cache @@ -1350,7 +1347,7 @@ SqliteMetaImpl::CleanUpCacheWithTTL(uint64_t seconds, const Table2FileIDs& ignor } Status -SqliteMetaImpl::CleanUpFilesWithTTL(uint64_t seconds, const Table2FileIDs& ignore_files) { +SqliteMetaImpl::CleanUpFilesWithTTL(uint64_t seconds, CleanUpFilter* filter) { auto now = utils::GetMicroSecTimeStamp(); std::set table_ids; @@ -1382,13 +1379,10 @@ SqliteMetaImpl::CleanUpFilesWithTTL(uint64_t seconds, const Table2FileIDs& ignor table_file.date_ = std::get<3>(file); // check if the file can be deleted - auto iter = ignore_files.find(table_file.table_id_); - if (iter != ignore_files.end()) { - if (iter->second.find(table_file.file_id_) != iter->second.end()) { - ENGINE_LOG_DEBUG << "File:" << table_file.file_id_ - << " currently is in use, not able to delete now"; - continue; // ignore this file, don't delete it - } + if (filter && filter->IsIgnored(table_file)) { + ENGINE_LOG_DEBUG << "File:" << table_file.file_id_ + << " currently is in use, not able to delete now"; + continue; // ignore this file, don't delete it } // delete file from meta diff --git a/core/src/db/meta/SqliteMetaImpl.h b/core/src/db/meta/SqliteMetaImpl.h index 4ef4542022..16bbc0a205 100644 --- a/core/src/db/meta/SqliteMetaImpl.h +++ b/core/src/db/meta/SqliteMetaImpl.h @@ -120,10 +120,10 @@ class SqliteMetaImpl : public Meta { CleanUpShadowFiles() override; Status - CleanUpCacheWithTTL(uint64_t seconds, const Table2FileIDs& ignore_files) override; + CleanUpCacheWithTTL(uint64_t seconds, CleanUpFilter* filter = nullptr) override; Status - CleanUpFilesWithTTL(uint64_t seconds, const Table2FileIDs& ignore_files) override; + CleanUpFilesWithTTL(uint64_t seconds, CleanUpFilter* filter = nullptr) override; Status DropAll() override; diff --git a/core/unittest/db/test_meta.cpp b/core/unittest/db/test_meta.cpp index c2a9f8e8bc..b89c73c296 100644 --- a/core/unittest/db/test_meta.cpp +++ b/core/unittest/db/test_meta.cpp @@ -335,8 +335,7 @@ TEST_F(MetaTest, TABLE_FILES_TEST) { status = impl_->DropTable(table_id); ASSERT_TRUE(status.ok()); - milvus::engine::meta::Table2FileIDs ignore_files; - status = impl_->CleanUpFilesWithTTL(1UL, ignore_files); + status = impl_->CleanUpFilesWithTTL(1UL); ASSERT_TRUE(status.ok()); } diff --git a/core/unittest/db/test_meta_mysql.cpp b/core/unittest/db/test_meta_mysql.cpp index cdd8aa32a8..9a52a01b7b 100644 --- a/core/unittest/db/test_meta_mysql.cpp +++ b/core/unittest/db/test_meta_mysql.cpp @@ -349,8 +349,7 @@ TEST_F(MySqlMetaTest, TABLE_FILES_TEST) { status = impl_->DropTable(table_id); ASSERT_TRUE(status.ok()); - milvus::engine::meta::Table2FileIDs ignore_files; - status = impl_->CleanUpFilesWithTTL(0UL, ignore_files); + status = impl_->CleanUpFilesWithTTL(0UL); ASSERT_TRUE(status.ok()); } diff --git a/core/unittest/db/test_misc.cpp b/core/unittest/db/test_misc.cpp index 1ed02afb53..3e9d597c3f 100644 --- a/core/unittest/db/test_misc.cpp +++ b/core/unittest/db/test_misc.cpp @@ -20,6 +20,7 @@ #include "db/Options.h" #include "db/Utils.h" #include "db/engine/EngineFactory.h" +#include "db/meta/MetaTypes.h" #include "db/meta/SqliteMetaImpl.h" #include "utils/Exception.h" #include "utils/Status.h" @@ -161,25 +162,21 @@ TEST(DBMiscTest, CHECKER_TEST) { schema.file_id_ = "5000"; checker.MarkOngoingFile(schema); - auto ongoing_files = checker.GetOngoingFiles(); - ASSERT_EQ(ongoing_files.size(), 1UL); + ASSERT_TRUE(checker.IsIgnored(schema)); schema.table_id_ = "bbb"; schema.file_id_ = "5001"; milvus::engine::meta::TableFilesSchema table_files = {schema}; checker.MarkOngoingFiles(table_files); - ongoing_files = checker.GetOngoingFiles(); - ASSERT_EQ(ongoing_files.size(), 2UL); + ASSERT_TRUE(checker.IsIgnored(schema)); checker.UnmarkOngoingFile(schema); - ongoing_files = checker.GetOngoingFiles(); - ASSERT_EQ(ongoing_files.size(), 1UL); + ASSERT_FALSE(checker.IsIgnored(schema)); schema.table_id_ = "aaa"; schema.file_id_ = "5000"; checker.UnmarkOngoingFile(schema); - ongoing_files = checker.GetOngoingFiles(); - ASSERT_EQ(ongoing_files.size(), 0UL); + ASSERT_FALSE(checker.IsIgnored(schema)); } } From 22e86eb0a0ede3fc6a9eae3dc18f31500b40478e Mon Sep 17 00:00:00 2001 From: groot Date: Fri, 29 Nov 2019 12:47:38 +0800 Subject: [PATCH 13/44] fix a bug --- core/src/db/OngoingFileChecker.cpp | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/core/src/db/OngoingFileChecker.cpp b/core/src/db/OngoingFileChecker.cpp index daaf6a2a4f..308649573d 100644 --- a/core/src/db/OngoingFileChecker.cpp +++ b/core/src/db/OngoingFileChecker.cpp @@ -108,15 +108,21 @@ OngoingFileChecker::UnmarkOngoingFileNoLock(const meta::TableFileSchema& table_f auto iter = ongoing_files_.find(table_file.table_id_); if (iter != ongoing_files_.end()) { - iter->second.erase(table_file.file_id_); - if (iter->second.empty()) { - ongoing_files_.erase(table_file.table_id_); + auto it_file = iter->second.find(table_file.file_id_); + if (it_file != iter->second.end()) { + it_file->second--; + + ENGINE_LOG_DEBUG << "Unmark ongoing file:" << table_file.file_id_ << " refcount:" << it_file->second; + + if (it_file->second <= 0) { + iter->second.erase(table_file.file_id_); + if (iter->second.empty()) { + ongoing_files_.erase(table_file.table_id_); + } + } } } - ENGINE_LOG_DEBUG << "Mark ongoing file:" << table_file.file_id_ - << " refcount:" << ongoing_files_[table_file.table_id_][table_file.file_id_]; - return Status::OK(); } From c220e0e1577eb96aac782a507a002bdf9c5b3545 Mon Sep 17 00:00:00 2001 From: groot Date: Fri, 29 Nov 2019 16:18:57 +0800 Subject: [PATCH 14/44] refine code --- core/src/db/IndexFailedChecker.cpp | 4 ++-- core/src/db/IndexFailedChecker.h | 3 ++- core/src/db/OngoingFileChecker.cpp | 2 +- core/src/db/OngoingFileChecker.h | 3 ++- core/src/db/Types.h | 6 ++++++ core/src/db/engine/ExecutionEngineImpl.cpp | 2 ++ core/src/db/meta/MetaTypes.h | 4 ---- core/unittest/db/test_misc.cpp | 1 - 8 files changed, 15 insertions(+), 10 deletions(-) diff --git a/core/src/db/IndexFailedChecker.cpp b/core/src/db/IndexFailedChecker.cpp index bdd152dc4b..2ed22d75c4 100644 --- a/core/src/db/IndexFailedChecker.cpp +++ b/core/src/db/IndexFailedChecker.cpp @@ -38,7 +38,7 @@ IndexFailedChecker::GetFailedIndexFileOfTable(const std::string& table_id, std:: std::lock_guard lck(mutex_); auto iter = index_failed_files_.find(table_id); if (iter != index_failed_files_.end()) { - meta::File2RefCount& failed_map = iter->second; + File2RefCount& failed_map = iter->second; for (auto it_file = failed_map.begin(); it_file != failed_map.end(); ++it_file) { failed_files.push_back(it_file->first); } @@ -53,7 +53,7 @@ IndexFailedChecker::MarkFailedIndexFile(const meta::TableFileSchema& file) { auto iter = index_failed_files_.find(file.table_id_); if (iter == index_failed_files_.end()) { - meta::File2RefCount failed_files; + File2RefCount failed_files; failed_files.insert(std::make_pair(file.file_id_, 1)); index_failed_files_.insert(std::make_pair(file.table_id_, failed_files)); } else { diff --git a/core/src/db/IndexFailedChecker.h b/core/src/db/IndexFailedChecker.h index edd6a114bf..cf9ea990fe 100644 --- a/core/src/db/IndexFailedChecker.h +++ b/core/src/db/IndexFailedChecker.h @@ -17,6 +17,7 @@ #pragma once +#include "db/Types.h" #include "meta/Meta.h" #include "utils/Status.h" @@ -47,7 +48,7 @@ class IndexFailedChecker { private: std::mutex mutex_; - meta::Table2Files index_failed_files_; // table id mapping to (file id mapping to failed times) + Table2Files index_failed_files_; // table id mapping to (file id mapping to failed times) }; } // namespace engine diff --git a/core/src/db/OngoingFileChecker.cpp b/core/src/db/OngoingFileChecker.cpp index 308649573d..3c3eb4011a 100644 --- a/core/src/db/OngoingFileChecker.cpp +++ b/core/src/db/OngoingFileChecker.cpp @@ -82,7 +82,7 @@ OngoingFileChecker::MarkOngoingFileNoLock(const meta::TableFileSchema& table_fil auto iter = ongoing_files_.find(table_file.table_id_); if (iter == ongoing_files_.end()) { - meta::File2RefCount files_refcount; + File2RefCount files_refcount; files_refcount.insert(std::make_pair(table_file.file_id_, 1)); ongoing_files_.insert(std::make_pair(table_file.table_id_, files_refcount)); } else { diff --git a/core/src/db/OngoingFileChecker.h b/core/src/db/OngoingFileChecker.h index c7995e425a..2e52fdeea6 100644 --- a/core/src/db/OngoingFileChecker.h +++ b/core/src/db/OngoingFileChecker.h @@ -17,6 +17,7 @@ #pragma once +#include "db/Types.h" #include "meta/Meta.h" #include "utils/Status.h" @@ -54,7 +55,7 @@ class OngoingFileChecker : public meta::Meta::CleanUpFilter { private: std::mutex mutex_; - meta::Table2Files ongoing_files_; // table id mapping to (file id mapping to ongoing ref-count) + Table2Files ongoing_files_; // table id mapping to (file id mapping to ongoing ref-count) }; } // namespace engine diff --git a/core/src/db/Types.h b/core/src/db/Types.h index 76c06126f8..ca6f97849a 100644 --- a/core/src/db/Types.h +++ b/core/src/db/Types.h @@ -21,6 +21,9 @@ #include #include +#include +#include +#include #include #include @@ -40,5 +43,8 @@ struct TableIndex { int32_t metric_type_ = (int)MetricType::L2; }; +using File2RefCount = std::map; +using Table2Files = std::map; + } // namespace engine } // namespace milvus diff --git a/core/src/db/engine/ExecutionEngineImpl.cpp b/core/src/db/engine/ExecutionEngineImpl.cpp index 5a3d6e5e2a..a2ef4c70d4 100644 --- a/core/src/db/engine/ExecutionEngineImpl.cpp +++ b/core/src/db/engine/ExecutionEngineImpl.cpp @@ -258,6 +258,7 @@ ExecutionEngineImpl::PhysicalSize() const { Status ExecutionEngineImpl::Serialize() { + ENGINE_LOG_DEBUG << "Serialize index size: " << index_->Size() << " to file: " << location_; auto status = write_index(index_, location_); // here we reset index size by file size, @@ -495,6 +496,7 @@ ExecutionEngineImpl::BuildIndex(const std::string& location, EngineType engine_t throw Exception(DB_ERROR, status.message()); } + ENGINE_LOG_DEBUG << "Sucessfully build index file: " << location << " size: " << to_index->Size(); return std::make_shared(to_index, location, engine_type, metric_type_, nlist_); } diff --git a/core/src/db/meta/MetaTypes.h b/core/src/db/meta/MetaTypes.h index f8c082032e..d98b74be7d 100644 --- a/core/src/db/meta/MetaTypes.h +++ b/core/src/db/meta/MetaTypes.h @@ -23,7 +23,6 @@ #include #include -#include #include #include @@ -98,9 +97,6 @@ using TableFileSchemaPtr = std::shared_ptr; using TableFilesSchema = std::vector; using DatePartionedTableFilesSchema = std::map; -using File2RefCount = std::map; -using Table2Files = std::map; - } // namespace meta } // namespace engine } // namespace milvus diff --git a/core/unittest/db/test_misc.cpp b/core/unittest/db/test_misc.cpp index 3e9d597c3f..326f705184 100644 --- a/core/unittest/db/test_misc.cpp +++ b/core/unittest/db/test_misc.cpp @@ -20,7 +20,6 @@ #include "db/Options.h" #include "db/Utils.h" #include "db/engine/EngineFactory.h" -#include "db/meta/MetaTypes.h" #include "db/meta/SqliteMetaImpl.h" #include "utils/Exception.h" #include "utils/Status.h" From 736c348cfc1c848d0843cb4e14bc41fa719763c5 Mon Sep 17 00:00:00 2001 From: groot Date: Fri, 29 Nov 2019 17:20:16 +0800 Subject: [PATCH 15/44] add log --- core/src/db/DBImpl.cpp | 4 ++-- core/src/db/engine/ExecutionEngineImpl.cpp | 4 ++-- core/src/scheduler/task/BuildIndexTask.cpp | 13 +++++++++++-- 3 files changed, 15 insertions(+), 6 deletions(-) diff --git a/core/src/db/DBImpl.cpp b/core/src/db/DBImpl.cpp index c88936a08e..1db686b5ba 100644 --- a/core/src/db/DBImpl.cpp +++ b/core/src/db/DBImpl.cpp @@ -792,12 +792,12 @@ DBImpl::BackgroundCompaction(std::set table_ids) { meta_ptr_->Archive(); { - uint64_t ttl = 1 * meta::SECOND; // default: file data will be erase from cache after few seconds + uint64_t ttl = 10 * meta::SECOND; // default: file data will be erase from cache after few seconds meta_ptr_->CleanUpCacheWithTTL(ttl, &ongoing_files_checker_); } { - uint64_t ttl = 1 * meta::SECOND; // default: file will be deleted after few seconds + uint64_t ttl = 20 * meta::SECOND; // default: file will be deleted after few seconds if (options_.mode_ == DBOptions::MODE::CLUSTER_WRITABLE) { ttl = meta::H_SEC; } diff --git a/core/src/db/engine/ExecutionEngineImpl.cpp b/core/src/db/engine/ExecutionEngineImpl.cpp index a2ef4c70d4..c81d5d8e54 100644 --- a/core/src/db/engine/ExecutionEngineImpl.cpp +++ b/core/src/db/engine/ExecutionEngineImpl.cpp @@ -258,12 +258,12 @@ ExecutionEngineImpl::PhysicalSize() const { Status ExecutionEngineImpl::Serialize() { - ENGINE_LOG_DEBUG << "Serialize index size: " << index_->Size() << " to file: " << location_; auto status = write_index(index_, location_); // here we reset index size by file size, // since some index type(such as SQ8) data size become smaller after serialized index_->set_size(PhysicalSize()); + ENGINE_LOG_DEBUG << "Finish serialize index file: " << location_ << " size: " << index_->Size(); return status; } @@ -496,7 +496,7 @@ ExecutionEngineImpl::BuildIndex(const std::string& location, EngineType engine_t throw Exception(DB_ERROR, status.message()); } - ENGINE_LOG_DEBUG << "Sucessfully build index file: " << location << " size: " << to_index->Size(); + ENGINE_LOG_DEBUG << "Finish build index file: " << location << " size: " << to_index->Size(); return std::make_shared(to_index, location, engine_type, metric_type_, nlist_); } diff --git a/core/src/scheduler/task/BuildIndexTask.cpp b/core/src/scheduler/task/BuildIndexTask.cpp index e952bd0938..681968c402 100644 --- a/core/src/scheduler/task/BuildIndexTask.cpp +++ b/core/src/scheduler/task/BuildIndexTask.cpp @@ -168,7 +168,10 @@ XBuildIndexTask::Execute() { // step 5: save index file try { - index->Serialize(); + status = index->Serialize(); + if (status.ok()) { + ENGINE_LOG_DEBUG << "Failed to serilize index file: " << status.message(); + } } catch (std::exception& ex) { // typical error: out of disk space or permition denied std::string msg = "Serialize index encounter exception: " + std::string(ex.what()); @@ -196,7 +199,13 @@ XBuildIndexTask::Execute() { origin_file.file_type_ = engine::meta::TableFileSchema::BACKUP; engine::meta::TableFilesSchema update_files = {table_file, origin_file}; - status = meta_ptr->UpdateTableFiles(update_files); + + if (table_file.file_size_ > 0) { // makesure index file is sucessfully serialized to disk + status = meta_ptr->UpdateTableFiles(update_files); + } else { + status = Status(DB_ERROR, "Illegal index file: out of disk space or memory"); + } + if (status.ok()) { ENGINE_LOG_DEBUG << "New index file " << table_file.file_id_ << " of size " << index->PhysicalSize() << " bytes" From 4a183c411678282b7e38a8d9eeedd64172a4461d Mon Sep 17 00:00:00 2001 From: groot Date: Fri, 29 Nov 2019 17:27:01 +0800 Subject: [PATCH 16/44] add log --- core/src/cache/Cache.inl | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/core/src/cache/Cache.inl b/core/src/cache/Cache.inl index 9ebec7cfdd..e5e0baf9e7 100644 --- a/core/src/cache/Cache.inl +++ b/core/src/cache/Cache.inl @@ -179,6 +179,11 @@ Cache::print() { } SERVER_LOG_DEBUG << "[Cache item count]: " << cache_count; +#if 0 + for (auto it = lru_.begin(); it != lru_.end(); ++it) { + SERVER_LOG_DEBUG << it->first; + } +#endif SERVER_LOG_DEBUG << "[Cache usage]: " << usage_ << " bytes"; SERVER_LOG_DEBUG << "[Cache capacity]: " << capacity_ << " bytes"; } From bc2dba26a289fa532ac82915e8d0829164cf5848 Mon Sep 17 00:00:00 2001 From: groot Date: Fri, 29 Nov 2019 18:15:28 +0800 Subject: [PATCH 17/44] refine code --- core/src/cache/Cache.inl | 2 +- core/src/db/DBImpl.cpp | 24 ++++----- core/src/db/engine/ExecutionEngineImpl.cpp | 5 ++ core/src/db/meta/Meta.h | 3 -- core/src/db/meta/MySQLMetaImpl.cpp | 52 ++------------------ core/src/db/meta/MySQLMetaImpl.h | 3 -- core/src/db/meta/SqliteMetaImpl.cpp | 57 ++-------------------- core/src/db/meta/SqliteMetaImpl.h | 3 -- core/src/scheduler/task/BuildIndexTask.cpp | 16 +++--- 9 files changed, 35 insertions(+), 130 deletions(-) diff --git a/core/src/cache/Cache.inl b/core/src/cache/Cache.inl index e5e0baf9e7..a3acda41d2 100644 --- a/core/src/cache/Cache.inl +++ b/core/src/cache/Cache.inl @@ -179,7 +179,7 @@ Cache::print() { } SERVER_LOG_DEBUG << "[Cache item count]: " << cache_count; -#if 0 +#if 1 for (auto it = lru_.begin(); it != lru_.end(); ++it) { SERVER_LOG_DEBUG << it->first; } diff --git a/core/src/db/DBImpl.cpp b/core/src/db/DBImpl.cpp index 1db686b5ba..129a9e9928 100644 --- a/core/src/db/DBImpl.cpp +++ b/core/src/db/DBImpl.cpp @@ -705,20 +705,27 @@ DBImpl::MergeFiles(const std::string& table_id, const meta::DateT& date, const m // step 3: serialize to disk try { - index->Serialize(); + status = index->Serialize(); + if (status.ok()) { + ENGINE_LOG_ERROR << status.message(); + } } catch (std::exception& ex) { - // typical error: out of disk space or permition denied std::string msg = "Serialize merged index encounter exception: " + std::string(ex.what()); ENGINE_LOG_ERROR << msg; + status = Status(DB_ERROR, msg); + } + if (!status.ok()) { + // if failed to serialize merge file to disk + // typical error: out of disk space, out of memory or permition denied table_file.file_type_ = meta::TableFileSchema::TO_DELETE; status = meta_ptr_->UpdateTableFile(table_file); ENGINE_LOG_DEBUG << "Failed to update file to index, mark file: " << table_file.file_id_ << " to to_delete"; - std::cout << "ERROR: failed to persist merged index file: " << table_file.location_ - << ", possible out of disk space" << std::endl; + ENGINE_LOG_ERROR << "ERROR: failed to persist merged file: " << table_file.location_ + << ", possible out of disk space or memory"; - return Status(DB_ERROR, msg); + return status; } // step 4: update table files state @@ -792,12 +799,7 @@ DBImpl::BackgroundCompaction(std::set table_ids) { meta_ptr_->Archive(); { - uint64_t ttl = 10 * meta::SECOND; // default: file data will be erase from cache after few seconds - meta_ptr_->CleanUpCacheWithTTL(ttl, &ongoing_files_checker_); - } - - { - uint64_t ttl = 20 * meta::SECOND; // default: file will be deleted after few seconds + uint64_t ttl = 1 * meta::SECOND; // default: file will be deleted after few seconds if (options_.mode_ == DBOptions::MODE::CLUSTER_WRITABLE) { ttl = meta::H_SEC; } diff --git a/core/src/db/engine/ExecutionEngineImpl.cpp b/core/src/db/engine/ExecutionEngineImpl.cpp index c81d5d8e54..a87690c6d6 100644 --- a/core/src/db/engine/ExecutionEngineImpl.cpp +++ b/core/src/db/engine/ExecutionEngineImpl.cpp @@ -265,6 +265,11 @@ ExecutionEngineImpl::Serialize() { index_->set_size(PhysicalSize()); ENGINE_LOG_DEBUG << "Finish serialize index file: " << location_ << " size: " << index_->Size(); + if (index_->Size() == 0) { + std::string msg = "Failed to serialize file: " + location_ + " reason: out of disk space or memory"; + status = Status(DB_ERROR, msg); + } + return status; } diff --git a/core/src/db/meta/Meta.h b/core/src/db/meta/Meta.h index 8dabb58203..1929414fdc 100644 --- a/core/src/db/meta/Meta.h +++ b/core/src/db/meta/Meta.h @@ -127,9 +127,6 @@ class Meta { virtual Status CleanUpShadowFiles() = 0; - virtual Status - CleanUpCacheWithTTL(uint64_t seconds, CleanUpFilter* filter = nullptr) = 0; - virtual Status CleanUpFilesWithTTL(uint64_t seconds, CleanUpFilter* filter = nullptr) = 0; diff --git a/core/src/db/meta/MySQLMetaImpl.cpp b/core/src/db/meta/MySQLMetaImpl.cpp index 1fab7a0f77..7f7bd704d5 100644 --- a/core/src/db/meta/MySQLMetaImpl.cpp +++ b/core/src/db/meta/MySQLMetaImpl.cpp @@ -1782,55 +1782,6 @@ MySQLMetaImpl::CleanUpShadowFiles() { return Status::OK(); } -Status -MySQLMetaImpl::CleanUpCacheWithTTL(uint64_t seconds, CleanUpFilter* filter) { - auto now = utils::GetMicroSecTimeStamp(); - - // erase deleted/backup files from cache - try { - server::MetricCollector metric; - - mysqlpp::ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); - - if (connectionPtr == nullptr) { - return Status(DB_ERROR, "Failed to connect to meta server(mysql)"); - } - - mysqlpp::Query query = connectionPtr->query(); - query << "SELECT id, table_id, file_id, date" - << " FROM " << META_TABLEFILES << " WHERE file_type IN (" << std::to_string(TableFileSchema::TO_DELETE) - << "," << std::to_string(TableFileSchema::BACKUP) << ")" - << " AND updated_time < " << std::to_string(now - seconds * US_PS) << ";"; - - mysqlpp::StoreQueryResult res = query.store(); - - TableFileSchema table_file; - std::vector idsToDelete; - - for (auto& resRow : res) { - table_file.id_ = resRow["id"]; // implicit conversion - resRow["table_id"].to_string(table_file.table_id_); - resRow["file_id"].to_string(table_file.file_id_); - table_file.date_ = resRow["date"]; - - // check if the file can be erased - if (filter && filter->IsIgnored(table_file)) { - ENGINE_LOG_DEBUG << "File:" << table_file.file_id_ - << " currently is in use, not able to erase from cache now"; - continue; // ignore this file, don't erase it - } - - // erase file data from cache - utils::GetTableFilePath(options_, table_file); - server::CommonUtil::EraseFromCache(table_file.location_); - } - } catch (std::exception& e) { - return HandleException("GENERAL ERROR WHEN CLEANING UP FILES WITH TTL", e.what()); - } - - return Status::OK(); -} - Status MySQLMetaImpl::CleanUpFilesWithTTL(uint64_t seconds, CleanUpFilter* filter) { auto now = utils::GetMicroSecTimeStamp(); @@ -1876,6 +1827,9 @@ MySQLMetaImpl::CleanUpFilesWithTTL(uint64_t seconds, CleanUpFilter* filter) { utils::DeleteTableFilePath(options_, table_file); ENGINE_LOG_DEBUG << "Removing file id:" << table_file.id_ << " location:" << table_file.location_; + // erase file data from cache + server::CommonUtil::EraseFromCache(table_file.location_); + idsToDelete.emplace_back(std::to_string(table_file.id_)); table_ids.insert(table_file.table_id_); } diff --git a/core/src/db/meta/MySQLMetaImpl.h b/core/src/db/meta/MySQLMetaImpl.h index d2a94f6cd3..6d2e7cf4c2 100644 --- a/core/src/db/meta/MySQLMetaImpl.h +++ b/core/src/db/meta/MySQLMetaImpl.h @@ -119,9 +119,6 @@ class MySQLMetaImpl : public Meta { Status CleanUpShadowFiles() override; - Status - CleanUpCacheWithTTL(uint64_t seconds, CleanUpFilter* filter = nullptr) override; - Status CleanUpFilesWithTTL(uint64_t seconds, CleanUpFilter* filter = nullptr) override; diff --git a/core/src/db/meta/SqliteMetaImpl.cpp b/core/src/db/meta/SqliteMetaImpl.cpp index 9b63f122da..2a3867574f 100644 --- a/core/src/db/meta/SqliteMetaImpl.cpp +++ b/core/src/db/meta/SqliteMetaImpl.cpp @@ -1293,59 +1293,6 @@ SqliteMetaImpl::CleanUpShadowFiles() { return Status::OK(); } -Status -SqliteMetaImpl::CleanUpCacheWithTTL(uint64_t seconds, CleanUpFilter* filter) { - auto now = utils::GetMicroSecTimeStamp(); - - // erase deleted/backup files from cache - try { - server::MetricCollector metric; - - // multi-threads call sqlite update may get exception('bad logic', etc), so we add a lock here - std::lock_guard meta_lock(meta_mutex_); - - std::vector file_types = { - (int)TableFileSchema::TO_DELETE, - (int)TableFileSchema::BACKUP, - }; - - // collect files to be erased - auto files = ConnectorPtr->select(columns(&TableFileSchema::id_, - &TableFileSchema::table_id_, - &TableFileSchema::file_id_, - &TableFileSchema::date_), - where( - in(&TableFileSchema::file_type_, file_types) - and - c(&TableFileSchema::updated_time_) - < now - seconds * US_PS)); - - for (auto& file : files) { - TableFileSchema table_file; - table_file.id_ = std::get<0>(file); - table_file.table_id_ = std::get<1>(file); - table_file.file_id_ = std::get<2>(file); - table_file.date_ = std::get<3>(file); - - // check if the file can be erased - if (filter && filter->IsIgnored(table_file)) { - ENGINE_LOG_DEBUG << "File:" << table_file.file_id_ - << " currently is in use, not able to erase from cache now"; - continue; // ignore this file, don't erase it - } - - // erase file data from cache - utils::GetTableFilePath(options_, table_file); - server::CommonUtil::EraseFromCache(table_file.location_); - } - - } catch (std::exception& e) { - return HandleException("Encounter exception when clean cache", e.what()); - } - - return Status::OK(); -} - Status SqliteMetaImpl::CleanUpFilesWithTTL(uint64_t seconds, CleanUpFilter* filter) { auto now = utils::GetMicroSecTimeStamp(); @@ -1390,6 +1337,10 @@ SqliteMetaImpl::CleanUpFilesWithTTL(uint64_t seconds, CleanUpFilter* filter) { // delete file from disk storage utils::DeleteTableFilePath(options_, table_file); + + // erase from cache + server::CommonUtil::EraseFromCache(table_file.location_); + ENGINE_LOG_DEBUG << "Removing file id:" << table_file.file_id_ << " location:" << table_file.location_; table_ids.insert(table_file.table_id_); } diff --git a/core/src/db/meta/SqliteMetaImpl.h b/core/src/db/meta/SqliteMetaImpl.h index 16bbc0a205..f50fa452f3 100644 --- a/core/src/db/meta/SqliteMetaImpl.h +++ b/core/src/db/meta/SqliteMetaImpl.h @@ -119,9 +119,6 @@ class SqliteMetaImpl : public Meta { Status CleanUpShadowFiles() override; - Status - CleanUpCacheWithTTL(uint64_t seconds, CleanUpFilter* filter = nullptr) override; - Status CleanUpFilesWithTTL(uint64_t seconds, CleanUpFilter* filter = nullptr) override; diff --git a/core/src/scheduler/task/BuildIndexTask.cpp b/core/src/scheduler/task/BuildIndexTask.cpp index 681968c402..b2d74e794b 100644 --- a/core/src/scheduler/task/BuildIndexTask.cpp +++ b/core/src/scheduler/task/BuildIndexTask.cpp @@ -170,22 +170,26 @@ XBuildIndexTask::Execute() { try { status = index->Serialize(); if (status.ok()) { - ENGINE_LOG_DEBUG << "Failed to serilize index file: " << status.message(); + ENGINE_LOG_ERROR << status.message(); } } catch (std::exception& ex) { - // typical error: out of disk space or permition denied std::string msg = "Serialize index encounter exception: " + std::string(ex.what()); ENGINE_LOG_ERROR << msg; + status = Status(DB_ERROR, msg); + } + if (!status.ok()) { + // if failed to serialize index file to disk + // typical error: out of disk space, out of memory or permition denied table_file.file_type_ = engine::meta::TableFileSchema::TO_DELETE; status = meta_ptr->UpdateTableFile(table_file); ENGINE_LOG_DEBUG << "Failed to update file to index, mark file: " << table_file.file_id_ << " to to_delete"; ENGINE_LOG_ERROR << "Failed to persist index file: " << table_file.location_ - << ", possible out of disk space"; + << ", possible out of disk space or memory"; build_index_job->BuildIndexDone(to_index_id_); - build_index_job->GetStatus() = Status(DB_ERROR, msg); + build_index_job->GetStatus() = status; to_index_engine_ = nullptr; return; } @@ -200,10 +204,8 @@ XBuildIndexTask::Execute() { engine::meta::TableFilesSchema update_files = {table_file, origin_file}; - if (table_file.file_size_ > 0) { // makesure index file is sucessfully serialized to disk + if (status.ok()) { // makesure index file is sucessfully serialized to disk status = meta_ptr->UpdateTableFiles(update_files); - } else { - status = Status(DB_ERROR, "Illegal index file: out of disk space or memory"); } if (status.ok()) { From 844feab9260c31da82c042cced6265a2d0f2dfc6 Mon Sep 17 00:00:00 2001 From: groot Date: Fri, 29 Nov 2019 19:15:11 +0800 Subject: [PATCH 18/44] more log --- core/src/db/DBImpl.cpp | 2 +- core/src/scheduler/task/BuildIndexTask.cpp | 2 +- core/src/server/grpc_impl/request/InsertRequest.cpp | 4 +++- core/src/server/grpc_impl/request/SearchRequest.cpp | 4 +++- 4 files changed, 8 insertions(+), 4 deletions(-) diff --git a/core/src/db/DBImpl.cpp b/core/src/db/DBImpl.cpp index 129a9e9928..11f45501ce 100644 --- a/core/src/db/DBImpl.cpp +++ b/core/src/db/DBImpl.cpp @@ -706,7 +706,7 @@ DBImpl::MergeFiles(const std::string& table_id, const meta::DateT& date, const m // step 3: serialize to disk try { status = index->Serialize(); - if (status.ok()) { + if (!status.ok()) { ENGINE_LOG_ERROR << status.message(); } } catch (std::exception& ex) { diff --git a/core/src/scheduler/task/BuildIndexTask.cpp b/core/src/scheduler/task/BuildIndexTask.cpp index b2d74e794b..571bb279a3 100644 --- a/core/src/scheduler/task/BuildIndexTask.cpp +++ b/core/src/scheduler/task/BuildIndexTask.cpp @@ -169,7 +169,7 @@ XBuildIndexTask::Execute() { // step 5: save index file try { status = index->Serialize(); - if (status.ok()) { + if (!status.ok()) { ENGINE_LOG_ERROR << status.message(); } } catch (std::exception& ex) { diff --git a/core/src/server/grpc_impl/request/InsertRequest.cpp b/core/src/server/grpc_impl/request/InsertRequest.cpp index f436db074e..16d16ca8dc 100644 --- a/core/src/server/grpc_impl/request/InsertRequest.cpp +++ b/core/src/server/grpc_impl/request/InsertRequest.cpp @@ -45,7 +45,9 @@ InsertRequest::Create(const ::milvus::grpc::InsertParam* insert_param, ::milvus: Status InsertRequest::OnExecute() { try { - TimeRecorder rc("InsertRequest"); + std::string hdr = "InsertRequest(table=" + insert_param_->table_name() + + ", n=" + std::to_string(insert_param_->row_record_array_size()) + ")"; + TimeRecorder rc(hdr); // step 1: check arguments auto status = ValidationUtil::ValidateTableName(insert_param_->table_name()); diff --git a/core/src/server/grpc_impl/request/SearchRequest.cpp b/core/src/server/grpc_impl/request/SearchRequest.cpp index 28f4ff723e..db0a55d48e 100644 --- a/core/src/server/grpc_impl/request/SearchRequest.cpp +++ b/core/src/server/grpc_impl/request/SearchRequest.cpp @@ -51,7 +51,9 @@ SearchRequest::OnExecute() { int64_t top_k = search_param_->topk(); int64_t nprobe = search_param_->nprobe(); - std::string hdr = "SearchRequest(k=" + std::to_string(top_k) + ", nprob=" + std::to_string(nprobe) + ")"; + std::string hdr = "SearchRequest(table=" + search_param_->table_name() + + ", nq=" + std::to_string(search_param_->query_record_array_size()) + + ", k=" + std::to_string(top_k) + ", nprob=" + std::to_string(nprobe) + ")"; TimeRecorder rc(hdr); // step 1: check table name From 2e3caca4dacdfbbd4aadfe3c513bf3fc4923cdfe Mon Sep 17 00:00:00 2001 From: groot Date: Sat, 30 Nov 2019 15:38:14 +0800 Subject: [PATCH 19/44] more log --- core/src/cache/Cache.inl | 10 +++--- core/src/db/DBImpl.cpp | 16 ++++----- core/src/db/engine/ExecutionEngineImpl.cpp | 2 +- core/src/db/meta/MySQLMetaImpl.cpp | 28 ++++++++++----- core/src/db/meta/SqliteMetaImpl.cpp | 41 ++++++++++++++-------- core/src/utils/CommonUtil.cpp | 2 +- 6 files changed, 60 insertions(+), 39 deletions(-) diff --git a/core/src/cache/Cache.inl b/core/src/cache/Cache.inl index a3acda41d2..5b68e54249 100644 --- a/core/src/cache/Cache.inl +++ b/core/src/cache/Cache.inl @@ -176,14 +176,14 @@ Cache::print() { { std::lock_guard lock(mutex_); cache_count = lru_.size(); +#if 0 + for (auto it = lru_.begin(); it != lru_.end(); ++it) { + SERVER_LOG_DEBUG << it->first; + } +#endif } SERVER_LOG_DEBUG << "[Cache item count]: " << cache_count; -#if 1 - for (auto it = lru_.begin(); it != lru_.end(); ++it) { - SERVER_LOG_DEBUG << it->first; - } -#endif SERVER_LOG_DEBUG << "[Cache usage]: " << usage_ << " bytes"; SERVER_LOG_DEBUG << "[Cache capacity]: " << capacity_ << " bytes"; } diff --git a/core/src/db/DBImpl.cpp b/core/src/db/DBImpl.cpp index 11f45501ce..c5e04fb5c6 100644 --- a/core/src/db/DBImpl.cpp +++ b/core/src/db/DBImpl.cpp @@ -722,7 +722,7 @@ DBImpl::MergeFiles(const std::string& table_id, const meta::DateT& date, const m status = meta_ptr_->UpdateTableFile(table_file); ENGINE_LOG_DEBUG << "Failed to update file to index, mark file: " << table_file.file_id_ << " to to_delete"; - ENGINE_LOG_ERROR << "ERROR: failed to persist merged file: " << table_file.location_ + ENGINE_LOG_ERROR << "Failed to persist merged file: " << table_file.location_ << ", possible out of disk space or memory"; return status; @@ -803,6 +803,7 @@ DBImpl::BackgroundCompaction(std::set table_ids) { if (options_.mode_ == DBOptions::MODE::CLUSTER_WRITABLE) { ttl = meta::H_SEC; } + meta_ptr_->CleanUpFilesWithTTL(ttl, &ongoing_files_checker_); } @@ -839,14 +840,13 @@ DBImpl::StartBuildIndexTask(bool force) { void DBImpl::BackgroundBuildIndex() { - // ENGINE_LOG_TRACE << "Background build index thread start"; - std::unique_lock lock(build_index_mutex_); meta::TableFilesSchema to_index_files; meta_ptr_->FilesToIndex(to_index_files); Status status = index_failed_checker_.IgnoreFailedIndexFiles(to_index_files); if (!to_index_files.empty()) { + ENGINE_LOG_DEBUG << "Background build index thread begin"; status = ongoing_files_checker_.MarkOngoingFiles(to_index_files); // step 2: put build index task to scheduler @@ -870,17 +870,15 @@ DBImpl::BackgroundBuildIndex() { index_failed_checker_.MarkFailedIndexFile(file_schema); } else { - index_failed_checker_.MarkSucceedIndexFile(file_schema); ENGINE_LOG_DEBUG << "Building index job " << job->id() << " succeed."; - } - } - status = ongoing_files_checker_.UnmarkOngoingFiles(to_index_files); + index_failed_checker_.MarkSucceedIndexFile(file_schema); + } + status = ongoing_files_checker_.UnmarkOngoingFile(file_schema); + } ENGINE_LOG_DEBUG << "Background build index thread finished"; } - - // ENGINE_LOG_TRACE << "Background build index thread exit"; } Status diff --git a/core/src/db/engine/ExecutionEngineImpl.cpp b/core/src/db/engine/ExecutionEngineImpl.cpp index a87690c6d6..6bb0b97817 100644 --- a/core/src/db/engine/ExecutionEngineImpl.cpp +++ b/core/src/db/engine/ExecutionEngineImpl.cpp @@ -463,7 +463,7 @@ ExecutionEngineImpl::Merge(const std::string& location) { if (auto file_index = std::dynamic_pointer_cast(to_merge)) { auto status = index_->Add(file_index->Count(), file_index->GetRawVectors(), file_index->GetRawIds()); if (!status.ok()) { - ENGINE_LOG_ERROR << "Merge: Add Error"; + ENGINE_LOG_ERROR << "Failed to merge: " << location << " to: " << location_; } return status; } else { diff --git a/core/src/db/meta/MySQLMetaImpl.cpp b/core/src/db/meta/MySQLMetaImpl.cpp index 7f7bd704d5..96e7109c93 100644 --- a/core/src/db/meta/MySQLMetaImpl.cpp +++ b/core/src/db/meta/MySQLMetaImpl.cpp @@ -1800,7 +1800,9 @@ MySQLMetaImpl::CleanUpFilesWithTTL(uint64_t seconds, CleanUpFilter* filter) { mysqlpp::Query query = connectionPtr->query(); query << "SELECT id, table_id, file_id, date" - << " FROM " << META_TABLEFILES << " WHERE file_type = " << std::to_string(TableFileSchema::TO_DELETE) + << " FROM " << META_TABLEFILES << " WHERE file_type IN (" + << std::to_string(TableFileSchema::TO_DELETE) << "," + << std::to_string(TableFileSchema::BACKUP) << ")" << " AND updated_time < " << std::to_string(now - seconds * US_PS) << ";"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUpFilesWithTTL: " << query.str(); @@ -1810,11 +1812,13 @@ MySQLMetaImpl::CleanUpFilesWithTTL(uint64_t seconds, CleanUpFilter* filter) { TableFileSchema table_file; std::vector idsToDelete; + int64_t clean_files = 0; for (auto& resRow : res) { table_file.id_ = resRow["id"]; // implicit conversion resRow["table_id"].to_string(table_file.table_id_); resRow["file_id"].to_string(table_file.file_id_); table_file.date_ = resRow["date"]; + table_file.file_type_ = resRow["file_type"]; // check if the file can be deleted if (filter && filter->IsIgnored(table_file)) { @@ -1823,15 +1827,21 @@ MySQLMetaImpl::CleanUpFilesWithTTL(uint64_t seconds, CleanUpFilter* filter) { continue; // ignore this file, don't delete it } - // delete file from disk storage - utils::DeleteTableFilePath(options_, table_file); - ENGINE_LOG_DEBUG << "Removing file id:" << table_file.id_ << " location:" << table_file.location_; - // erase file data from cache + // because GetTableFilePath won't able to generate file path after the file is deleted + utils::GetTableFilePath(options_, table_file); server::CommonUtil::EraseFromCache(table_file.location_); - idsToDelete.emplace_back(std::to_string(table_file.id_)); - table_ids.insert(table_file.table_id_); + if (table_file.file_type_ == (int)TableFileSchema::TO_DELETE) { + // delete file from disk storage + utils::DeleteTableFilePath(options_, table_file); + ENGINE_LOG_DEBUG << "Removing file id:" << table_file.id_ << " location:" << table_file.location_; + + idsToDelete.emplace_back(std::to_string(table_file.id_)); + table_ids.insert(table_file.table_id_); + } + + clean_files++; } // delete file from meta @@ -1852,8 +1862,8 @@ MySQLMetaImpl::CleanUpFilesWithTTL(uint64_t seconds, CleanUpFilter* filter) { } } - if (res.size() > 0) { - ENGINE_LOG_DEBUG << "Clean " << res.size() << " files deleted in " << seconds << " seconds"; + if (clean_files > 0) { + ENGINE_LOG_DEBUG << "Clean " << clean_files << " files deleted in " << seconds << " seconds"; } } // Scoped Connection } catch (std::exception& e) { diff --git a/core/src/db/meta/SqliteMetaImpl.cpp b/core/src/db/meta/SqliteMetaImpl.cpp index 2a3867574f..846655676d 100644 --- a/core/src/db/meta/SqliteMetaImpl.cpp +++ b/core/src/db/meta/SqliteMetaImpl.cpp @@ -1302,6 +1302,11 @@ SqliteMetaImpl::CleanUpFilesWithTTL(uint64_t seconds, CleanUpFilter* filter) { try { server::MetricCollector metric; + std::vector file_types = { + (int)TableFileSchema::TO_DELETE, + (int)TableFileSchema::BACKUP, + }; + // multi-threads call sqlite update may get exception('bad logic', etc), so we add a lock here std::lock_guard meta_lock(meta_mutex_); @@ -1309,21 +1314,23 @@ SqliteMetaImpl::CleanUpFilesWithTTL(uint64_t seconds, CleanUpFilter* filter) { auto files = ConnectorPtr->select(columns(&TableFileSchema::id_, &TableFileSchema::table_id_, &TableFileSchema::file_id_, + &TableFileSchema::file_type_, &TableFileSchema::date_), where( - c(&TableFileSchema::file_type_) == - (int)TableFileSchema::TO_DELETE + in(&TableFileSchema::file_type_, file_types) and c(&TableFileSchema::updated_time_) < now - seconds * US_PS)); + int64_t clean_files = 0; auto commited = ConnectorPtr->transaction([&]() mutable { TableFileSchema table_file; for (auto& file : files) { table_file.id_ = std::get<0>(file); table_file.table_id_ = std::get<1>(file); table_file.file_id_ = std::get<2>(file); - table_file.date_ = std::get<3>(file); + table_file.file_type_ = std::get<3>(file); + table_file.date_ = std::get<4>(file); // check if the file can be deleted if (filter && filter->IsIgnored(table_file)) { @@ -1332,17 +1339,23 @@ SqliteMetaImpl::CleanUpFilesWithTTL(uint64_t seconds, CleanUpFilter* filter) { continue; // ignore this file, don't delete it } - // delete file from meta - ConnectorPtr->remove(table_file.id_); - - // delete file from disk storage - utils::DeleteTableFilePath(options_, table_file); - - // erase from cache + // erase from cache, must do this before file deleted, + // because GetTableFilePath won't able to generate file path after the file is deleted + utils::GetTableFilePath(options_, table_file); server::CommonUtil::EraseFromCache(table_file.location_); - ENGINE_LOG_DEBUG << "Removing file id:" << table_file.file_id_ << " location:" << table_file.location_; - table_ids.insert(table_file.table_id_); + if (table_file.file_type_ == (int)TableFileSchema::TO_DELETE) { + // delete file from meta + ConnectorPtr->remove(table_file.id_); + + // delete file from disk storage + utils::DeleteTableFilePath(options_, table_file); + + ENGINE_LOG_DEBUG << "Removing file id:" << table_file.file_id_ << " location:" << table_file.location_; + table_ids.insert(table_file.table_id_); + } + + clean_files++; } return true; }); @@ -1351,8 +1364,8 @@ SqliteMetaImpl::CleanUpFilesWithTTL(uint64_t seconds, CleanUpFilter* filter) { return HandleException("CleanUpFilesWithTTL error: sqlite transaction failed"); } - if (files.size() > 0) { - ENGINE_LOG_DEBUG << "Clean " << files.size() << " files deleted in " << seconds << " seconds"; + if (clean_files > 0) { + ENGINE_LOG_DEBUG << "Clean " << clean_files << " files deleted in " << seconds << " seconds"; } } catch (std::exception& e) { return HandleException("Encounter exception when clean table files", e.what()); diff --git a/core/src/utils/CommonUtil.cpp b/core/src/utils/CommonUtil.cpp index cfadb2fcc4..cdfae8f1e5 100644 --- a/core/src/utils/CommonUtil.cpp +++ b/core/src/utils/CommonUtil.cpp @@ -229,7 +229,7 @@ CommonUtil::ConvertTime(tm time_struct, time_t& time_integer) { void CommonUtil::EraseFromCache(const std::string& item_key) { if (item_key.empty()) { - // SERVER_LOG_ERROR << "Empty key cannot be erased from cache"; + SERVER_LOG_ERROR << "Empty key cannot be erased from cache"; return; } From 66b698186c16f15015f3bcd6f3b1b33f26394252 Mon Sep 17 00:00:00 2001 From: groot Date: Sat, 30 Nov 2019 16:19:37 +0800 Subject: [PATCH 20/44] modify log --- core/src/db/DBImpl.cpp | 2 +- core/src/db/meta/MySQLMetaImpl.cpp | 7 +++---- core/src/db/meta/SqliteMetaImpl.cpp | 4 ++-- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/core/src/db/DBImpl.cpp b/core/src/db/DBImpl.cpp index c5e04fb5c6..b51ce571fd 100644 --- a/core/src/db/DBImpl.cpp +++ b/core/src/db/DBImpl.cpp @@ -799,7 +799,7 @@ DBImpl::BackgroundCompaction(std::set table_ids) { meta_ptr_->Archive(); { - uint64_t ttl = 1 * meta::SECOND; // default: file will be deleted after few seconds + uint64_t ttl = 10 * meta::SECOND; // default: file will be hard-deleted few seconds after soft-deleted if (options_.mode_ == DBOptions::MODE::CLUSTER_WRITABLE) { ttl = meta::H_SEC; } diff --git a/core/src/db/meta/MySQLMetaImpl.cpp b/core/src/db/meta/MySQLMetaImpl.cpp index 96e7109c93..401ef62bbd 100644 --- a/core/src/db/meta/MySQLMetaImpl.cpp +++ b/core/src/db/meta/MySQLMetaImpl.cpp @@ -1801,8 +1801,7 @@ MySQLMetaImpl::CleanUpFilesWithTTL(uint64_t seconds, CleanUpFilter* filter) { mysqlpp::Query query = connectionPtr->query(); query << "SELECT id, table_id, file_id, date" << " FROM " << META_TABLEFILES << " WHERE file_type IN (" - << std::to_string(TableFileSchema::TO_DELETE) << "," - << std::to_string(TableFileSchema::BACKUP) << ")" + << std::to_string(TableFileSchema::TO_DELETE) << "," << std::to_string(TableFileSchema::BACKUP) << ")" << " AND updated_time < " << std::to_string(now - seconds * US_PS) << ";"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUpFilesWithTTL: " << query.str(); @@ -1835,7 +1834,7 @@ MySQLMetaImpl::CleanUpFilesWithTTL(uint64_t seconds, CleanUpFilter* filter) { if (table_file.file_type_ == (int)TableFileSchema::TO_DELETE) { // delete file from disk storage utils::DeleteTableFilePath(options_, table_file); - ENGINE_LOG_DEBUG << "Removing file id:" << table_file.id_ << " location:" << table_file.location_; + ENGINE_LOG_DEBUG << "Remove file id:" << table_file.id_ << " location:" << table_file.location_; idsToDelete.emplace_back(std::to_string(table_file.id_)); table_ids.insert(table_file.table_id_); @@ -1863,7 +1862,7 @@ MySQLMetaImpl::CleanUpFilesWithTTL(uint64_t seconds, CleanUpFilter* filter) { } if (clean_files > 0) { - ENGINE_LOG_DEBUG << "Clean " << clean_files << " files deleted in " << seconds << " seconds"; + ENGINE_LOG_DEBUG << "Clean " << clean_files << " files expired in " << seconds << " seconds"; } } // Scoped Connection } catch (std::exception& e) { diff --git a/core/src/db/meta/SqliteMetaImpl.cpp b/core/src/db/meta/SqliteMetaImpl.cpp index 846655676d..c74212e6e7 100644 --- a/core/src/db/meta/SqliteMetaImpl.cpp +++ b/core/src/db/meta/SqliteMetaImpl.cpp @@ -1351,7 +1351,7 @@ SqliteMetaImpl::CleanUpFilesWithTTL(uint64_t seconds, CleanUpFilter* filter) { // delete file from disk storage utils::DeleteTableFilePath(options_, table_file); - ENGINE_LOG_DEBUG << "Removing file id:" << table_file.file_id_ << " location:" << table_file.location_; + ENGINE_LOG_DEBUG << "Remove file id:" << table_file.file_id_ << " location:" << table_file.location_; table_ids.insert(table_file.table_id_); } @@ -1365,7 +1365,7 @@ SqliteMetaImpl::CleanUpFilesWithTTL(uint64_t seconds, CleanUpFilter* filter) { } if (clean_files > 0) { - ENGINE_LOG_DEBUG << "Clean " << clean_files << " files deleted in " << seconds << " seconds"; + ENGINE_LOG_DEBUG << "Clean " << clean_files << " files expired in " << seconds << " seconds"; } } catch (std::exception& e) { return HandleException("Encounter exception when clean table files", e.what()); From 8ca2d52bff0c482fed854fa882339f50e659cb4c Mon Sep 17 00:00:00 2001 From: groot Date: Sat, 30 Nov 2019 17:02:40 +0800 Subject: [PATCH 21/44] fix mysql bug --- core/src/db/meta/MySQLMetaImpl.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/db/meta/MySQLMetaImpl.cpp b/core/src/db/meta/MySQLMetaImpl.cpp index 401ef62bbd..48bceca7d8 100644 --- a/core/src/db/meta/MySQLMetaImpl.cpp +++ b/core/src/db/meta/MySQLMetaImpl.cpp @@ -1799,7 +1799,7 @@ MySQLMetaImpl::CleanUpFilesWithTTL(uint64_t seconds, CleanUpFilter* filter) { } mysqlpp::Query query = connectionPtr->query(); - query << "SELECT id, table_id, file_id, date" + query << "SELECT id, table_id, file_id, file_type, date" << " FROM " << META_TABLEFILES << " WHERE file_type IN (" << std::to_string(TableFileSchema::TO_DELETE) << "," << std::to_string(TableFileSchema::BACKUP) << ")" << " AND updated_time < " << std::to_string(now - seconds * US_PS) << ";"; From 8f09b1f2306bd2833dd9ffd2d91efd174f3c9b0e Mon Sep 17 00:00:00 2001 From: "xiaojun.lin" Date: Sat, 30 Nov 2019 17:23:10 +0800 Subject: [PATCH 22/44] solve part of the problem about nsg --- CHANGELOG.md | 1 + .../index/vector_index/IndexGPUIDMAP.cpp | 34 +++ .../index/vector_index/IndexGPUIDMAP.h | 3 + .../index/vector_index/IndexIDMAP.cpp | 20 ++ .../knowhere/index/vector_index/IndexIDMAP.h | 14 ++ .../knowhere/index/vector_index/IndexIVF.cpp | 25 +-- .../knowhere/index/vector_index/IndexIVF.h | 2 +- .../knowhere/index/vector_index/IndexNSG.cpp | 18 +- .../knowhere/index/vector_index/nsg/NSG.cpp | 212 +++++++----------- .../knowhere/index/vector_index/nsg/NSG.h | 5 +- core/src/index/unittest/test_nsg/test_nsg.cpp | 172 ++++++++++++++ core/src/index/unittest/utils.cpp | 69 ++++++ core/src/index/unittest/utils.h | 9 + core/src/wrapper/ConfAdapter.cpp | 7 +- 14 files changed, 436 insertions(+), 155 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4b86ca0d74..12d679c646 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,6 +27,7 @@ Please mark all change in change log and use the ticket from JIRA. - \#527 - faiss benchmark not compatible with faiss 1.6.0 - \#530 - BuildIndex stop when do build index and search simultaneously - \#533 - NSG build failed with MetricType Inner Product +- \#548 - NSG search accuracy is too low ## Feature - \#12 - Pure CPU version for Milvus diff --git a/core/src/index/knowhere/knowhere/index/vector_index/IndexGPUIDMAP.cpp b/core/src/index/knowhere/knowhere/index/vector_index/IndexGPUIDMAP.cpp index 1aded3ddaa..edf42abc8d 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/IndexGPUIDMAP.cpp +++ b/core/src/index/knowhere/knowhere/index/vector_index/IndexGPUIDMAP.cpp @@ -126,4 +126,38 @@ GPUIDMAP::search_impl(int64_t n, const float* data, int64_t k, float* distances, index_->search(n, (float*)data, k, distances, labels); } +void +GPUIDMAP::GenGraph(float* data, const int64_t& k, Graph& graph, const Config& config) { + int64_t K = k + 1; + auto ntotal = Count(); + + size_t dim = config->d; + auto batch_size = 1000; + auto tail_batch_size = ntotal % batch_size; + auto batch_search_count = ntotal / batch_size; + auto total_search_count = tail_batch_size == 0 ? batch_search_count : batch_search_count + 1; + + std::vector res_dis(K * batch_size); + graph.resize(ntotal); + Graph res_vec(total_search_count); + for (int i = 0; i < total_search_count; ++i) { + auto b_size = (i == (total_search_count - 1)) && tail_batch_size != 0 ? tail_batch_size : batch_size; + + auto& res = res_vec[i]; + res.resize(K * b_size); + + auto xq = data + batch_size * dim * i; + search_impl(b_size, (float*)xq, K, res_dis.data(), res.data(), config); + + for (int j = 0; j < b_size; ++j) { + auto& node = graph[batch_size * i + j]; + node.resize(k); + auto start_pos = j * K + 1; + for (int m = 0, cursor = start_pos; m < k && cursor < start_pos + k; ++m, ++cursor) { + node[m] = res[cursor]; + } + } + } +} + } // namespace knowhere diff --git a/core/src/index/knowhere/knowhere/index/vector_index/IndexGPUIDMAP.h b/core/src/index/knowhere/knowhere/index/vector_index/IndexGPUIDMAP.h index f1dfe2f21a..b9325a9cc1 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/IndexGPUIDMAP.h +++ b/core/src/index/knowhere/knowhere/index/vector_index/IndexGPUIDMAP.h @@ -47,6 +47,9 @@ class GPUIDMAP : public IDMAP, public GPUIndex { VectorIndexPtr CopyGpuToGpu(const int64_t& device_id, const Config& config) override; + void + GenGraph(float* data, const int64_t& k, Graph& graph, const Config& config); + protected: void search_impl(int64_t n, const float* data, int64_t k, float* distances, int64_t* labels, const Config& cfg) override; diff --git a/core/src/index/knowhere/knowhere/index/vector_index/IndexIDMAP.cpp b/core/src/index/knowhere/knowhere/index/vector_index/IndexIDMAP.cpp index 7aedf98613..351209c10f 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/IndexIDMAP.cpp +++ b/core/src/index/knowhere/knowhere/index/vector_index/IndexIDMAP.cpp @@ -121,6 +121,26 @@ IDMAP::Add(const DatasetPtr& dataset, const Config& config) { index_->add_with_ids(rows, (float*)p_data, p_ids); } +void +IDMAP::AddWithoutId(const DatasetPtr& dataset, const Config& config) { + if (!index_) { + KNOWHERE_THROW_MSG("index not initialize"); + } + + std::lock_guard lk(mutex_); + GETTENSOR(dataset) + + // TODO: magic here. + auto array = dataset->array()[0]; + + std::vector new_ids(rows); + for (int i = 0; i < rows; ++i) { + new_ids[i] = i; + } + + index_->add_with_ids(rows, (float*)p_data, new_ids.data()); +} + int64_t IDMAP::Count() { return index_->ntotal; diff --git a/core/src/index/knowhere/knowhere/index/vector_index/IndexIDMAP.h b/core/src/index/knowhere/knowhere/index/vector_index/IndexIDMAP.h index 0f66e8fac0..9f1369c7d3 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/IndexIDMAP.h +++ b/core/src/index/knowhere/knowhere/index/vector_index/IndexIDMAP.h @@ -34,20 +34,31 @@ class IDMAP : public VectorIndex, public FaissBaseIndex { BinarySet Serialize() override; + void Load(const BinarySet& index_binary) override; + void Train(const Config& config); + DatasetPtr Search(const DatasetPtr& dataset, const Config& config) override; + int64_t Count() override; + VectorIndexPtr Clone() override; + int64_t Dimension() override; + void Add(const DatasetPtr& dataset, const Config& config) override; + + void + AddWithoutId(const DatasetPtr& dataset, const Config& config); + VectorIndexPtr CopyCpuToGpu(const int64_t& device_id, const Config& config); void @@ -55,12 +66,15 @@ class IDMAP : public VectorIndex, public FaissBaseIndex { virtual float* GetRawVectors(); + virtual int64_t* GetRawIds(); protected: virtual void search_impl(int64_t n, const float* data, int64_t k, float* distances, int64_t* labels, const Config& cfg); + + protected: std::mutex mutex_; }; diff --git a/core/src/index/knowhere/knowhere/index/vector_index/IndexIVF.cpp b/core/src/index/knowhere/knowhere/index/vector_index/IndexIVF.cpp index 8b734abdc6..a74bddc94c 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/IndexIVF.cpp +++ b/core/src/index/knowhere/knowhere/index/vector_index/IndexIVF.cpp @@ -189,35 +189,34 @@ IVF::Dimension() { } void -IVF::GenGraph(const int64_t& k, Graph& graph, const DatasetPtr& dataset, const Config& config) { - GETTENSOR(dataset) - +IVF::GenGraph(float* data, const int64_t& k, Graph& graph, const Config& config) { + int64_t K = k + 1; auto ntotal = Count(); - auto batch_size = 100; + size_t dim = config->d; + auto batch_size = 1000; auto tail_batch_size = ntotal % batch_size; auto batch_search_count = ntotal / batch_size; auto total_search_count = tail_batch_size == 0 ? batch_search_count : batch_search_count + 1; - std::vector res_dis(k * batch_size); + std::vector res_dis(K * batch_size); graph.resize(ntotal); Graph res_vec(total_search_count); for (int i = 0; i < total_search_count; ++i) { - auto b_size = i == total_search_count - 1 && tail_batch_size != 0 ? tail_batch_size : batch_size; + auto b_size = (i == (total_search_count - 1)) && tail_batch_size != 0 ? tail_batch_size : batch_size; auto& res = res_vec[i]; - res.resize(k * b_size); + res.resize(K * b_size); - auto xq = p_data + batch_size * dim * i; - search_impl(b_size, (float*)xq, k, res_dis.data(), res.data(), config); + auto xq = data + batch_size * dim * i; + search_impl(b_size, (float*)xq, K, res_dis.data(), res.data(), config); - int tmp = 0; for (int j = 0; j < b_size; ++j) { auto& node = graph[batch_size * i + j]; node.resize(k); - for (int m = 0; m < k && tmp < k * b_size; ++m, ++tmp) { - // TODO(linxj): avoid memcopy here. - node[m] = res[tmp]; + auto start_pos = j * K + 1; + for (int m = 0, cursor = start_pos; m < k && cursor < start_pos + k; ++m, ++cursor) { + node[m] = res[cursor]; } } } diff --git a/core/src/index/knowhere/knowhere/index/vector_index/IndexIVF.h b/core/src/index/knowhere/knowhere/index/vector_index/IndexIVF.h index e064b6f08c..24b006a565 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/IndexIVF.h +++ b/core/src/index/knowhere/knowhere/index/vector_index/IndexIVF.h @@ -57,7 +57,7 @@ class IVF : public VectorIndex, public FaissBaseIndex { Search(const DatasetPtr& dataset, const Config& config) override; void - GenGraph(const int64_t& k, Graph& graph, const DatasetPtr& dataset, const Config& config); + GenGraph(float* data, const int64_t& k, Graph& graph, const Config& config); BinarySet Serialize() override; diff --git a/core/src/index/knowhere/knowhere/index/vector_index/IndexNSG.cpp b/core/src/index/knowhere/knowhere/index/vector_index/IndexNSG.cpp index 3cf0122233..9571571945 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/IndexNSG.cpp +++ b/core/src/index/knowhere/knowhere/index/vector_index/IndexNSG.cpp @@ -21,6 +21,8 @@ #include "knowhere/common/Timer.h" #ifdef MILVUS_GPU_VERSION #include "knowhere/index/vector_index/IndexGPUIVF.h" +#include "knowhere/index/vector_index/IndexGPUIDMAP.h" +#include "knowhere/index/vector_index/helpers/Cloner.h" #endif #include "knowhere/index/vector_index/IndexIVF.h" @@ -110,6 +112,7 @@ NSG::Search(const DatasetPtr& dataset, const Config& config) { IndexModelPtr NSG::Train(const DatasetPtr& dataset, const Config& config) { + config->Dump(); auto build_cfg = std::dynamic_pointer_cast(config); if (build_cfg != nullptr) { build_cfg->CheckValid(); // throw exception @@ -117,23 +120,26 @@ NSG::Train(const DatasetPtr& dataset, const Config& config) { // TODO(linxj): dev IndexFactory, support more IndexType #ifdef MILVUS_GPU_VERSION - auto preprocess_index = std::make_shared(build_cfg->gpu_id); +// auto preprocess_index = std::make_shared(build_cfg->gpu_id); #else auto preprocess_index = std::make_shared(); #endif - auto model = preprocess_index->Train(dataset, config); - preprocess_index->set_index_model(model); - preprocess_index->AddWithoutIds(dataset, config); + auto preprocess_index = std::make_shared(); + preprocess_index->Train(config); + preprocess_index->AddWithoutId(dataset, config); + float* raw_data = preprocess_index->GetRawVectors(); + auto xx = cloner::CopyCpuToGpu(preprocess_index, 0, config); + auto ss = std::dynamic_pointer_cast(xx); Graph knng; - preprocess_index->GenGraph(build_cfg->knng, knng, dataset, config); + ss->GenGraph(raw_data, build_cfg->knng, knng, config); + GETTENSOR(dataset) algo::BuildParams b_params; b_params.candidate_pool_size = build_cfg->candidate_pool_size; b_params.out_degree = build_cfg->out_degree; b_params.search_length = build_cfg->search_length; - GETTENSOR(dataset) auto array = dataset->array()[0]; auto p_ids = array->data()->GetValues(1, 0); diff --git a/core/src/index/knowhere/knowhere/index/vector_index/nsg/NSG.cpp b/core/src/index/knowhere/knowhere/index/vector_index/nsg/NSG.cpp index e9e65b1191..f303543259 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/nsg/NSG.cpp +++ b/core/src/index/knowhere/knowhere/index/vector_index/nsg/NSG.cpp @@ -18,7 +18,6 @@ #include #include #include -#include #include #include #include @@ -29,12 +28,13 @@ #include "knowhere/index/vector_index/nsg/NSG.h" #include "knowhere/index/vector_index/nsg/NSGHelper.h" -// TODO: enable macro //#include namespace knowhere { namespace algo { +unsigned int seed = 100; + NsgIndex::NsgIndex(const size_t& dimension, const size_t& n, METRICTYPE metric) : dimension(dimension), ntotal(n), metric_type(metric) { switch (metric) { @@ -55,8 +55,6 @@ NsgIndex::~NsgIndex() { void NsgIndex::Build_with_ids(size_t nb, const float* data, const int64_t* ids, const BuildParams& parameters) { - TimeRecorder rc("NSG"); - ntotal = nb; ori_data_ = new float[ntotal * dimension]; ids_ = new int64_t[ntotal]; @@ -67,25 +65,17 @@ NsgIndex::Build_with_ids(size_t nb, const float* data, const int64_t* ids, const out_degree = parameters.out_degree; candidate_pool_size = parameters.candidate_pool_size; + TimeRecorder rc("NSG", 1); + InitNavigationPoint(); rc.RecordSection("init"); Link(); rc.RecordSection("Link"); - //>> Debug code - ///// - // int count = 0; - // for (int i = 0; i < ntotal; ++i) { - // count += nsg[i].size(); - //} - ///// - CheckConnectivity(); rc.RecordSection("Connect"); - //>> Debug code - /// int total_degree = 0; for (size_t i = 0; i < ntotal; ++i) { total_degree += nsg[i].size(); @@ -93,9 +83,17 @@ NsgIndex::Build_with_ids(size_t nb, const float* data, const int64_t* ids, const KNOWHERE_LOG_DEBUG << "Graph physical size: " << total_degree * sizeof(node_t) / 1024 / 1024 << "m"; KNOWHERE_LOG_DEBUG << "Average degree: " << total_degree / ntotal; - ///// is_trained = true; + + // Debug code + // for (size_t i = 0; i < ntotal; i++) { + // auto& x = nsg[i]; + // for (size_t j = 0; j < x.size(); j++) { + // std::cout << "id: " << x[j] << std::endl; + // } + // std::cout << std::endl; + // } } void @@ -114,28 +112,22 @@ NsgIndex::InitNavigationPoint() { } // select navigation point - std::vector resset, fullset; - unsigned int seed = 100; + std::vector resset; navigation_point = rand_r(&seed) % ntotal; // random initialize navigating point - - //>> Debug code - ///// - // navigation_point = drand48(); - ///// - GetNeighbors(center, resset, knng); navigation_point = resset[0].id; - //>> Debug code - ///// + // Debug code // std::cout << "ep: " << navigation_point << std::endl; - ///// - - //>> Debug code - ///// + // for (int k = 0; k < resset.size(); ++k) { + // std::cout << "id: " << resset[k].id << ", dis: " << resset[k].distance << std::endl; + // } + // std::cout << std::endl; + // + // std::cout << "ep: " << navigation_point << std::endl; + // // float r1 = distance_->Compare(center, ori_data_ + navigation_point * dimension, dimension); // assert(r1 == resset[0].distance); - ///// } // Specify Link @@ -149,7 +141,9 @@ NsgIndex::GetNeighbors(const float* query, std::vector& resset, std::v // TODO: throw exception here. } - std::vector init_ids; + resset.resize(search_length); + std::vector init_ids(buffer_size); + // std::vector init_ids; { /* @@ -158,25 +152,26 @@ NsgIndex::GetNeighbors(const float* query, std::vector& resset, std::v size_t count = 0; // Get all neighbors - for (size_t i = 0; i < graph[navigation_point].size(); ++i) { - init_ids.push_back(graph[navigation_point][i]); + for (size_t i = 0; i < init_ids.size() && i < graph[navigation_point].size(); ++i) { + // for (size_t i = 0; i < graph[navigation_point].size(); ++i) { + // init_ids.push_back(graph[navigation_point][i]); + init_ids[i] = graph[navigation_point][i]; has_calculated_dist[init_ids[i]] = true; ++count; } - - unsigned int seed = 100; while (count < buffer_size) { node_t id = rand_r(&seed) % ntotal; if (has_calculated_dist[id]) continue; // duplicate id - init_ids.push_back(id); + // init_ids.push_back(id); + init_ids[count] = id; ++count; has_calculated_dist[id] = true; } } { - resset.resize(init_ids.size()); + // resset.resize(init_ids.size()); // init resset and sort by distance for (size_t i = 0; i < init_ids.size(); ++i) { @@ -190,7 +185,7 @@ NsgIndex::GetNeighbors(const float* query, std::vector& resset, std::v float dist = distance_->Compare(ori_data_ + dimension * id, query, dimension); resset[i] = Neighbor(id, dist, false); - ///////////// difference from other GetNeighbors /////////////// + //// difference from other GetNeighbors fullset.push_back(resset[i]); /////////////////////////////////////// } @@ -247,8 +242,10 @@ NsgIndex::GetNeighbors(const float* query, std::vector& resset, std::v // TODO: throw exception here. } - std::vector init_ids; - boost::dynamic_bitset<> has_calculated_dist{ntotal, 0}; // TODO: ? + // std::vector init_ids; + std::vector init_ids(buffer_size); + resset.resize(buffer_size); + boost::dynamic_bitset<> has_calculated_dist{ntotal, 0}; { /* @@ -257,24 +254,26 @@ NsgIndex::GetNeighbors(const float* query, std::vector& resset, std::v size_t count = 0; // Get all neighbors - for (size_t i = 0; i < graph[navigation_point].size(); ++i) { - init_ids.push_back(graph[navigation_point][i]); + for (size_t i = 0; i < init_ids.size() && i < graph[navigation_point].size(); ++i) { + // for (size_t i = 0; i < graph[navigation_point].size(); ++i) { + // init_ids.push_back(graph[navigation_point][i]); + init_ids[i] = graph[navigation_point][i]; has_calculated_dist[init_ids[i]] = true; ++count; } - unsigned int seed = 100; while (count < buffer_size) { node_t id = rand_r(&seed) % ntotal; if (has_calculated_dist[id]) continue; // duplicate id - init_ids.push_back(id); + // init_ids.push_back(id); + init_ids[count] = id; ++count; has_calculated_dist[id] = true; } } { - resset.resize(init_ids.size()); + // resset.resize(init_ids.size()); // init resset and sort by distance for (size_t i = 0; i < init_ids.size(); ++i) { @@ -333,13 +332,15 @@ NsgIndex::GetNeighbors(const float* query, std::vector& resset, std::v void NsgIndex::GetNeighbors(const float* query, std::vector& resset, Graph& graph, SearchParams* params) { - size_t& buffer_size = params ? params->search_length : search_length; + size_t buffer_size = params ? params->search_length : search_length; if (buffer_size > ntotal) { // TODO: throw exception here. } - std::vector init_ids; + // std::vector init_ids; + std::vector init_ids(buffer_size); + resset.resize(buffer_size); boost::dynamic_bitset<> has_calculated_dist{ntotal, 0}; { @@ -349,33 +350,33 @@ NsgIndex::GetNeighbors(const float* query, std::vector& resset, Graph& size_t count = 0; // Get all neighbors - for (size_t i = 0; i < graph[navigation_point].size(); ++i) { - init_ids.push_back(graph[navigation_point][i]); + for (size_t i = 0; i < init_ids.size() && i < graph[navigation_point].size(); ++i) { + // for (size_t i = 0; i < graph[navigation_point].size(); ++i) { + // init_ids.push_back(graph[navigation_point][i]); + init_ids[i] = graph[navigation_point][i]; has_calculated_dist[init_ids[i]] = true; ++count; } - unsigned int seed = 100; while (count < buffer_size) { node_t id = rand_r(&seed) % ntotal; if (has_calculated_dist[id]) continue; // duplicate id - init_ids.push_back(id); + // init_ids.push_back(id); + init_ids[count] = id; ++count; has_calculated_dist[id] = true; } } { - resset.resize(init_ids.size()); + // resset.resize(init_ids.size()); // init resset and sort by distance for (size_t i = 0; i < init_ids.size(); ++i) { node_t id = init_ids[i]; - // assert(id < ntotal); if (id >= static_cast(ntotal)) { KNOWHERE_THROW_MSG("Build Index Error, id > ntotal"); - continue; } float dist = distance_->Compare(ori_data_ + id * dimension, query, dimension); @@ -383,13 +384,6 @@ NsgIndex::GetNeighbors(const float* query, std::vector& resset, Graph& } std::sort(resset.begin(), resset.end()); // sort by distance - //>> Debug code - ///// - // for (int j = 0; j < buffer_size; ++j) { - // std::cout << "resset_id: " << resset[j].id << ", resset_dist: " << resset[j].distance << std::endl; - //} - ///// - // search nearest neighbor size_t cursor = 0; while (cursor < buffer_size) { @@ -410,7 +404,8 @@ NsgIndex::GetNeighbors(const float* query, std::vector& resset, Graph& if (dist >= resset[buffer_size - 1].distance) continue; - ///////////// difference from other GetNeighbors /////////////// + + //// difference from other GetNeighbors Neighbor nn(id, dist, false); /////////////////////////////////////// @@ -440,59 +435,50 @@ NsgIndex::GetNeighbors(const float* query, std::vector& resset, Graph& void NsgIndex::Link() { - auto cut_graph_dist = new float[ntotal * out_degree]; + float* cut_graph_dist = new float[ntotal * out_degree]; nsg.resize(ntotal); #pragma omp parallel { std::vector fullset; std::vector temp; - boost::dynamic_bitset<> flags{ntotal, 0}; // TODO: ? + boost::dynamic_bitset<> flags{ntotal, 0}; #pragma omp for schedule(dynamic, 100) for (size_t n = 0; n < ntotal; ++n) { fullset.clear(); + temp.clear(); flags.reset(); GetNeighbors(ori_data_ + dimension * n, temp, fullset, flags); - - //>> Debug code - ///// - // float r1 = distance_->Compare(ori_data_ + n * dimension, ori_data_ + temp[0].id * dimension, dimension); - // assert(r1 == temp[0].distance); - ///// SyncPrune(n, fullset, flags, cut_graph_dist); } + + // Debug code + // std::cout << "ep: " << 0 << std::endl; + // for (int k = 0; k < fullset.size(); ++k) { + // std::cout << "id: " << fullset[k].id << ", dis: " << fullset[k].distance << std::endl; + // } } - //>> Debug code - ///// - // auto bak_nsg = nsg; - ///// + // Debug code + // for (size_t i = 0; i < ntotal; i++) + // { + // auto& x = nsg[i]; + // for (size_t j=0; j < x.size(); j++) + // { + // std::cout << "id: " << x[j] << std::endl; + // } + // std::cout << std::endl; + // } knng.clear(); - knng.shrink_to_fit(); std::vector mutex_vec(ntotal); - #pragma omp for schedule(dynamic, 100) for (unsigned n = 0; n < ntotal; ++n) { InterInsert(n, mutex_vec, cut_graph_dist); } + delete[] cut_graph_dist; - - //>> Debug code - ///// - // int count = 0; - // for (int i = 0; i < ntotal; ++i) { - // if (bak_nsg[i].size() != nsg[i].size()) { - // //count += nsg[i].size() - bak_nsg[i].size(); - // count += nsg[i].size(); - // } - //} - ///// - - for (size_t i = 0; i < ntotal; ++i) { - nsg[i].shrink_to_fit(); - } } void @@ -654,9 +640,9 @@ NsgIndex::DFS(size_t root, boost::dynamic_bitset<>& has_linked, int64_t& linked_ std::stack s; s.push(root); if (!has_linked[root]) { - linked_count++; // not link - has_linked[root] = true; // link start... + linked_count++; // not link } + has_linked[root] = true; // link start... while (!s.empty()) { size_t next = ntotal + 1; @@ -709,7 +695,6 @@ NsgIndex::FindUnconnectedNode(boost::dynamic_bitset<>& has_linked, int64_t& root } } if (found == 0) { - unsigned int seed = 100; while (true) { // random a linked-node and add unlinked-node as its neighbor size_t rid = rand_r(&seed) % ntotal; if (has_linked[rid]) { @@ -726,7 +711,10 @@ NsgIndex::Search(const float* query, const unsigned& nq, const unsigned& dim, co int64_t* ids, SearchParams& params) { std::vector> resset(nq); - params.search_length = k; + if (k >= 45) { + params.search_length = k; + } + TimeRecorder rc("NsgIndex::search", 1); // TODO(linxj): when to use openmp if (nq <= 4) { @@ -734,7 +722,7 @@ NsgIndex::Search(const float* query, const unsigned& nq, const unsigned& dim, co } else { #pragma omp parallel for for (unsigned int i = 0; i < nq; ++i) { - auto single_query = query + i * dim; + const float* single_query = query + i * dim; GetNeighbors(single_query, resset[i], nsg, ¶ms); } } @@ -759,13 +747,6 @@ NsgIndex::Search(const float* query, const unsigned& nq, const unsigned& dim, co } rc.RecordSection("merge"); - //>> Debug: test single insert - // int x_0 = resset[0].size(); - // for (int l = 0; l < resset[0].size(); ++l) { - // resset[0].pop_back(); - //} - // resset.clear(); - // ProfilerStart("xx.prof"); // std::vector resset; // GetNeighbors(query, resset, nsg, ¶ms); @@ -781,30 +762,5 @@ NsgIndex::SetKnnGraph(Graph& g) { knng = std::move(g); } -// void NsgIndex::GetKnnGraphFromFile() { -// //std::string filename = "sift.1M.50NN.graph"; -// std::string filename = "sift.50NN.graph"; -// -// std::ifstream in(filename, std::ios::binary); -// unsigned k; -// in.read((char *) &k, sizeof(unsigned)); -// in.seekg(0, std::ios::end); -// std::ios::pos_type ss = in.tellg(); -// size_t fsize = (size_t) ss; -// size_t num = (unsigned) (fsize / (k + 1) / 4); -// in.seekg(0, std::ios::beg); -// -// knng.resize(num); -// knng.reserve(num); -// unsigned kk = (k + 3) / 4 * 4; -// for (size_t i = 0; i < num; i++) { -// in.seekg(4, std::ios::cur); -// knng[i].resize(k); -// knng[i].reserve(kk); -// in.read((char *) knng[i].data(), k * sizeof(unsigned)); -// } -// in.close(); -//} - } // namespace algo } // namespace knowhere diff --git a/core/src/index/knowhere/knowhere/index/vector_index/nsg/NSG.h b/core/src/index/knowhere/knowhere/index/vector_index/nsg/NSG.h index 5dd128610f..3dd827466a 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/nsg/NSG.h +++ b/core/src/index/knowhere/knowhere/index/vector_index/nsg/NSG.h @@ -52,7 +52,7 @@ class NsgIndex { Distance* distance_; float* ori_data_; - int64_t* ids_; // TODO: support different type + int64_t* ids_; Graph nsg; // final graph Graph knng; // reset after build @@ -134,9 +134,6 @@ class NsgIndex { void FindUnconnectedNode(boost::dynamic_bitset<>& flags, int64_t& root); - - // private: - // void GetKnnGraphFromFile(); }; } // namespace algo diff --git a/core/src/index/unittest/test_nsg/test_nsg.cpp b/core/src/index/unittest/test_nsg/test_nsg.cpp index 4722c7e8f6..a5eac12b2a 100644 --- a/core/src/index/unittest/test_nsg/test_nsg.cpp +++ b/core/src/index/unittest/test_nsg/test_nsg.cpp @@ -23,6 +23,8 @@ #include "knowhere/index/vector_index/IndexNSG.h" #ifdef MILVUS_GPU_VERSION #include "knowhere/index/vector_index/helpers/FaissGpuResourceMgr.h" +#include "knowhere/index/vector_index/IndexGPUIDMAP.h" +#include "knowhere/index/vector_index/helpers/Cloner.h" #endif #include "knowhere/common/Timer.h" @@ -113,3 +115,173 @@ TEST_F(NSGInterfaceTest, comparetest) { } tc.RecordSection("IP"); } + +//#include +//TEST(test, ori_nsg) { +// // float* p_data = nullptr; +// size_t rows, dim; +// char* filename = "/mnt/112d53a6-5592-4360-a33b-7fd789456fce/workspace/Data/sift/sift_base.fvecs"; +// // loads_data(filename, p_data, rows, dim); +// float* p_data = fvecs_read(filename, &dim, &rows); +// +// std::string knng_filename = +// "/mnt/112d53a6-5592-4360-a33b-7fd789456fce/workspace/Cellar/anns/efanna_graph/tests/sift.1M.50NN.graph"; +// std::vector> knng; +// Load_nns_graph(knng, knng_filename.c_str()); +// +// // float* search_data = nullptr; +// size_t nq, search_dim; +// char* searchfile = "/mnt/112d53a6-5592-4360-a33b-7fd789456fce/workspace/Data/sift/sift_query.fvecs"; +// // loads_data(searchfile, search_data, nq, search_dim); +// float* search_data = fvecs_read(searchfile, &search_dim, &nq); +// assert(search_dim == dim); +// +// size_t k, nq2; +// char* gtfile = "/mnt/112d53a6-5592-4360-a33b-7fd789456fce/workspace/Data/sift/sift_groundtruth.ivecs"; +// int* gt_int = ivecs_read(gtfile, &k, &nq2); +// int64_t* gt = new int64_t[k * nq2]; +// for (int i = 0; i < k * nq2; i++) { +// gt[i] = gt_int[i]; +// } +// delete[] gt_int; +// +// std::vector store_ids(rows); +// for (int i = 0; i < rows; ++i) { +// store_ids[i] = i; +// } +// +// int64_t* I = new int64_t[nq * k]; +// float* D = new float[nq * k]; +//#if 0 +// efanna2e::Parameters params; +// params.Set("L", 50); +// params.Set("R", 55); +// params.Set("C", 300); +// auto orinsg = std::make_shared(dim, rows, efanna2e::Metric::L2, nullptr); +// orinsg->Load_nn_graph(knng); +// orinsg->Build(rows, (float*)p_data, params); +// +// efanna2e::Parameters paras; +// paras.Set("L_search", 45); +// paras.Set("P_search",100); +// k = 10; +// std::vector > res; +// for (unsigned i = 0; i < nq; i++) { +// std::vector tmp(k); +// orinsg->Search(search_data + i * dim, p_data, k, paras, tmp.data()); +// res.push_back(tmp); +// } +// } +//#else +// knowhere::algo::BuildParams params; +// params.search_length = 50; +// params.out_degree = 55; +// params.candidate_pool_size = 300; +// auto nsg = std::make_shared(dim, rows); +//#if 1 +// knowhere::FaissGpuResourceMgr::GetInstance().InitDevice(DEVICEID, 1024 * 1024 * 200, 1024 * 1024 * 600, 2); +// auto dataset = generate_dataset(int64_t(rows), int64_t(dim), p_data, store_ids.data()); +// auto config = std::make_shared(); +// config->d = dim; +// config->gpu_id = 0; +// config->metric_type = knowhere::METRICTYPE::L2; +// auto preprocess_index = std::make_shared(); +// preprocess_index->Train(config); +// preprocess_index->AddWithoutId(dataset, config); +// auto xx = knowhere::cloner::CopyCpuToGpu(preprocess_index, 0, config); +// auto ss = std::dynamic_pointer_cast(xx); +// +// std::vector> kng; +// ss->GenGraph(p_data, 50, kng, config); +// nsg->SetKnnGraph(kng); +// knowhere::FaissGpuResourceMgr::GetInstance().Free(); +//#else +// nsg->SetKnnGraph(knng); +//#endif +// nsg->Build_with_ids(rows, (float*)p_data, store_ids.data(), params); +// knowhere::algo::SearchParams s_params; +// s_params.search_length = 45; +// nsg->Search(search_data, nq, dim, k, D, I, s_params); +//#endif +// +// int n_1 = 0, n_10 = 0, n_100 = 0; +// for (int i = 0; i < nq; i++) { +// int gt_nn = gt[i * k]; +// for (int j = 0; j < k; j++) { +// if (I[i * k + j] == gt_nn) { +// if (j < 1) +// n_1++; +// if (j < 10) +// n_10++; +// if (j < 100) +// n_100++; +// } +// } +// } +// printf("R@1 = %.4f\n", n_1 / float(nq)); +// printf("R@10 = %.4f\n", n_10 / float(nq)); +// printf("R@100 = %.4f\n", n_100 / float(nq)); +//} +// +//TEST(testxx, test_idmap){ +// int k = 50; +// std::string knng_filename = +// "/mnt/112d53a6-5592-4360-a33b-7fd789456fce/workspace/Cellar/anns/efanna_graph/tests/sift.50NN.graph"; +// std::vector> gt_knng; +// Load_nns_graph(gt_knng, knng_filename.c_str()); +// +// size_t rows, dim; +// char* filename = "/mnt/112d53a6-5592-4360-a33b-7fd789456fce/workspace/Cellar/anns/efanna_graph/tests/siftsmall/siftsmall_base.fvecs"; +// float* p_data = fvecs_read(filename, &dim, &rows); +// +// std::vector store_ids(rows); +// for (int i = 0; i < rows; ++i) { +// store_ids[i] = i; +// } +// +// knowhere::FaissGpuResourceMgr::GetInstance().InitDevice(DEVICEID, 1024 * 1024 * 200, 1024 * 1024 * 600, 2); +// auto dataset = generate_dataset(int64_t(rows), int64_t(dim), p_data, store_ids.data()); +// auto config = std::make_shared(); +// config->d = dim; +// config->gpu_id = 0; +// config->metric_type = knowhere::METRICTYPE::L2; +// auto preprocess_index = std::make_shared(); +// preprocess_index->Train(config); +// preprocess_index->AddWithoutId(dataset, config); +// auto xx = knowhere::cloner::CopyCpuToGpu(preprocess_index, 0, config); +// auto ss = std::dynamic_pointer_cast(xx); +// std::vector> idmap_knng; +// ss->GenGraph(p_data, k, idmap_knng,config); +// knowhere::FaissGpuResourceMgr::GetInstance().Free(); +// +// int n_1 = 0, n_10 = 0, n_100 = 0; +// for (int i = 0; i < rows; i++) { +// int gt_nn = gt_knng[i][0]; +// int l_n_1 = 0; +// int l_n_10 = 0; +// int l_n_100 = 0; +// for (int j = 0; j < k; j++) { +// if (idmap_knng[i][j] == gt_nn) { +// if (j < 1){ +// n_1++; +// l_n_1++; +// } +// if (j < 10){ +// n_10++; +// l_n_10++; +// } +// if (j < 100){ +// n_100++; +// l_n_100++; +// } +// +// } +// if ((j == k-1) && (l_n_100 == 0)){ +// std::cout << "error id: " << i << std::endl; +// } +// } +// } +// printf("R@1 = %.4f\n", n_1 / float(rows)); +// printf("R@10 = %.4f\n", n_10 / float(rows)); +// printf("R@100 = %.4f\n", n_100 / float(rows)); +//} diff --git a/core/src/index/unittest/utils.cpp b/core/src/index/unittest/utils.cpp index 11dad4a8b9..a2ff6fd829 100644 --- a/core/src/index/unittest/utils.cpp +++ b/core/src/index/unittest/utils.cpp @@ -178,3 +178,72 @@ PrintResult(const knowhere::DatasetPtr& result, const int& nq, const int& k) { std::cout << "id\n" << ss_id.str() << std::endl; std::cout << "dist\n" << ss_dist.str() << std::endl; } + +void +Load_nns_graph(std::vector>& final_graph, const char* filename) { + std::vector> knng; + + std::ifstream in(filename, std::ios::binary); + unsigned k; + in.read((char*)&k, sizeof(unsigned)); + in.seekg(0, std::ios::end); + std::ios::pos_type ss = in.tellg(); + size_t fsize = (size_t)ss; + size_t num = (size_t)(fsize / (k + 1) / 4); + in.seekg(0, std::ios::beg); + + knng.resize(num); + knng.reserve(num); + int64_t kk = (k + 3) / 4 * 4; + for (size_t i = 0; i < num; i++) { + in.seekg(4, std::ios::cur); + knng[i].resize(k); + knng[i].reserve(kk); + in.read((char*)knng[i].data(), k * sizeof(unsigned)); + } + in.close(); + + final_graph.resize(knng.size()); + for (int i = 0; i < knng.size(); ++i) { + final_graph[i].resize(knng[i].size()); + for (int j = 0; j < knng[i].size(); ++j) { + final_graph[i][j] = knng[i][j]; + } + } +} + +float* +fvecs_read(const char* fname, size_t* d_out, size_t* n_out) { + FILE* f = fopen(fname, "r"); + if (!f) { + fprintf(stderr, "could not open %s\n", fname); + perror(""); + abort(); + } + int d; + fread(&d, 1, sizeof(int), f); + assert((d > 0 && d < 1000000) || !"unreasonable dimension"); + fseek(f, 0, SEEK_SET); + struct stat st; + fstat(fileno(f), &st); + size_t sz = st.st_size; + assert(sz % ((d + 1) * 4) == 0 || !"weird file size"); + size_t n = sz / ((d + 1) * 4); + + *d_out = d; + *n_out = n; + float* x = new float[n * (d + 1)]; + size_t nr = fread(x, sizeof(float), n * (d + 1), f); + assert(nr == n * (d + 1) || !"could not read whole file"); + + // shift array to remove row headers + for (size_t i = 0; i < n; i++) memmove(x + i * d, x + 1 + i * (d + 1), d * sizeof(*x)); + + fclose(f); + return x; +} + +int* // not very clean, but works as long as sizeof(int) == sizeof(float) +ivecs_read(const char* fname, size_t* d_out, size_t* n_out) { + return (int*)fvecs_read(fname, d_out, n_out); +} diff --git a/core/src/index/unittest/utils.h b/core/src/index/unittest/utils.h index b39cf9ea14..f11ad28163 100644 --- a/core/src/index/unittest/utils.h +++ b/core/src/index/unittest/utils.h @@ -93,3 +93,12 @@ struct FileIOReader { size_t operator()(void* ptr, size_t size); }; + +void +Load_nns_graph(std::vector>& final_graph_, const char* filename); + +float* +fvecs_read(const char* fname, size_t* d_out, size_t* n_out); + +int* +ivecs_read(const char* fname, size_t* d_out, size_t* n_out); \ No newline at end of file diff --git a/core/src/wrapper/ConfAdapter.cpp b/core/src/wrapper/ConfAdapter.cpp index d49747d8f4..7ad1b8b74b 100644 --- a/core/src/wrapper/ConfAdapter.cpp +++ b/core/src/wrapper/ConfAdapter.cpp @@ -201,10 +201,11 @@ NSGConfAdapter::Match(const TempMetaConf& metaconf) { auto scale_factor = round(metaconf.dim / 128.0); scale_factor = scale_factor >= 4 ? 4 : scale_factor; conf->nprobe = int64_t(conf->nlist * 0.01); - conf->knng = 40 + 10 * scale_factor; // the size of knng - conf->search_length = 40 + 5 * scale_factor; +// conf->knng = 40 + 10 * scale_factor; // the size of knng + conf->knng = 50; + conf->search_length = 50 + 5 * scale_factor; conf->out_degree = 50 + 5 * scale_factor; - conf->candidate_pool_size = 200 + 100 * scale_factor; + conf->candidate_pool_size = 300; MatchBase(conf); return conf; } From 31da89d9a286a4bf9b359e085fef7b6cf493333a Mon Sep 17 00:00:00 2001 From: "xiaojun.lin" Date: Sat, 30 Nov 2019 18:07:26 +0800 Subject: [PATCH 23/44] format code --- .../knowhere/knowhere/index/vector_index/IndexGPUIDMAP.h | 1 + .../knowhere/knowhere/index/vector_index/IndexNSG.cpp | 2 +- .../index/knowhere/knowhere/index/vector_index/nsg/NSG.h | 4 ++-- core/src/index/unittest/test_nsg/test_nsg.cpp | 9 +++++---- core/src/index/unittest/utils.h | 2 +- core/src/wrapper/ConfAdapter.cpp | 2 +- 6 files changed, 11 insertions(+), 9 deletions(-) diff --git a/core/src/index/knowhere/knowhere/index/vector_index/IndexGPUIDMAP.h b/core/src/index/knowhere/knowhere/index/vector_index/IndexGPUIDMAP.h index d538f2d0da..31c7039f50 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/IndexGPUIDMAP.h +++ b/core/src/index/knowhere/knowhere/index/vector_index/IndexGPUIDMAP.h @@ -23,6 +23,7 @@ #include #include +#include namespace knowhere { diff --git a/core/src/index/knowhere/knowhere/index/vector_index/IndexNSG.cpp b/core/src/index/knowhere/knowhere/index/vector_index/IndexNSG.cpp index 661bde3bf3..db8b05f992 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/IndexNSG.cpp +++ b/core/src/index/knowhere/knowhere/index/vector_index/IndexNSG.cpp @@ -25,8 +25,8 @@ #include "knowhere/index/vector_index/helpers/Cloner.h" #endif -#include "knowhere/index/vector_index/IndexIVF.h" #include "knowhere/index/vector_index/IndexIDMAP.h" +#include "knowhere/index/vector_index/IndexIVF.h" #include "knowhere/index/vector_index/nsg/NSG.h" #include "knowhere/index/vector_index/nsg/NSGIO.h" diff --git a/core/src/index/knowhere/knowhere/index/vector_index/nsg/NSG.h b/core/src/index/knowhere/knowhere/index/vector_index/nsg/NSG.h index 3dd827466a..f4eefb476e 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/nsg/NSG.h +++ b/core/src/index/knowhere/knowhere/index/vector_index/nsg/NSG.h @@ -53,8 +53,8 @@ class NsgIndex { float* ori_data_; int64_t* ids_; - Graph nsg; // final graph - Graph knng; // reset after build + Graph nsg; // final graph + Graph knng; // reset after build node_t navigation_point; // offset of node in origin data diff --git a/core/src/index/unittest/test_nsg/test_nsg.cpp b/core/src/index/unittest/test_nsg/test_nsg.cpp index 450f5a7723..c67fec0e32 100644 --- a/core/src/index/unittest/test_nsg/test_nsg.cpp +++ b/core/src/index/unittest/test_nsg/test_nsg.cpp @@ -22,9 +22,9 @@ #include "knowhere/index/vector_index/FaissBaseIndex.h" #include "knowhere/index/vector_index/IndexNSG.h" #ifdef MILVUS_GPU_VERSION -#include "knowhere/index/vector_index/helpers/FaissGpuResourceMgr.h" #include "knowhere/index/vector_index/IndexGPUIDMAP.h" #include "knowhere/index/vector_index/helpers/Cloner.h" +#include "knowhere/index/vector_index/helpers/FaissGpuResourceMgr.h" #endif #include "knowhere/common/Timer.h" @@ -120,7 +120,7 @@ TEST_F(NSGInterfaceTest, comparetest) { } //#include -//TEST(test, ori_nsg) { +// TEST(test, ori_nsg) { // // float* p_data = nullptr; // size_t rows, dim; // char* filename = "/mnt/112d53a6-5592-4360-a33b-7fd789456fce/workspace/Data/sift/sift_base.fvecs"; @@ -226,7 +226,7 @@ TEST_F(NSGInterfaceTest, comparetest) { // printf("R@100 = %.4f\n", n_100 / float(nq)); //} // -//TEST(testxx, test_idmap){ +// TEST(testxx, test_idmap){ // int k = 50; // std::string knng_filename = // "/mnt/112d53a6-5592-4360-a33b-7fd789456fce/workspace/Cellar/anns/efanna_graph/tests/sift.50NN.graph"; @@ -234,7 +234,8 @@ TEST_F(NSGInterfaceTest, comparetest) { // Load_nns_graph(gt_knng, knng_filename.c_str()); // // size_t rows, dim; -// char* filename = "/mnt/112d53a6-5592-4360-a33b-7fd789456fce/workspace/Cellar/anns/efanna_graph/tests/siftsmall/siftsmall_base.fvecs"; +// char* filename = +// "/mnt/112d53a6-5592-4360-a33b-7fd789456fce/workspace/Cellar/anns/efanna_graph/tests/siftsmall/siftsmall_base.fvecs"; // float* p_data = fvecs_read(filename, &dim, &rows); // // std::vector store_ids(rows); diff --git a/core/src/index/unittest/utils.h b/core/src/index/unittest/utils.h index f11ad28163..03fd157222 100644 --- a/core/src/index/unittest/utils.h +++ b/core/src/index/unittest/utils.h @@ -101,4 +101,4 @@ float* fvecs_read(const char* fname, size_t* d_out, size_t* n_out); int* -ivecs_read(const char* fname, size_t* d_out, size_t* n_out); \ No newline at end of file +ivecs_read(const char* fname, size_t* d_out, size_t* n_out); diff --git a/core/src/wrapper/ConfAdapter.cpp b/core/src/wrapper/ConfAdapter.cpp index b96f2c80e5..6b1667f9d7 100644 --- a/core/src/wrapper/ConfAdapter.cpp +++ b/core/src/wrapper/ConfAdapter.cpp @@ -204,7 +204,7 @@ NSGConfAdapter::Match(const TempMetaConf& metaconf) { auto scale_factor = round(metaconf.dim / 128.0); scale_factor = scale_factor >= 4 ? 4 : scale_factor; conf->nprobe = int64_t(conf->nlist * 0.01); -// conf->knng = 40 + 10 * scale_factor; // the size of knng + // conf->knng = 40 + 10 * scale_factor; // the size of knng conf->knng = 50; conf->search_length = 50 + 5 * scale_factor; conf->out_degree = 50 + 5 * scale_factor; From 28ca297115c2af6d14b7f9dccf09411031ba20ab Mon Sep 17 00:00:00 2001 From: "xiaojun.lin" Date: Sat, 30 Nov 2019 19:19:01 +0800 Subject: [PATCH 24/44] fix unittest --- core/src/index/unittest/test_nsg/test_nsg.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/core/src/index/unittest/test_nsg/test_nsg.cpp b/core/src/index/unittest/test_nsg/test_nsg.cpp index c67fec0e32..3f7fc0fba5 100644 --- a/core/src/index/unittest/test_nsg/test_nsg.cpp +++ b/core/src/index/unittest/test_nsg/test_nsg.cpp @@ -52,6 +52,7 @@ class NSGInterfaceTest : public DataGen, public ::testing::Test { auto tmp_conf = std::make_shared(); tmp_conf->gpu_id = DEVICEID; + tmp_conf->d = 256; tmp_conf->knng = 20; tmp_conf->nprobe = 8; tmp_conf->nlist = 163; From c1bd16a55e1e229afe56b58f009a0dd1097ffce7 Mon Sep 17 00:00:00 2001 From: bugfixer Date: Sun, 1 Dec 2019 01:40:11 +0000 Subject: [PATCH 25/44] #631 - FAISS isn't compiled with O3 option --- CHANGELOG.md | 1 + core/src/index/cmake/ThirdPartyPackagesCore.cmake | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cb082d2772..c7739c5b4a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -42,6 +42,7 @@ Please mark all change in change log and use the ticket from JIRA. - \#599 - Build index log is incorrect - \#602 - Optimizer specify wrong gpu_id - \#606 - No log generated during building index with CPU +- \#631 - FAISS isn't compiled with O3 option ## Feature - \#12 - Pure CPU version for Milvus diff --git a/core/src/index/cmake/ThirdPartyPackagesCore.cmake b/core/src/index/cmake/ThirdPartyPackagesCore.cmake index 624f1f422e..9c9187d2cc 100644 --- a/core/src/index/cmake/ThirdPartyPackagesCore.cmake +++ b/core/src/index/cmake/ThirdPartyPackagesCore.cmake @@ -708,7 +708,7 @@ macro(build_faiss) set(FAISS_CONFIGURE_ARGS "--prefix=${FAISS_PREFIX}" "CFLAGS=${EP_C_FLAGS}" - "CXXFLAGS=${EP_CXX_FLAGS} -mavx2 -mf16c" + "CXXFLAGS=${EP_CXX_FLAGS} -mavx2 -mf16c -O3" --without-python) if (FAISS_WITH_MKL) From 216a534eff5753b42b12d82fcdbfaad6b72095e4 Mon Sep 17 00:00:00 2001 From: bugfixer Date: Sun, 1 Dec 2019 01:52:17 +0000 Subject: [PATCH 26/44] #634 - FAISS GPU version is compiled with O0 --- CHANGELOG.md | 1 + core/src/index/thirdparty/faiss/makefile.inc.in | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cb082d2772..684b0c51fc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -76,6 +76,7 @@ Please mark all change in change log and use the ticket from JIRA. - \#470 - Small raw files should not be build index - \#584 - Intergrate internal FAISS - \#611 - Remove MILVUS_CPU_VERSION +- \#634 - FAISS GPU version is compiled with O0 ## Task diff --git a/core/src/index/thirdparty/faiss/makefile.inc.in b/core/src/index/thirdparty/faiss/makefile.inc.in index 2aaaf3cd19..744ca6d50e 100644 --- a/core/src/index/thirdparty/faiss/makefile.inc.in +++ b/core/src/index/thirdparty/faiss/makefile.inc.in @@ -17,7 +17,7 @@ NVCC = @NVCC@ CUDA_ROOT = @CUDA_PREFIX@ CUDA_ARCH = @CUDA_ARCH@ NVCCFLAGS = -I $(CUDA_ROOT)/targets/x86_64-linux/include/ \ --O0 -g \ +-O3 \ -Xcompiler -fPIC \ -Xcudafe --diag_suppress=unrecognized_attribute \ $(CUDA_ARCH) \ From fef2a9f9ae80b933a500ad8e429b550dc3f1295d Mon Sep 17 00:00:00 2001 From: zhenwu Date: Sun, 1 Dec 2019 10:50:01 +0800 Subject: [PATCH 27/44] Remove some pq cases --- tests/milvus_python_test/test_add_vectors.py | 2 + tests/milvus_python_test/test_index.py | 293 +++++++++--------- .../milvus_python_test/test_search_vectors.py | 12 +- tests/milvus_python_test/test_table.py | 2 + tests/milvus_python_test/test_table_count.py | 2 + 5 files changed, 167 insertions(+), 144 deletions(-) diff --git a/tests/milvus_python_test/test_add_vectors.py b/tests/milvus_python_test/test_add_vectors.py index 7c9d9e691c..5d06a4f43b 100644 --- a/tests/milvus_python_test/test_add_vectors.py +++ b/tests/milvus_python_test/test_add_vectors.py @@ -31,6 +31,8 @@ class TestAddBase: if "internal" not in args: if request.param["index_type"] == IndexType.IVF_SQ8H: pytest.skip("sq8h not support in open source") + if request.param["index_type"] == IndexType.IVF_PQ: + pytest.skip("Skip PQ Temporary") return request.param def test_add_vector_create_table(self, connect, table): diff --git a/tests/milvus_python_test/test_index.py b/tests/milvus_python_test/test_index.py index b89f682039..924aee270e 100644 --- a/tests/milvus_python_test/test_index.py +++ b/tests/milvus_python_test/test_index.py @@ -14,7 +14,7 @@ from utils import * nb = 10000 dim = 128 -index_file_size = 10 +index_file_size = 20 vectors = gen_vectors(nb, dim) vectors = sklearn.preprocessing.normalize(vectors, axis=1, norm='l2') vectors = vectors.tolist() @@ -63,6 +63,18 @@ class TestIndexBase: status = connect.create_index(table, index_params) assert status.OK() + @pytest.mark.timeout(BUILD_TIMEOUT) + def test_create_index_no_vectors(self, connect, table, get_simple_index_params): + ''' + target: test create index interface + method: create table and add vectors in it, create index + expected: return code equals to 0, and search success + ''' + index_params = get_simple_index_params + logging.getLogger().info(index_params) + status = connect.create_index(table, index_params) + assert status.OK() + @pytest.mark.timeout(BUILD_TIMEOUT) def test_create_index_partition(self, connect, table, get_simple_index_params): ''' @@ -72,6 +84,8 @@ class TestIndexBase: ''' partition_name = gen_unique_str() index_params = get_simple_index_params + if index_params["index_type"] == IndexType.IVF_PQ: + pytest.skip("Skip some PQ cases") logging.getLogger().info(index_params) status = connect.create_partition(table, partition_name, tag) status, ids = connect.add_vectors(table, vectors, partition_tag=tag) @@ -242,6 +256,8 @@ class TestIndexBase: expected: return code equals to 0 ''' index_param = get_simple_index_params + if index_param["index_type"] == IndexType.IVF_PQ: + pytest.skip("Skip some PQ cases") status = connect.create_index(table, index_param) status, ids = connect.add_vectors(table, vectors) assert status.OK() @@ -255,6 +271,8 @@ class TestIndexBase: ''' status, ids = connect.add_vectors(table, vectors) index_param = get_simple_index_params + if index_param["index_type"] == IndexType.IVF_PQ: + pytest.skip("Skip some PQ cases") status = connect.create_index(table, index_param) status = connect.create_index(table, index_param) assert status.OK() @@ -291,15 +309,15 @@ class TestIndexBase: ****************************************************************** """ - def test_describe_index(self, connect, table, get_simple_index_params): + def test_describe_index(self, connect, table, get_index_params): ''' target: test describe index interface method: create table and add vectors in it, create index, call describe index expected: return code 0, and index instructure ''' - index_params = get_simple_index_params + index_params = get_index_params logging.getLogger().info(index_params) - status, ids = connect.add_vectors(table, vectors) + # status, ids = connect.add_vectors(table, vectors) status = connect.create_index(table, index_params) status, result = connect.describe_index(table) logging.getLogger().info(result) @@ -325,6 +343,8 @@ class TestIndexBase: 'metric_type': MetricType.L2} connect.create_table(param) index_params = get_simple_index_params + if index_params["index_type"] == IndexType.IVF_PQ: + pytest.skip("Skip some PQ cases") logging.getLogger().info(index_params) status, ids = connect.add_vectors(table_name=table_name, records=vectors) status = connect.create_index(table_name, index_params) @@ -405,7 +425,7 @@ class TestIndexBase: expected: return code 0, and default index param ''' index_param = get_simple_index_params - status, ids = connect.add_vectors(table, vectors) + # status, ids = connect.add_vectors(table, vectors) status = connect.create_index(table, index_param) assert status.OK() status, result = connect.describe_index(table) @@ -425,7 +445,7 @@ class TestIndexBase: expected: return code 0 ''' index_param = get_simple_index_params - status, ids = connect.add_vectors(table, vectors) + # status, ids = connect.add_vectors(table, vectors) status = connect.create_index(table, index_param) assert status.OK() status, result = connect.describe_index(table) @@ -494,10 +514,9 @@ class TestIndexBase: expected: return code 0 ''' index_params = get_simple_index_params - status, ids = connect.add_vectors(table, vectors) + # status, ids = connect.add_vectors(table, vectors) for i in range(2): status = connect.create_index(table, index_params) - assert status.OK() status, result = connect.describe_index(table) logging.getLogger().info(result) @@ -517,7 +536,7 @@ class TestIndexBase: ''' nlist = 16384 index_params = [{"index_type": IndexType.IVFLAT, "nlist": nlist}, {"index_type": IndexType.IVF_SQ8, "nlist": nlist}] - status, ids = connect.add_vectors(table, vectors) + # status, ids = connect.add_vectors(table, vectors) for i in range(2): status = connect.create_index(table, index_params[i]) assert status.OK() @@ -570,10 +589,7 @@ class TestIndexIP: logging.getLogger().info(index_params) status, ids = connect.add_vectors(ip_table, vectors) status = connect.create_index(ip_table, index_params) - if index_params["index_type"] == IndexType.IVF_PQ: - assert not status.OK() - else: - assert status.OK() + assert status.OK() @pytest.mark.timeout(BUILD_TIMEOUT) def test_create_index_partition(self, connect, ip_table, get_simple_index_params): @@ -584,14 +600,13 @@ class TestIndexIP: ''' partition_name = gen_unique_str() index_params = get_simple_index_params + if index_params["index_type"] == IndexType.IVF_PQ: + pytest.skip("Skip some PQ cases") logging.getLogger().info(index_params) status = connect.create_partition(ip_table, partition_name, tag) status, ids = connect.add_vectors(ip_table, vectors, partition_tag=tag) status = connect.create_index(partition_name, index_params) - if index_params["index_type"] == IndexType.IVF_PQ: - assert not status.OK() - else: - assert status.OK() + assert status.OK() @pytest.mark.level(2) def test_create_index_without_connect(self, dis_connect, ip_table): @@ -616,17 +631,16 @@ class TestIndexIP: logging.getLogger().info(index_params) status, ids = connect.add_vectors(ip_table, vectors) status = connect.create_index(ip_table, index_params) + logging.getLogger().info(connect.describe_index(ip_table)) + query_vecs = [vectors[0], vectors[1], vectors[2]] + top_k = 5 + status, result = connect.search_vectors(ip_table, top_k, nprobe, query_vecs) + logging.getLogger().info(result) if index_params["index_type"] == IndexType.IVF_PQ: assert not status.OK() else: assert status.OK() - logging.getLogger().info(connect.describe_index(ip_table)) - query_vecs = [vectors[0], vectors[1], vectors[2]] - top_k = 5 - status, result = connect.search_vectors(ip_table, top_k, nprobe, query_vecs) - logging.getLogger().info(result) - assert status.OK() - assert len(result) == len(query_vecs) + assert len(result) == len(query_vecs) # TODO: enable @pytest.mark.timeout(BUILD_TIMEOUT) @@ -734,6 +748,8 @@ class TestIndexIP: expected: return code equals to 0 ''' index_param = get_simple_index_params + if index_param["index_type"] == IndexType.IVF_PQ: + pytest.skip("Skip some PQ cases") status = connect.create_index(ip_table, index_param) status, ids = connect.add_vectors(ip_table, vectors) assert status.OK() @@ -792,7 +808,7 @@ class TestIndexIP: ''' index_params = get_simple_index_params logging.getLogger().info(index_params) - status, ids = connect.add_vectors(ip_table, vectors) + # status, ids = connect.add_vectors(ip_table, vectors[:5000]) status = connect.create_index(ip_table, index_params) status, result = connect.describe_index(ip_table) logging.getLogger().info(result) @@ -808,6 +824,8 @@ class TestIndexIP: ''' partition_name = gen_unique_str() index_params = get_simple_index_params + if index_params["index_type"] == IndexType.IVF_PQ: + pytest.skip("Skip some PQ cases") logging.getLogger().info(index_params) status = connect.create_partition(ip_table, partition_name, tag) status, ids = connect.add_vectors(ip_table, vectors, partition_tag=tag) @@ -831,6 +849,8 @@ class TestIndexIP: ''' partition_name = gen_unique_str() index_params = get_simple_index_params + if index_params["index_type"] == IndexType.IVF_PQ: + pytest.skip("Skip some PQ cases") logging.getLogger().info(index_params) status = connect.create_partition(ip_table, partition_name, tag) status, ids = connect.add_vectors(ip_table, vectors, partition_tag=tag) @@ -856,6 +876,8 @@ class TestIndexIP: new_partition_name = gen_unique_str() new_tag = "new_tag" index_params = get_simple_index_params + if index_params["index_type"] == IndexType.IVF_PQ: + pytest.skip("Skip some PQ cases") logging.getLogger().info(index_params) status = connect.create_partition(ip_table, partition_name, tag) status = connect.create_partition(ip_table, new_partition_name, new_tag) @@ -892,6 +914,8 @@ class TestIndexIP: 'metric_type': MetricType.IP} connect.create_table(param) index_params = get_simple_index_params + if index_params["index_type"] == IndexType.IVF_PQ: + pytest.skip("Skip some PQ cases") logging.getLogger().info(index_params) status, ids = connect.add_vectors(table_name=table_name, records=vectors) status = connect.create_index(table_name, index_params) @@ -944,28 +968,25 @@ class TestIndexIP: ****************************************************************** """ - def test_drop_index(self, connect, ip_table, get_index_params): + def test_drop_index(self, connect, ip_table, get_simple_index_params): ''' target: test drop index interface method: create table and add vectors in it, create index, call drop index expected: return code 0, and default index param ''' - index_params = get_index_params + index_params = get_simple_index_params status, ids = connect.add_vectors(ip_table, vectors) status = connect.create_index(ip_table, index_params) - if index_params["index_type"] == IndexType.IVF_PQ: - assert not status.OK() - else: - assert status.OK() - status, result = connect.describe_index(ip_table) - logging.getLogger().info(result) - status = connect.drop_index(ip_table) - assert status.OK() - status, result = connect.describe_index(ip_table) - logging.getLogger().info(result) - assert result._nlist == 16384 - assert result._table_name == ip_table - assert result._index_type == IndexType.FLAT + assert status.OK() + status, result = connect.describe_index(ip_table) + logging.getLogger().info(result) + status = connect.drop_index(ip_table) + assert status.OK() + status, result = connect.describe_index(ip_table) + logging.getLogger().info(result) + assert result._nlist == 16384 + assert result._table_name == ip_table + assert result._index_type == IndexType.FLAT def test_drop_index_partition(self, connect, ip_table, get_simple_index_params): ''' @@ -975,22 +996,21 @@ class TestIndexIP: ''' partition_name = gen_unique_str() index_params = get_simple_index_params + if index_params["index_type"] == IndexType.IVF_PQ: + pytest.skip("Skip some PQ cases") status = connect.create_partition(ip_table, partition_name, tag) status, ids = connect.add_vectors(ip_table, vectors, partition_tag=tag) status = connect.create_index(ip_table, index_params) - if index_params["index_type"] == IndexType.IVF_PQ: - assert not status.OK() - else: - assert status.OK() - status, result = connect.describe_index(ip_table) - logging.getLogger().info(result) - status = connect.drop_index(ip_table) - assert status.OK() - status, result = connect.describe_index(ip_table) - logging.getLogger().info(result) - assert result._nlist == 16384 - assert result._table_name == ip_table - assert result._index_type == IndexType.FLAT + assert status.OK() + status, result = connect.describe_index(ip_table) + logging.getLogger().info(result) + status = connect.drop_index(ip_table) + assert status.OK() + status, result = connect.describe_index(ip_table) + logging.getLogger().info(result) + assert result._nlist == 16384 + assert result._table_name == ip_table + assert result._index_type == IndexType.FLAT def test_drop_index_partition_A(self, connect, ip_table, get_simple_index_params): ''' @@ -1000,25 +1020,24 @@ class TestIndexIP: ''' partition_name = gen_unique_str() index_params = get_simple_index_params + if index_params["index_type"] == IndexType.IVF_PQ: + pytest.skip("Skip some PQ cases") status = connect.create_partition(ip_table, partition_name, tag) status, ids = connect.add_vectors(ip_table, vectors, partition_tag=tag) status = connect.create_index(partition_name, index_params) - if index_params["index_type"] == IndexType.IVF_PQ: - assert not status.OK() - else: - assert status.OK() - status = connect.drop_index(ip_table) - assert status.OK() - status, result = connect.describe_index(ip_table) - logging.getLogger().info(result) - assert result._nlist == 16384 - assert result._table_name == ip_table - assert result._index_type == IndexType.FLAT - status, result = connect.describe_index(partition_name) - logging.getLogger().info(result) - assert result._nlist == 16384 - assert result._table_name == partition_name - assert result._index_type == IndexType.FLAT + assert status.OK() + status = connect.drop_index(ip_table) + assert status.OK() + status, result = connect.describe_index(ip_table) + logging.getLogger().info(result) + assert result._nlist == 16384 + assert result._table_name == ip_table + assert result._index_type == IndexType.FLAT + status, result = connect.describe_index(partition_name) + logging.getLogger().info(result) + assert result._nlist == 16384 + assert result._table_name == partition_name + assert result._index_type == IndexType.FLAT def test_drop_index_partition_B(self, connect, ip_table, get_simple_index_params): ''' @@ -1028,25 +1047,24 @@ class TestIndexIP: ''' partition_name = gen_unique_str() index_params = get_simple_index_params + if index_params["index_type"] == IndexType.IVF_PQ: + pytest.skip("Skip some PQ cases") status = connect.create_partition(ip_table, partition_name, tag) status, ids = connect.add_vectors(ip_table, vectors, partition_tag=tag) status = connect.create_index(partition_name, index_params) - if index_params["index_type"] == IndexType.IVF_PQ: - assert not status.OK() - else: - assert status.OK() - status = connect.drop_index(partition_name) - assert status.OK() - status, result = connect.describe_index(ip_table) - logging.getLogger().info(result) - assert result._nlist == 16384 - assert result._table_name == ip_table - assert result._index_type == IndexType.FLAT - status, result = connect.describe_index(partition_name) - logging.getLogger().info(result) - assert result._nlist == 16384 - assert result._table_name == partition_name - assert result._index_type == IndexType.FLAT + assert status.OK() + status = connect.drop_index(partition_name) + assert status.OK() + status, result = connect.describe_index(ip_table) + logging.getLogger().info(result) + assert result._nlist == 16384 + assert result._table_name == ip_table + assert result._index_type == IndexType.FLAT + status, result = connect.describe_index(partition_name) + logging.getLogger().info(result) + assert result._nlist == 16384 + assert result._table_name == partition_name + assert result._index_type == IndexType.FLAT def test_drop_index_partition_C(self, connect, ip_table, get_simple_index_params): ''' @@ -1058,31 +1076,30 @@ class TestIndexIP: new_partition_name = gen_unique_str() new_tag = "new_tag" index_params = get_simple_index_params + if index_params["index_type"] == IndexType.IVF_PQ: + pytest.skip("Skip some PQ cases") status = connect.create_partition(ip_table, partition_name, tag) status = connect.create_partition(ip_table, new_partition_name, new_tag) status, ids = connect.add_vectors(ip_table, vectors) status = connect.create_index(ip_table, index_params) - if index_params["index_type"] == IndexType.IVF_PQ: - assert not status.OK() - else: - assert status.OK() - status = connect.drop_index(new_partition_name) - assert status.OK() - status, result = connect.describe_index(new_partition_name) - logging.getLogger().info(result) - assert result._nlist == 16384 - assert result._table_name == new_partition_name - assert result._index_type == IndexType.FLAT - status, result = connect.describe_index(partition_name) - logging.getLogger().info(result) - assert result._nlist == index_params["nlist"] - assert result._table_name == partition_name - assert result._index_type == index_params["index_type"] - status, result = connect.describe_index(ip_table) - logging.getLogger().info(result) - assert result._nlist == index_params["nlist"] - assert result._table_name == ip_table - assert result._index_type == index_params["index_type"] + assert status.OK() + status = connect.drop_index(new_partition_name) + assert status.OK() + status, result = connect.describe_index(new_partition_name) + logging.getLogger().info(result) + assert result._nlist == 16384 + assert result._table_name == new_partition_name + assert result._index_type == IndexType.FLAT + status, result = connect.describe_index(partition_name) + logging.getLogger().info(result) + assert result._nlist == index_params["nlist"] + assert result._table_name == partition_name + assert result._index_type == index_params["index_type"] + status, result = connect.describe_index(ip_table) + logging.getLogger().info(result) + assert result._nlist == index_params["nlist"] + assert result._table_name == ip_table + assert result._index_type == index_params["index_type"] def test_drop_index_repeatly(self, connect, ip_table, get_simple_index_params): ''' @@ -1091,23 +1108,20 @@ class TestIndexIP: expected: return code 0 ''' index_params = get_simple_index_params - status, ids = connect.add_vectors(ip_table, vectors) + # status, ids = connect.add_vectors(ip_table, vectors) status = connect.create_index(ip_table, index_params) - if index_params["index_type"] == IndexType.IVF_PQ: - assert not status.OK() - else: - assert status.OK() - status, result = connect.describe_index(ip_table) - logging.getLogger().info(result) - status = connect.drop_index(ip_table) - assert status.OK() - status = connect.drop_index(ip_table) - assert status.OK() - status, result = connect.describe_index(ip_table) - logging.getLogger().info(result) - assert result._nlist == 16384 - assert result._table_name == ip_table - assert result._index_type == IndexType.FLAT + assert status.OK() + status, result = connect.describe_index(ip_table) + logging.getLogger().info(result) + status = connect.drop_index(ip_table) + assert status.OK() + status = connect.drop_index(ip_table) + assert status.OK() + status, result = connect.describe_index(ip_table) + logging.getLogger().info(result) + assert result._nlist == 16384 + assert result._table_name == ip_table + assert result._index_type == IndexType.FLAT @pytest.mark.level(2) def test_drop_index_without_connect(self, dis_connect, ip_table): @@ -1145,22 +1159,21 @@ class TestIndexIP: expected: return code 0 ''' index_params = get_simple_index_params + if index_params["index_type"] == IndexType.IVF_PQ: + pytest.skip("Skip some PQ cases") status, ids = connect.add_vectors(ip_table, vectors) for i in range(2): status = connect.create_index(ip_table, index_params) - if index_params["index_type"] == IndexType.IVF_PQ: - assert not status.OK() - else: - assert status.OK() - status, result = connect.describe_index(ip_table) - logging.getLogger().info(result) - status = connect.drop_index(ip_table) - assert status.OK() - status, result = connect.describe_index(ip_table) - logging.getLogger().info(result) - assert result._nlist == 16384 - assert result._table_name == ip_table - assert result._index_type == IndexType.FLAT + assert status.OK() + status, result = connect.describe_index(ip_table) + logging.getLogger().info(result) + status = connect.drop_index(ip_table) + assert status.OK() + status, result = connect.describe_index(ip_table) + logging.getLogger().info(result) + assert result._nlist == 16384 + assert result._table_name == ip_table + assert result._index_type == IndexType.FLAT def test_create_drop_index_repeatly_different_index_params(self, connect, ip_table): ''' @@ -1200,7 +1213,7 @@ class TestIndexTableInvalid(object): def get_table_name(self, request): yield request.param - @pytest.mark.level(2) + @pytest.mark.level(1) def test_create_index_with_invalid_tablename(self, connect, get_table_name): table_name = get_table_name nlist = 16384 @@ -1208,13 +1221,13 @@ class TestIndexTableInvalid(object): status = connect.create_index(table_name, index_param) assert not status.OK() - @pytest.mark.level(2) + @pytest.mark.level(1) def test_describe_index_with_invalid_tablename(self, connect, get_table_name): table_name = get_table_name status, result = connect.describe_index(table_name) assert not status.OK() - @pytest.mark.level(2) + @pytest.mark.level(1) def test_drop_index_with_invalid_tablename(self, connect, get_table_name): table_name = get_table_name status = connect.drop_index(table_name) @@ -1232,13 +1245,13 @@ class TestCreateIndexParamsInvalid(object): def get_index_params(self, request): yield request.param - @pytest.mark.level(2) + @pytest.mark.level(1) def test_create_index_with_invalid_index_params(self, connect, table, get_index_params): index_params = get_index_params index_type = index_params["index_type"] nlist = index_params["nlist"] logging.getLogger().info(index_params) - status, ids = connect.add_vectors(table, vectors) + # status, ids = connect.add_vectors(table, vectors) if (not index_type) or (not nlist) or (not isinstance(index_type, IndexType)) or (not isinstance(nlist, int)): with pytest.raises(Exception) as e: status = connect.create_index(table, index_params) diff --git a/tests/milvus_python_test/test_search_vectors.py b/tests/milvus_python_test/test_search_vectors.py index 7aebc78e31..464a28efea 100644 --- a/tests/milvus_python_test/test_search_vectors.py +++ b/tests/milvus_python_test/test_search_vectors.py @@ -48,6 +48,8 @@ class TestSearchBase: if "internal" not in args: if request.param["index_type"] == IndexType.IVF_SQ8H: pytest.skip("sq8h not support in open source") + if request.param["index_type"] == IndexType.IVF_PQ: + pytest.skip("skip pq case temporary") return request.param @pytest.fixture( @@ -58,6 +60,8 @@ class TestSearchBase: if "internal" not in args: if request.param["index_type"] == IndexType.IVF_SQ8H: pytest.skip("sq8h not support in open source") + if request.param["index_type"] == IndexType.IVF_PQ: + pytest.skip("skip pq case temporary") return request.param """ generate top-k params @@ -89,13 +93,13 @@ class TestSearchBase: else: assert not status.OK() - def test_search_l2_index_params(self, connect, table, get_index_params): + def test_search_l2_index_params(self, connect, table, get_simple_index_params): ''' target: test basic search fuction, all the search params is corrent, test all index params, and build method: search with the given vectors, check the result expected: search status ok, and the length of the result is top_k ''' - index_params = get_index_params + index_params = get_simple_index_params logging.getLogger().info(index_params) vectors, ids = self.init_data(connect, table) status = connect.create_index(table, index_params) @@ -297,14 +301,14 @@ class TestSearchBase: assert result[0][0].distance <= epsilon assert result[1][0].distance <= epsilon - def test_search_ip_index_params(self, connect, ip_table, get_index_params): + def test_search_ip_index_params(self, connect, ip_table, get_simple_index_params): ''' target: test basic search fuction, all the search params is corrent, test all index params, and build method: search with the given vectors, check the result expected: search status ok, and the length of the result is top_k ''' - index_params = get_index_params + index_params = get_simple_index_params logging.getLogger().info(index_params) vectors, ids = self.init_data(connect, ip_table) status = connect.create_index(ip_table, index_params) diff --git a/tests/milvus_python_test/test_table.py b/tests/milvus_python_test/test_table.py index 40b0850859..db1b0c7a08 100644 --- a/tests/milvus_python_test/test_table.py +++ b/tests/milvus_python_test/test_table.py @@ -594,6 +594,8 @@ class TestTable: if "internal" not in args: if request.param["index_type"] == IndexType.IVF_SQ8H: pytest.skip("sq8h not support in open source") + # if request.param["index_type"] == IndexType.IVF_PQ: + # pytest.skip("sq8h not support in open source") return request.param @pytest.mark.level(1) diff --git a/tests/milvus_python_test/test_table_count.py b/tests/milvus_python_test/test_table_count.py index a96ccb12b3..bcff3aca1f 100644 --- a/tests/milvus_python_test/test_table_count.py +++ b/tests/milvus_python_test/test_table_count.py @@ -270,6 +270,8 @@ class TestTableCountIP: if "internal" not in args: if request.param["index_type"] == IndexType.IVF_SQ8H: pytest.skip("sq8h not support in open source") + if request.param["index_type"] == IndexType.IVF_PQ: + pytest.skip("skip pq case temporary") return request.param def test_table_rows_count(self, connect, ip_table, add_vectors_nb): From ba3625677a19d7ab1e2e7c10c6f74287059d2538 Mon Sep 17 00:00:00 2001 From: zhenwu Date: Sun, 1 Dec 2019 12:43:19 +0800 Subject: [PATCH 28/44] assert failed if create index with pq on ip_table --- tests/milvus_python_test/test_index.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/milvus_python_test/test_index.py b/tests/milvus_python_test/test_index.py index 924aee270e..3a3ffc3857 100644 --- a/tests/milvus_python_test/test_index.py +++ b/tests/milvus_python_test/test_index.py @@ -636,10 +636,7 @@ class TestIndexIP: top_k = 5 status, result = connect.search_vectors(ip_table, top_k, nprobe, query_vecs) logging.getLogger().info(result) - if index_params["index_type"] == IndexType.IVF_PQ: - assert not status.OK() - else: - assert status.OK() + assert status.OK() assert len(result) == len(query_vecs) # TODO: enable @@ -975,9 +972,12 @@ class TestIndexIP: expected: return code 0, and default index param ''' index_params = get_simple_index_params - status, ids = connect.add_vectors(ip_table, vectors) + # status, ids = connect.add_vectors(ip_table, vectors) status = connect.create_index(ip_table, index_params) - assert status.OK() + if index_params["index_type"] == IndexType.IVF_PQ: + assert not status.OK() + else: + assert status.OK() status, result = connect.describe_index(ip_table) logging.getLogger().info(result) status = connect.drop_index(ip_table) From 8862756d11f71899acc60aa168fa2308cf789089 Mon Sep 17 00:00:00 2001 From: groot Date: Mon, 2 Dec 2019 10:56:11 +0800 Subject: [PATCH 29/44] typo --- core/src/db/meta/MySQLMetaImpl.cpp | 4 ++-- core/src/db/meta/SqliteMetaImpl.cpp | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/core/src/db/meta/MySQLMetaImpl.cpp b/core/src/db/meta/MySQLMetaImpl.cpp index 48bceca7d8..da91585b92 100644 --- a/core/src/db/meta/MySQLMetaImpl.cpp +++ b/core/src/db/meta/MySQLMetaImpl.cpp @@ -1838,9 +1838,9 @@ MySQLMetaImpl::CleanUpFilesWithTTL(uint64_t seconds, CleanUpFilter* filter) { idsToDelete.emplace_back(std::to_string(table_file.id_)); table_ids.insert(table_file.table_id_); - } - clean_files++; + clean_files++; + } } // delete file from meta diff --git a/core/src/db/meta/SqliteMetaImpl.cpp b/core/src/db/meta/SqliteMetaImpl.cpp index c74212e6e7..9e323a4fac 100644 --- a/core/src/db/meta/SqliteMetaImpl.cpp +++ b/core/src/db/meta/SqliteMetaImpl.cpp @@ -1353,9 +1353,9 @@ SqliteMetaImpl::CleanUpFilesWithTTL(uint64_t seconds, CleanUpFilter* filter) { ENGINE_LOG_DEBUG << "Remove file id:" << table_file.file_id_ << " location:" << table_file.location_; table_ids.insert(table_file.table_id_); - } - clean_files++; + clean_files++; + } } return true; }); From b12f861b5113a9038cad5e24abda16ea15856fcf Mon Sep 17 00:00:00 2001 From: wxyu Date: Mon, 2 Dec 2019 11:17:24 +0800 Subject: [PATCH 30/44] Add a new rpc command to get milvus build version whether cpu or gpu close#644 --- CHANGELOG.md | 1 + core/src/server/grpc_impl/request/CmdRequest.cpp | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 19a9c4519d..d78fe75cba 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -57,6 +57,7 @@ Please mark all change in change log and use the ticket from JIRA. - \#502 - C++ SDK support IVFPQ and SPTAG - \#560 - Add version in server config file - \#605 - Print more messages when server start +- \#644 - Add a new rpc command to get milvus build version whether cpu or gpu ## Improvement - \#255 - Add ivfsq8 test report detailed version diff --git a/core/src/server/grpc_impl/request/CmdRequest.cpp b/core/src/server/grpc_impl/request/CmdRequest.cpp index b215f94d31..3afbe6d6e2 100644 --- a/core/src/server/grpc_impl/request/CmdRequest.cpp +++ b/core/src/server/grpc_impl/request/CmdRequest.cpp @@ -39,6 +39,12 @@ CmdRequest::OnExecute() { result_ = MILVUS_VERSION; } else if (cmd_ == "tasktable") { result_ = scheduler::ResMgrInst::GetInstance()->DumpTaskTables(); + } else if (cmd_ == "mode") { +#ifdef MILVUS_GPU_VERSION + result_ = "GPU"; +#else + result_ = "CPU"; +#endif } else { result_ = "OK"; } From 9c9af89f7b2f7c0aad2ae1bc4a4fd259a0e9dd07 Mon Sep 17 00:00:00 2001 From: jielinxu <52057195+jielinxu@users.noreply.github.com> Date: Mon, 2 Dec 2019 11:46:29 +0800 Subject: [PATCH 31/44] [skip ci] Add note about troubleshooting --- install.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/install.md b/install.md index fd11121f39..d9914f66b8 100644 --- a/install.md +++ b/install.md @@ -3,6 +3,8 @@ - [Build from source](#build-from-source) - [Compile Milvus on Docker](#compile-milvus-on-docker) +If you encounter any problems/issues compiling Milvus from source, please refer to [Troubleshooting](#troubleshooting). + ## Build from source ### Requirements @@ -151,11 +153,11 @@ $ ./start_server.sh 1. If you encounter the following error when compiling: `protocol https not supported or disabled in libcurl`. First, make sure you have `libcurl4-openssl-dev` installed in your system. -Then try reinstall CMake from source with `--system-curl` option: +Then try reinstalling the latest CMake from source with `--system-curl` option: ```shell $ ./bootstrap --system-curl $ make $ sudo make install ``` - +If the `--system-curl` doesn't work, you can also reinstall CMake in **Ubuntu Software** on your local computer. From a78fdb2fd6d035f3b110f1db32ead2302cce6e7f Mon Sep 17 00:00:00 2001 From: jielinxu <52057195+jielinxu@users.noreply.github.com> Date: Mon, 2 Dec 2019 11:47:13 +0800 Subject: [PATCH 32/44] [skip ci] minor change --- install.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install.md b/install.md index d9914f66b8..35884ffd52 100644 --- a/install.md +++ b/install.md @@ -160,4 +160,4 @@ Then try reinstalling the latest CMake from source with `--system-curl` option: $ make $ sudo make install ``` -If the `--system-curl` doesn't work, you can also reinstall CMake in **Ubuntu Software** on your local computer. +If the `--system-curl` command doesn't work, you can also reinstall CMake in **Ubuntu Software** on your local computer. From 62b6db567d0990144b28b7e684b255d0cce2c030 Mon Sep 17 00:00:00 2001 From: groot Date: Mon, 2 Dec 2019 11:57:43 +0800 Subject: [PATCH 33/44] add more logs --- core/src/db/DBImpl.cpp | 1 - core/src/db/engine/ExecutionEngineImpl.cpp | 2 ++ core/src/server/grpc_impl/request/CmdRequest.cpp | 5 +++++ core/src/server/grpc_impl/request/CountTableRequest.cpp | 3 ++- core/src/server/grpc_impl/request/CreateIndexRequest.cpp | 3 ++- .../src/server/grpc_impl/request/CreatePartitionRequest.cpp | 6 +++++- core/src/server/grpc_impl/request/CreateTableRequest.cpp | 5 ++++- core/src/server/grpc_impl/request/DescribeIndexRequest.cpp | 3 ++- core/src/server/grpc_impl/request/DescribeTableRequest.cpp | 3 ++- core/src/server/grpc_impl/request/DropIndexRequest.cpp | 3 ++- core/src/server/grpc_impl/request/DropPartitionRequest.cpp | 5 +++++ core/src/server/grpc_impl/request/DropTableRequest.cpp | 3 ++- core/src/server/grpc_impl/request/HasTableRequest.cpp | 3 ++- core/src/server/grpc_impl/request/InsertRequest.cpp | 3 ++- core/src/server/grpc_impl/request/PreloadTableRequest.cpp | 3 ++- core/src/server/grpc_impl/request/ShowPartitionsRequest.cpp | 3 +++ core/src/server/grpc_impl/request/ShowTablesRequest.cpp | 2 ++ 17 files changed, 44 insertions(+), 12 deletions(-) diff --git a/core/src/db/DBImpl.cpp b/core/src/db/DBImpl.cpp index 779e1b731f..8f8516770f 100644 --- a/core/src/db/DBImpl.cpp +++ b/core/src/db/DBImpl.cpp @@ -695,7 +695,6 @@ DBImpl::MergeFiles(const std::string& table_id, const meta::DateT& date, const m auto file_schema = file; file_schema.file_type_ = meta::TableFileSchema::TO_DELETE; updated.push_back(file_schema); - ENGINE_LOG_DEBUG << "Merging file " << file_schema.file_id_; index_size = index->Size(); if (index_size >= file_schema.index_file_size_) { diff --git a/core/src/db/engine/ExecutionEngineImpl.cpp b/core/src/db/engine/ExecutionEngineImpl.cpp index 9464d66f28..f3fd4a67cf 100644 --- a/core/src/db/engine/ExecutionEngineImpl.cpp +++ b/core/src/db/engine/ExecutionEngineImpl.cpp @@ -472,6 +472,8 @@ ExecutionEngineImpl::Merge(const std::string& location) { auto status = index_->Add(file_index->Count(), file_index->GetRawVectors(), file_index->GetRawIds()); if (!status.ok()) { ENGINE_LOG_ERROR << "Failed to merge: " << location << " to: " << location_; + } else { + ENGINE_LOG_DEBUG << "Finish merge index file: " << location; } return status; } else { diff --git a/core/src/server/grpc_impl/request/CmdRequest.cpp b/core/src/server/grpc_impl/request/CmdRequest.cpp index b215f94d31..e3e2c06ad8 100644 --- a/core/src/server/grpc_impl/request/CmdRequest.cpp +++ b/core/src/server/grpc_impl/request/CmdRequest.cpp @@ -17,6 +17,8 @@ #include "server/grpc_impl/request/CmdRequest.h" #include "scheduler/SchedInst.h" +#include "utils/Log.h" +#include "utils/TimeRecorder.h" #include @@ -35,6 +37,9 @@ CmdRequest::Create(const std::string& cmd, std::string& result) { Status CmdRequest::OnExecute() { + std::string hdr = "CmdRequest(cmd=" + cmd_ + ")"; + TimeRecorder rc(hdr); + if (cmd_ == "version") { result_ = MILVUS_VERSION; } else if (cmd_ == "tasktable") { diff --git a/core/src/server/grpc_impl/request/CountTableRequest.cpp b/core/src/server/grpc_impl/request/CountTableRequest.cpp index 8559890ad6..b90a33bf61 100644 --- a/core/src/server/grpc_impl/request/CountTableRequest.cpp +++ b/core/src/server/grpc_impl/request/CountTableRequest.cpp @@ -39,7 +39,8 @@ CountTableRequest::Create(const std::string& table_name, int64_t& row_count) { Status CountTableRequest::OnExecute() { try { - TimeRecorder rc("CountTableRequest"); + std::string hdr = "CountTableRequest(table=" + table_name_ + ")"; + TimeRecorder rc(hdr); // step 1: check arguments auto status = ValidationUtil::ValidateTableName(table_name_); diff --git a/core/src/server/grpc_impl/request/CreateIndexRequest.cpp b/core/src/server/grpc_impl/request/CreateIndexRequest.cpp index 72678aee87..c7628048ff 100644 --- a/core/src/server/grpc_impl/request/CreateIndexRequest.cpp +++ b/core/src/server/grpc_impl/request/CreateIndexRequest.cpp @@ -44,7 +44,8 @@ CreateIndexRequest::Create(const ::milvus::grpc::IndexParam* index_param) { Status CreateIndexRequest::OnExecute() { try { - TimeRecorder rc("CreateIndexRequest"); + std::string hdr = "CreateIndexRequest(table=" + index_param_->table_name() + ")"; + TimeRecorder rc(hdr); // step 1: check arguments std::string table_name_ = index_param_->table_name(); diff --git a/core/src/server/grpc_impl/request/CreatePartitionRequest.cpp b/core/src/server/grpc_impl/request/CreatePartitionRequest.cpp index 3bd4a86ef6..1a18cdfcad 100644 --- a/core/src/server/grpc_impl/request/CreatePartitionRequest.cpp +++ b/core/src/server/grpc_impl/request/CreatePartitionRequest.cpp @@ -22,6 +22,7 @@ #include "utils/ValidationUtil.h" #include +#include namespace milvus { namespace server { @@ -42,7 +43,10 @@ CreatePartitionRequest::Create(const ::milvus::grpc::PartitionParam* partition_p Status CreatePartitionRequest::OnExecute() { - TimeRecorder rc("CreatePartitionRequest"); + std::string hdr = "CreatePartitionRequest(table=" + partition_param_->table_name() + + ", partition_name=" + partition_param_->partition_name() + + ", partition_tag=" + partition_param_->tag() + ")"; + TimeRecorder rc(hdr); try { // step 1: check arguments diff --git a/core/src/server/grpc_impl/request/CreateTableRequest.cpp b/core/src/server/grpc_impl/request/CreateTableRequest.cpp index 67a3eaa877..55d953aa6f 100644 --- a/core/src/server/grpc_impl/request/CreateTableRequest.cpp +++ b/core/src/server/grpc_impl/request/CreateTableRequest.cpp @@ -22,6 +22,7 @@ #include "utils/ValidationUtil.h" #include +#include namespace milvus { namespace server { @@ -42,7 +43,9 @@ CreateTableRequest::Create(const ::milvus::grpc::TableSchema* schema) { Status CreateTableRequest::OnExecute() { - TimeRecorder rc("CreateTableRequest"); + std::string hdr = "CreateTableRequest(table=" + schema_->table_name() + + ", dimension=" + std::to_string(schema_->dimension()) + ")"; + TimeRecorder rc(hdr); try { // step 1: check arguments diff --git a/core/src/server/grpc_impl/request/DescribeIndexRequest.cpp b/core/src/server/grpc_impl/request/DescribeIndexRequest.cpp index b3a987c6b0..d57d47bf2c 100644 --- a/core/src/server/grpc_impl/request/DescribeIndexRequest.cpp +++ b/core/src/server/grpc_impl/request/DescribeIndexRequest.cpp @@ -39,7 +39,8 @@ DescribeIndexRequest::Create(const std::string& table_name, ::milvus::grpc::Inde Status DescribeIndexRequest::OnExecute() { try { - TimeRecorder rc("DescribeIndexRequest"); + std::string hdr = "DescribeIndexRequest(table=" + table_name_ + ")"; + TimeRecorder rc(hdr); // step 1: check arguments auto status = ValidationUtil::ValidateTableName(table_name_); diff --git a/core/src/server/grpc_impl/request/DescribeTableRequest.cpp b/core/src/server/grpc_impl/request/DescribeTableRequest.cpp index 28a5f327c5..3bd97ef7e8 100644 --- a/core/src/server/grpc_impl/request/DescribeTableRequest.cpp +++ b/core/src/server/grpc_impl/request/DescribeTableRequest.cpp @@ -38,7 +38,8 @@ DescribeTableRequest::Create(const std::string& table_name, ::milvus::grpc::Tabl Status DescribeTableRequest::OnExecute() { - TimeRecorder rc("DescribeTableRequest"); + std::string hdr = "DescribeTableRequest(table=" + table_name_ + ")"; + TimeRecorder rc(hdr); try { // step 1: check arguments diff --git a/core/src/server/grpc_impl/request/DropIndexRequest.cpp b/core/src/server/grpc_impl/request/DropIndexRequest.cpp index ab5c83b0e5..619ea753ba 100644 --- a/core/src/server/grpc_impl/request/DropIndexRequest.cpp +++ b/core/src/server/grpc_impl/request/DropIndexRequest.cpp @@ -39,7 +39,8 @@ DropIndexRequest::Create(const std::string& table_name) { Status DropIndexRequest::OnExecute() { try { - TimeRecorder rc("DropIndexRequest"); + std::string hdr = "DropIndexRequest(table=" + table_name_ + ")"; + TimeRecorder rc(hdr); // step 1: check arguments auto status = ValidationUtil::ValidateTableName(table_name_); diff --git a/core/src/server/grpc_impl/request/DropPartitionRequest.cpp b/core/src/server/grpc_impl/request/DropPartitionRequest.cpp index 0e29b6abe8..1ec189986a 100644 --- a/core/src/server/grpc_impl/request/DropPartitionRequest.cpp +++ b/core/src/server/grpc_impl/request/DropPartitionRequest.cpp @@ -39,6 +39,11 @@ DropPartitionRequest::Create(const ::milvus::grpc::PartitionParam* partition_par Status DropPartitionRequest::OnExecute() { + std::string hdr = "DropPartitionRequest(table=" + partition_param_->table_name() + + ", partition_name=" + partition_param_->partition_name() + + ", partition_tag=" + partition_param_->tag() + ")"; + TimeRecorder rc(hdr); + std::string table_name = partition_param_->table_name(); std::string partition_name = partition_param_->partition_name(); std::string partition_tag = partition_param_->tag(); diff --git a/core/src/server/grpc_impl/request/DropTableRequest.cpp b/core/src/server/grpc_impl/request/DropTableRequest.cpp index a678e5a6f6..6f81a5b2a0 100644 --- a/core/src/server/grpc_impl/request/DropTableRequest.cpp +++ b/core/src/server/grpc_impl/request/DropTableRequest.cpp @@ -40,7 +40,8 @@ DropTableRequest::Create(const std::string& table_name) { Status DropTableRequest::OnExecute() { try { - TimeRecorder rc("DropTableRequest"); + std::string hdr = "DropTableRequest(table=" + table_name_ + ")"; + TimeRecorder rc(hdr); // step 1: check arguments auto status = ValidationUtil::ValidateTableName(table_name_); diff --git a/core/src/server/grpc_impl/request/HasTableRequest.cpp b/core/src/server/grpc_impl/request/HasTableRequest.cpp index 2909c93056..434580efdf 100644 --- a/core/src/server/grpc_impl/request/HasTableRequest.cpp +++ b/core/src/server/grpc_impl/request/HasTableRequest.cpp @@ -39,7 +39,8 @@ HasTableRequest::Create(const std::string& table_name, bool& has_table) { Status HasTableRequest::OnExecute() { try { - TimeRecorder rc("HasTableRequest"); + std::string hdr = "HasTableRequest(table=" + table_name_ + ")"; + TimeRecorder rc(hdr); // step 1: check arguments auto status = ValidationUtil::ValidateTableName(table_name_); diff --git a/core/src/server/grpc_impl/request/InsertRequest.cpp b/core/src/server/grpc_impl/request/InsertRequest.cpp index 16d16ca8dc..df65c679ed 100644 --- a/core/src/server/grpc_impl/request/InsertRequest.cpp +++ b/core/src/server/grpc_impl/request/InsertRequest.cpp @@ -46,7 +46,8 @@ Status InsertRequest::OnExecute() { try { std::string hdr = "InsertRequest(table=" + insert_param_->table_name() + - ", n=" + std::to_string(insert_param_->row_record_array_size()) + ")"; + ", n=" + std::to_string(insert_param_->row_record_array_size()) + + ", partition_tag=" + insert_param_->partition_tag() + ")"; TimeRecorder rc(hdr); // step 1: check arguments diff --git a/core/src/server/grpc_impl/request/PreloadTableRequest.cpp b/core/src/server/grpc_impl/request/PreloadTableRequest.cpp index e26aa8a877..3c46524afe 100644 --- a/core/src/server/grpc_impl/request/PreloadTableRequest.cpp +++ b/core/src/server/grpc_impl/request/PreloadTableRequest.cpp @@ -39,7 +39,8 @@ PreloadTableRequest::Create(const std::string& table_name) { Status PreloadTableRequest::OnExecute() { try { - TimeRecorder rc("PreloadTableRequest"); + std::string hdr = "PreloadTableRequest(table=" + table_name_ + ")"; + TimeRecorder rc(hdr); // step 1: check arguments auto status = ValidationUtil::ValidateTableName(table_name_); diff --git a/core/src/server/grpc_impl/request/ShowPartitionsRequest.cpp b/core/src/server/grpc_impl/request/ShowPartitionsRequest.cpp index 32fa0672c5..0d0809b171 100644 --- a/core/src/server/grpc_impl/request/ShowPartitionsRequest.cpp +++ b/core/src/server/grpc_impl/request/ShowPartitionsRequest.cpp @@ -40,6 +40,9 @@ ShowPartitionsRequest::Create(const std::string& table_name, ::milvus::grpc::Par Status ShowPartitionsRequest::OnExecute() { + std::string hdr = "ShowPartitionsRequest(table=" + table_name_ + ")"; + TimeRecorder rc(hdr); + auto status = ValidationUtil::ValidateTableName(table_name_); if (!status.ok()) { return status; diff --git a/core/src/server/grpc_impl/request/ShowTablesRequest.cpp b/core/src/server/grpc_impl/request/ShowTablesRequest.cpp index 90dbad981f..404e08197e 100644 --- a/core/src/server/grpc_impl/request/ShowTablesRequest.cpp +++ b/core/src/server/grpc_impl/request/ShowTablesRequest.cpp @@ -38,6 +38,8 @@ ShowTablesRequest::Create(::milvus::grpc::TableNameList* table_name_list) { Status ShowTablesRequest::OnExecute() { + TimeRecorder rc("ShowTablesRequest"); + std::vector schema_array; auto statuts = DBWrapper::DB()->AllTables(schema_array); if (!statuts.ok()) { From 39d6b633376f0d8d72972ffbe562eec02130b944 Mon Sep 17 00:00:00 2001 From: jielinxu <52057195+jielinxu@users.noreply.github.com> Date: Mon, 2 Dec 2019 15:24:15 +0800 Subject: [PATCH 34/44] [skip ci] Replace zip with tar --- install.md | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/install.md b/install.md index 35884ffd52..ae9c262f8b 100644 --- a/install.md +++ b/install.md @@ -116,15 +116,13 @@ Download Milvus source code: ```shell $ cd /home -$ wget https://github.com/milvus-io/milvus/archive/0.6.0.zip +$ wget https://github.com/milvus-io/milvus/archive/0.6.0.tar.gz ``` -Unzip the source package: +Extract the source package: ```shell -$ apt-get update -$ apt-get install unzip -$ unzip ./0.6.0.zip +$ tar xvf ./v0.6.0.tar.gz ``` The source code is extracted into a folder called `milvus-0.6.0`. To enter its core directory: From 52b58366dc128b5b9f8b42db7adc0cca9892ee95 Mon Sep 17 00:00:00 2001 From: zhenwu Date: Mon, 2 Dec 2019 15:30:31 +0800 Subject: [PATCH 35/44] assert failed if create index with pq on ip_table on gpu, assert pass on cpu --- tests/milvus_python_test/test_index.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/milvus_python_test/test_index.py b/tests/milvus_python_test/test_index.py index 3a3ffc3857..9dc919424e 100644 --- a/tests/milvus_python_test/test_index.py +++ b/tests/milvus_python_test/test_index.py @@ -972,9 +972,11 @@ class TestIndexIP: expected: return code 0, and default index param ''' index_params = get_simple_index_params + status, mode = connect._cmd("mode") + assert status.OK() # status, ids = connect.add_vectors(ip_table, vectors) status = connect.create_index(ip_table, index_params) - if index_params["index_type"] == IndexType.IVF_PQ: + if str(mode) == "GPU" and index_params["index_type"] == IndexType.IVF_PQ: assert not status.OK() else: assert status.OK() From 3df535e11e9447d35f5c3017affebc14dd195249 Mon Sep 17 00:00:00 2001 From: groot Date: Mon, 2 Dec 2019 15:39:16 +0800 Subject: [PATCH 36/44] #649 Typo partiton should be partition --- CHANGELOG.md | 1 + core/src/db/DB.h | 2 +- core/src/db/DBImpl.cpp | 52 +++++++++---------- core/src/db/DBImpl.h | 2 +- core/src/db/meta/Meta.h | 2 +- core/src/db/meta/MySQLMetaImpl.cpp | 4 +- core/src/db/meta/MySQLMetaImpl.h | 2 +- core/src/db/meta/SqliteMetaImpl.cpp | 4 +- core/src/db/meta/SqliteMetaImpl.h | 2 +- .../sdk/examples/partition/src/ClientTest.cpp | 16 +++--- .../sdk/examples/simple/src/ClientTest.cpp | 8 +-- core/src/sdk/examples/utils/Utils.cpp | 4 +- core/src/sdk/examples/utils/Utils.h | 2 +- core/src/sdk/grpc/ClientProxy.cpp | 4 +- core/src/sdk/grpc/ClientProxy.h | 2 +- core/src/sdk/include/MilvusApi.h | 2 +- core/src/sdk/interface/ConnectionImpl.cpp | 4 +- core/src/sdk/interface/ConnectionImpl.h | 2 +- core/unittest/db/test_db.cpp | 8 +-- core/unittest/db/test_db_mysql.cpp | 8 +-- 20 files changed, 66 insertions(+), 65 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9262ae01aa..0e3284f9f6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -45,6 +45,7 @@ Please mark all change in change log and use the ticket from JIRA. - \#602 - Optimizer specify wrong gpu_id - \#606 - No log generated during building index with CPU - \#631 - FAISS isn't compiled with O3 option +- \#649 - Typo "partiton" should be "partition" ## Feature - \#12 - Pure CPU version for Milvus diff --git a/core/src/db/DB.h b/core/src/db/DB.h index 09bbd4af45..154686be60 100644 --- a/core/src/db/DB.h +++ b/core/src/db/DB.h @@ -80,7 +80,7 @@ class DB { DropPartitionByTag(const std::string& table_id, const std::string& partition_tag) = 0; virtual Status - ShowPartitions(const std::string& table_id, std::vector& partiton_schema_array) = 0; + ShowPartitions(const std::string& table_id, std::vector& partition_schema_array) = 0; virtual Status InsertVectors(const std::string& table_id, const std::string& partition_tag, uint64_t n, const float* vectors, diff --git a/core/src/db/DBImpl.cpp b/core/src/db/DBImpl.cpp index 8f8516770f..88689857ca 100644 --- a/core/src/db/DBImpl.cpp +++ b/core/src/db/DBImpl.cpp @@ -190,9 +190,9 @@ DBImpl::PreloadTable(const std::string& table_id) { } // step 2: get files from partition tables - std::vector partiton_array; - status = meta_ptr_->ShowPartitions(table_id, partiton_array); - for (auto& schema : partiton_array) { + std::vector partition_array; + status = meta_ptr_->ShowPartitions(table_id, partition_array); + for (auto& schema : partition_array) { status = GetFilesToSearch(schema.table_id_, ids, dates, files_array); } @@ -296,12 +296,12 @@ DBImpl::DropPartitionByTag(const std::string& table_id, const std::string& parti } Status -DBImpl::ShowPartitions(const std::string& table_id, std::vector& partiton_schema_array) { +DBImpl::ShowPartitions(const std::string& table_id, std::vector& partition_schema_array) { if (shutting_down_.load(std::memory_order_acquire)) { return SHUTDOWN_ERROR; } - return meta_ptr_->ShowPartitions(table_id, partiton_schema_array); + return meta_ptr_->ShowPartitions(table_id, partition_schema_array); } Status @@ -427,9 +427,9 @@ DBImpl::Query(const std::string& table_id, const std::vector& parti return status; } - std::vector partiton_array; - status = meta_ptr_->ShowPartitions(table_id, partiton_array); - for (auto& schema : partiton_array) { + std::vector partition_array; + status = meta_ptr_->ShowPartitions(table_id, partition_array); + for (auto& schema : partition_array) { status = GetFilesToSearch(schema.table_id_, ids, dates, files_array); } } else { @@ -917,15 +917,15 @@ DBImpl::GetFilesToSearch(const std::string& table_id, const std::vector& Status DBImpl::GetPartitionsByTags(const std::string& table_id, const std::vector& partition_tags, std::set& partition_name_array) { - std::vector partiton_array; - auto status = meta_ptr_->ShowPartitions(table_id, partiton_array); + std::vector partition_array; + auto status = meta_ptr_->ShowPartitions(table_id, partition_array); for (auto& tag : partition_tags) { // trim side-blank of tag, only compare valid characters // for example: " ab cd " is treated as "ab cd" std::string valid_tag = tag; server::StringHelpFunctions::TrimStringBlank(valid_tag); - for (auto& schema : partiton_array) { + for (auto& schema : partition_array) { if (server::StringHelpFunctions::IsRegexMatch(schema.partition_tag_, valid_tag)) { partition_name_array.insert(schema.table_id_); } @@ -955,9 +955,9 @@ DBImpl::DropTableRecursively(const std::string& table_id, const meta::DatesT& da status = meta_ptr_->DropDataByDate(table_id, dates); } - std::vector partiton_array; - status = meta_ptr_->ShowPartitions(table_id, partiton_array); - for (auto& schema : partiton_array) { + std::vector partition_array; + status = meta_ptr_->ShowPartitions(table_id, partition_array); + for (auto& schema : partition_array) { status = DropTableRecursively(schema.table_id_, dates); if (!status.ok()) { return status; @@ -977,9 +977,9 @@ DBImpl::UpdateTableIndexRecursively(const std::string& table_id, const TableInde return status; } - std::vector partiton_array; - status = meta_ptr_->ShowPartitions(table_id, partiton_array); - for (auto& schema : partiton_array) { + std::vector partition_array; + status = meta_ptr_->ShowPartitions(table_id, partition_array); + for (auto& schema : partition_array) { status = UpdateTableIndexRecursively(schema.table_id_, index); if (!status.ok()) { return status; @@ -1028,9 +1028,9 @@ DBImpl::BuildTableIndexRecursively(const std::string& table_id, const TableIndex } // build index for partition - std::vector partiton_array; - status = meta_ptr_->ShowPartitions(table_id, partiton_array); - for (auto& schema : partiton_array) { + std::vector partition_array; + status = meta_ptr_->ShowPartitions(table_id, partition_array); + for (auto& schema : partition_array) { status = BuildTableIndexRecursively(schema.table_id_, index); if (!status.ok()) { return status; @@ -1060,9 +1060,9 @@ DBImpl::DropTableIndexRecursively(const std::string& table_id) { } // drop partition index - std::vector partiton_array; - status = meta_ptr_->ShowPartitions(table_id, partiton_array); - for (auto& schema : partiton_array) { + std::vector partition_array; + status = meta_ptr_->ShowPartitions(table_id, partition_array); + for (auto& schema : partition_array) { status = DropTableIndexRecursively(schema.table_id_); if (!status.ok()) { return status; @@ -1081,9 +1081,9 @@ DBImpl::GetTableRowCountRecursively(const std::string& table_id, uint64_t& row_c } // get partition row count - std::vector partiton_array; - status = meta_ptr_->ShowPartitions(table_id, partiton_array); - for (auto& schema : partiton_array) { + std::vector partition_array; + status = meta_ptr_->ShowPartitions(table_id, partition_array); + for (auto& schema : partition_array) { uint64_t partition_row_count = 0; status = GetTableRowCountRecursively(schema.table_id_, partition_row_count); if (!status.ok()) { diff --git a/core/src/db/DBImpl.h b/core/src/db/DBImpl.h index 5091160d63..a79f26f844 100644 --- a/core/src/db/DBImpl.h +++ b/core/src/db/DBImpl.h @@ -89,7 +89,7 @@ class DBImpl : public DB { DropPartitionByTag(const std::string& table_id, const std::string& partition_tag) override; Status - ShowPartitions(const std::string& table_id, std::vector& partiton_schema_array) override; + ShowPartitions(const std::string& table_id, std::vector& partition_schema_array) override; Status InsertVectors(const std::string& table_id, const std::string& partition_tag, uint64_t n, const float* vectors, diff --git a/core/src/db/meta/Meta.h b/core/src/db/meta/Meta.h index 1929414fdc..da438a2546 100644 --- a/core/src/db/meta/Meta.h +++ b/core/src/db/meta/Meta.h @@ -100,7 +100,7 @@ class Meta { DropPartition(const std::string& partition_name) = 0; virtual Status - ShowPartitions(const std::string& table_name, std::vector& partiton_schema_array) = 0; + ShowPartitions(const std::string& table_name, std::vector& partition_schema_array) = 0; virtual Status GetPartitionName(const std::string& table_name, const std::string& tag, std::string& partition_name) = 0; diff --git a/core/src/db/meta/MySQLMetaImpl.cpp b/core/src/db/meta/MySQLMetaImpl.cpp index da91585b92..d4b5eee442 100644 --- a/core/src/db/meta/MySQLMetaImpl.cpp +++ b/core/src/db/meta/MySQLMetaImpl.cpp @@ -1212,7 +1212,7 @@ MySQLMetaImpl::DropPartition(const std::string& partition_name) { } Status -MySQLMetaImpl::ShowPartitions(const std::string& table_id, std::vector& partiton_schema_array) { +MySQLMetaImpl::ShowPartitions(const std::string& table_id, std::vector& partition_schema_array) { try { server::MetricCollector metric; mysqlpp::StoreQueryResult res; @@ -1236,7 +1236,7 @@ MySQLMetaImpl::ShowPartitions(const std::string& table_id, std::vector& partiton_schema_array) override; + ShowPartitions(const std::string& table_id, std::vector& partition_schema_array) override; Status GetPartitionName(const std::string& table_id, const std::string& tag, std::string& partition_name) override; diff --git a/core/src/db/meta/SqliteMetaImpl.cpp b/core/src/db/meta/SqliteMetaImpl.cpp index 9e323a4fac..cb6797ae47 100644 --- a/core/src/db/meta/SqliteMetaImpl.cpp +++ b/core/src/db/meta/SqliteMetaImpl.cpp @@ -804,7 +804,7 @@ SqliteMetaImpl::DropPartition(const std::string& partition_name) { } Status -SqliteMetaImpl::ShowPartitions(const std::string& table_id, std::vector& partiton_schema_array) { +SqliteMetaImpl::ShowPartitions(const std::string& table_id, std::vector& partition_schema_array) { try { server::MetricCollector metric; @@ -816,7 +816,7 @@ SqliteMetaImpl::ShowPartitions(const std::string& table_id, std::vector& partiton_schema_array) override; + ShowPartitions(const std::string& table_id, std::vector& partition_schema_array) override; Status GetPartitionName(const std::string& table_id, const std::string& tag, std::string& partition_name) override; diff --git a/core/src/sdk/examples/partition/src/ClientTest.cpp b/core/src/sdk/examples/partition/src/ClientTest.cpp index ecbf0a80e7..7a4bd43cb0 100644 --- a/core/src/sdk/examples/partition/src/ClientTest.cpp +++ b/core/src/sdk/examples/partition/src/ClientTest.cpp @@ -141,18 +141,18 @@ ClientTest::Test(const std::string& address, const std::string& port) { { // search vectors std::cout << "Search in correct partition" << std::endl; - std::vector partiton_tags = {std::to_string(TARGET_PARTITION)}; + std::vector partition_tags = {std::to_string(TARGET_PARTITION)}; milvus::TopKQueryResult topk_query_result; - milvus_sdk::Utils::DoSearch(conn, TABLE_NAME, partiton_tags, TOP_K, NPROBE, search_record_array, + milvus_sdk::Utils::DoSearch(conn, TABLE_NAME, partition_tags, TOP_K, NPROBE, search_record_array, topk_query_result); std::cout << "Search in wrong partition" << std::endl; - partiton_tags = {"0"}; - milvus_sdk::Utils::DoSearch(conn, TABLE_NAME, partiton_tags, TOP_K, NPROBE, search_record_array, + partition_tags = {"0"}; + milvus_sdk::Utils::DoSearch(conn, TABLE_NAME, partition_tags, TOP_K, NPROBE, search_record_array, topk_query_result); std::cout << "Search by regex matched partition tag" << std::endl; - partiton_tags = {"\\d"}; - milvus_sdk::Utils::DoSearch(conn, TABLE_NAME, partiton_tags, TOP_K, NPROBE, search_record_array, + partition_tags = {"\\d"}; + milvus_sdk::Utils::DoSearch(conn, TABLE_NAME, partition_tags, TOP_K, NPROBE, search_record_array, topk_query_result); } @@ -191,9 +191,9 @@ ClientTest::Test(const std::string& address, const std::string& port) { { // search vectors std::cout << "Search in whole table" << std::endl; - std::vector partiton_tags; + std::vector partition_tags; milvus::TopKQueryResult topk_query_result; - milvus_sdk::Utils::DoSearch(conn, TABLE_NAME, partiton_tags, TOP_K, NPROBE, search_record_array, + milvus_sdk::Utils::DoSearch(conn, TABLE_NAME, partition_tags, TOP_K, NPROBE, search_record_array, topk_query_result); } diff --git a/core/src/sdk/examples/simple/src/ClientTest.cpp b/core/src/sdk/examples/simple/src/ClientTest.cpp index 016c9eceac..14bd059f93 100644 --- a/core/src/sdk/examples/simple/src/ClientTest.cpp +++ b/core/src/sdk/examples/simple/src/ClientTest.cpp @@ -143,9 +143,9 @@ ClientTest::Test(const std::string& address, const std::string& port) { milvus_sdk::Utils::Sleep(3); { // search vectors - std::vector partiton_tags; + std::vector partition_tags; milvus::TopKQueryResult topk_query_result; - milvus_sdk::Utils::DoSearch(conn, TABLE_NAME, partiton_tags, TOP_K, NPROBE, search_record_array, + milvus_sdk::Utils::DoSearch(conn, TABLE_NAME, partition_tags, TOP_K, NPROBE, search_record_array, topk_query_result); } @@ -169,9 +169,9 @@ ClientTest::Test(const std::string& address, const std::string& port) { } { // search vectors - std::vector partiton_tags; + std::vector partition_tags; milvus::TopKQueryResult topk_query_result; - milvus_sdk::Utils::DoSearch(conn, TABLE_NAME, partiton_tags, TOP_K, NPROBE, search_record_array, + milvus_sdk::Utils::DoSearch(conn, TABLE_NAME, partition_tags, TOP_K, NPROBE, search_record_array, topk_query_result); } diff --git a/core/src/sdk/examples/utils/Utils.cpp b/core/src/sdk/examples/utils/Utils.cpp index d3bf9eec25..b45ebbea37 100644 --- a/core/src/sdk/examples/utils/Utils.cpp +++ b/core/src/sdk/examples/utils/Utils.cpp @@ -202,7 +202,7 @@ Utils::CheckSearchResult(const std::vector void Utils::DoSearch(std::shared_ptr conn, const std::string& table_name, - const std::vector& partiton_tags, int64_t top_k, int64_t nprobe, + const std::vector& partition_tags, int64_t top_k, int64_t nprobe, const std::vector>& search_record_array, milvus::TopKQueryResult& topk_query_result) { topk_query_result.clear(); @@ -222,7 +222,7 @@ Utils::DoSearch(std::shared_ptr conn, const std::string& tab BLOCK_SPLITER milvus_sdk::TimeRecorder rc("search"); milvus::Status stat = - conn->Search(table_name, partiton_tags, record_array, query_range_array, top_k, nprobe, topk_query_result); + conn->Search(table_name, partition_tags, record_array, query_range_array, top_k, nprobe, topk_query_result); std::cout << "SearchVector function call status: " << stat.message() << std::endl; BLOCK_SPLITER } diff --git a/core/src/sdk/examples/utils/Utils.h b/core/src/sdk/examples/utils/Utils.h index 8066a24ff4..31880d3c41 100644 --- a/core/src/sdk/examples/utils/Utils.h +++ b/core/src/sdk/examples/utils/Utils.h @@ -69,7 +69,7 @@ class Utils { static void DoSearch(std::shared_ptr conn, const std::string& table_name, - const std::vector& partiton_tags, int64_t top_k, int64_t nprobe, + const std::vector& partition_tags, int64_t top_k, int64_t nprobe, const std::vector>& search_record_array, milvus::TopKQueryResult& topk_query_result); }; diff --git a/core/src/sdk/grpc/ClientProxy.cpp b/core/src/sdk/grpc/ClientProxy.cpp index fd19281343..c6a087cfe4 100644 --- a/core/src/sdk/grpc/ClientProxy.cpp +++ b/core/src/sdk/grpc/ClientProxy.cpp @@ -221,7 +221,7 @@ ClientProxy::Insert(const std::string& table_name, const std::string& partition_ } Status -ClientProxy::Search(const std::string& table_name, const std::vector& partiton_tags, +ClientProxy::Search(const std::string& table_name, const std::vector& partition_tags, const std::vector& query_record_array, const std::vector& query_range_array, int64_t topk, int64_t nprobe, TopKQueryResult& topk_query_result) { try { @@ -230,7 +230,7 @@ ClientProxy::Search(const std::string& table_name, const std::vector& id_array) override; Status - Search(const std::string& table_name, const std::vector& partiton_tags, + Search(const std::string& table_name, const std::vector& partition_tags, const std::vector& query_record_array, const std::vector& query_range_array, int64_t topk, int64_t nprobe, TopKQueryResult& topk_query_result) override; diff --git a/core/src/sdk/include/MilvusApi.h b/core/src/sdk/include/MilvusApi.h index 0937ce2f8e..e631be36be 100644 --- a/core/src/sdk/include/MilvusApi.h +++ b/core/src/sdk/include/MilvusApi.h @@ -273,7 +273,7 @@ class Connection { * @return Indicate if query is successful. */ virtual Status - Search(const std::string& table_name, const std::vector& partiton_tags, + Search(const std::string& table_name, const std::vector& partition_tags, const std::vector& query_record_array, const std::vector& query_range_array, int64_t topk, int64_t nprobe, TopKQueryResult& topk_query_result) = 0; diff --git a/core/src/sdk/interface/ConnectionImpl.cpp b/core/src/sdk/interface/ConnectionImpl.cpp index 448c011c1f..3b544c00c0 100644 --- a/core/src/sdk/interface/ConnectionImpl.cpp +++ b/core/src/sdk/interface/ConnectionImpl.cpp @@ -89,10 +89,10 @@ ConnectionImpl::Insert(const std::string& table_name, const std::string& partiti } Status -ConnectionImpl::Search(const std::string& table_name, const std::vector& partiton_tags, +ConnectionImpl::Search(const std::string& table_name, const std::vector& partition_tags, const std::vector& query_record_array, const std::vector& query_range_array, int64_t topk, int64_t nprobe, TopKQueryResult& topk_query_result) { - return client_proxy_->Search(table_name, partiton_tags, query_record_array, query_range_array, topk, nprobe, + return client_proxy_->Search(table_name, partition_tags, query_record_array, query_range_array, topk, nprobe, topk_query_result); } diff --git a/core/src/sdk/interface/ConnectionImpl.h b/core/src/sdk/interface/ConnectionImpl.h index 126e59c375..e22f8d33d8 100644 --- a/core/src/sdk/interface/ConnectionImpl.h +++ b/core/src/sdk/interface/ConnectionImpl.h @@ -60,7 +60,7 @@ class ConnectionImpl : public Connection { std::vector& id_array) override; Status - Search(const std::string& table_name, const std::vector& partiton_tags, + Search(const std::string& table_name, const std::vector& partition_tags, const std::vector& query_record_array, const std::vector& query_range_array, int64_t topk, int64_t nprobe, TopKQueryResult& topk_query_result) override; diff --git a/core/unittest/db/test_db.cpp b/core/unittest/db/test_db.cpp index 0a47ac1b9b..768687a8a3 100644 --- a/core/unittest/db/test_db.cpp +++ b/core/unittest/db/test_db.cpp @@ -536,12 +536,12 @@ TEST_F(DBTest, PARTITION_TEST) { stat = db_->CreatePartition(table_name, "", "0"); ASSERT_FALSE(stat.ok()); - std::vector partiton_schema_array; - stat = db_->ShowPartitions(table_name, partiton_schema_array); + std::vector partition_schema_array; + stat = db_->ShowPartitions(table_name, partition_schema_array); ASSERT_TRUE(stat.ok()); - ASSERT_EQ(partiton_schema_array.size(), PARTITION_COUNT); + ASSERT_EQ(partition_schema_array.size(), PARTITION_COUNT); for (int64_t i = 0; i < PARTITION_COUNT; i++) { - ASSERT_EQ(partiton_schema_array[i].table_id_, table_name + "_" + std::to_string(i)); + ASSERT_EQ(partition_schema_array[i].table_id_, table_name + "_" + std::to_string(i)); } { // build index diff --git a/core/unittest/db/test_db_mysql.cpp b/core/unittest/db/test_db_mysql.cpp index 93eea2fd27..094e824142 100644 --- a/core/unittest/db/test_db_mysql.cpp +++ b/core/unittest/db/test_db_mysql.cpp @@ -323,12 +323,12 @@ TEST_F(MySqlDBTest, PARTITION_TEST) { stat = db_->CreatePartition(table_name, "", "0"); ASSERT_FALSE(stat.ok()); - std::vector partiton_schema_array; - stat = db_->ShowPartitions(table_name, partiton_schema_array); + std::vector partition_schema_array; + stat = db_->ShowPartitions(table_name, partition_schema_array); ASSERT_TRUE(stat.ok()); - ASSERT_EQ(partiton_schema_array.size(), PARTITION_COUNT); + ASSERT_EQ(partition_schema_array.size(), PARTITION_COUNT); for (int64_t i = 0; i < PARTITION_COUNT; i++) { - ASSERT_EQ(partiton_schema_array[i].table_id_, table_name + "_" + std::to_string(i)); + ASSERT_EQ(partition_schema_array[i].table_id_, table_name + "_" + std::to_string(i)); } { // build index From 6b52973b0491621fa1860b7b1766aadbfa0fc2b0 Mon Sep 17 00:00:00 2001 From: groot Date: Mon, 2 Dec 2019 19:51:33 +0800 Subject: [PATCH 37/44] #654 Random crash when frequently insert vector one by one --- CHANGELOG.md | 1 + core/src/db/insert/MemManagerImpl.cpp | 2 ++ 2 files changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0e3284f9f6..1e978754cd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -46,6 +46,7 @@ Please mark all change in change log and use the ticket from JIRA. - \#606 - No log generated during building index with CPU - \#631 - FAISS isn't compiled with O3 option - \#649 - Typo "partiton" should be "partition" +- \#654 - Random crash when frequently insert vector one by one ## Feature - \#12 - Pure CPU version for Milvus diff --git a/core/src/db/insert/MemManagerImpl.cpp b/core/src/db/insert/MemManagerImpl.cpp index 69c3397eb9..c7ca3b3c33 100644 --- a/core/src/db/insert/MemManagerImpl.cpp +++ b/core/src/db/insert/MemManagerImpl.cpp @@ -116,6 +116,7 @@ MemManagerImpl::EraseMemVector(const std::string& table_id) { size_t MemManagerImpl::GetCurrentMutableMem() { size_t total_mem = 0; + std::unique_lock lock(mutex_); for (auto& kv : mem_id_map_) { auto memTable = kv.second; total_mem += memTable->GetCurrentMem(); @@ -126,6 +127,7 @@ MemManagerImpl::GetCurrentMutableMem() { size_t MemManagerImpl::GetCurrentImmutableMem() { size_t total_mem = 0; + std::unique_lock lock(serialization_mtx_); for (auto& mem_table : immu_mem_list_) { total_mem += mem_table->GetCurrentMem(); } From 1da5ad48a1f01f888d320df843cd4600703829ae Mon Sep 17 00:00:00 2001 From: groot Date: Tue, 3 Dec 2019 10:38:35 +0800 Subject: [PATCH 38/44] modify insert log --- core/src/server/grpc_impl/request/InsertRequest.cpp | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/core/src/server/grpc_impl/request/InsertRequest.cpp b/core/src/server/grpc_impl/request/InsertRequest.cpp index df65c679ed..c178822468 100644 --- a/core/src/server/grpc_impl/request/InsertRequest.cpp +++ b/core/src/server/grpc_impl/request/InsertRequest.cpp @@ -122,8 +122,6 @@ InsertRequest::OnExecute() { table_info.dimension_ * sizeof(float)); } - rc.ElapseFromBegin("prepare vectors data"); - // step 5: insert vectors auto vec_count = static_cast(insert_param_->row_record_array_size()); std::vector vec_ids(insert_param_->row_id_array_size(), 0); @@ -133,9 +131,9 @@ InsertRequest::OnExecute() { memcpy(target_data, src_data, static_cast(sizeof(int64_t) * insert_param_->row_id_array_size())); } + rc.RecordSection("prepare vectors data"); status = DBWrapper::DB()->InsertVectors(insert_param_->table_name(), insert_param_->partition_tag(), vec_count, vec_f.data(), vec_ids); - rc.ElapseFromBegin("add vectors to engine"); if (!status.ok()) { return status; } From 6c14297a2920d4f6669948ed26f63f15a06cba1a Mon Sep 17 00:00:00 2001 From: Lizhou Gao Date: Tue, 3 Dec 2019 10:49:00 +0800 Subject: [PATCH 39/44] use better names for timing MACROs milvus-io#556 --- core/src/db/DBImpl.cpp | 2 +- core/src/db/meta/MetaConsts.h | 8 ++++---- core/src/db/meta/MySQLMetaImpl.cpp | 2 +- core/unittest/db/test_meta.cpp | 2 +- core/unittest/db/test_meta_mysql.cpp | 2 +- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/core/src/db/DBImpl.cpp b/core/src/db/DBImpl.cpp index 88689857ca..0f118e98df 100644 --- a/core/src/db/DBImpl.cpp +++ b/core/src/db/DBImpl.cpp @@ -800,7 +800,7 @@ DBImpl::BackgroundCompaction(std::set table_ids) { { uint64_t ttl = 10 * meta::SECOND; // default: file will be hard-deleted few seconds after soft-deleted if (options_.mode_ == DBOptions::MODE::CLUSTER_WRITABLE) { - ttl = meta::H_SEC; + ttl = meta::HOUR; } meta_ptr_->CleanUpFilesWithTTL(ttl, &ongoing_files_checker_); diff --git a/core/src/db/meta/MetaConsts.h b/core/src/db/meta/MetaConsts.h index 0c77dc2599..b85c693707 100644 --- a/core/src/db/meta/MetaConsts.h +++ b/core/src/db/meta/MetaConsts.h @@ -27,10 +27,10 @@ const size_t US_PS = 1000 * MS_PS; const size_t NS_PS = 1000 * US_PS; const size_t SECOND = 1UL; -const size_t M_SEC = 60 * SECOND; -const size_t H_SEC = 60 * M_SEC; -const size_t D_SEC = 24 * H_SEC; -const size_t W_SEC = 7 * D_SEC; +const size_t MINUTE = 60 * SECOND; +const size_t HOUR = 60 * MINUTE; +const size_t DAY = 24 * HOUR; +const size_t WEEK = 7 * DAY; // This value is to ignore small raw files when building index. // The reason is: diff --git a/core/src/db/meta/MySQLMetaImpl.cpp b/core/src/db/meta/MySQLMetaImpl.cpp index d4b5eee442..5072df9cad 100644 --- a/core/src/db/meta/MySQLMetaImpl.cpp +++ b/core/src/db/meta/MySQLMetaImpl.cpp @@ -1664,7 +1664,7 @@ MySQLMetaImpl::Archive() { auto& criteria = kv.first; auto& limit = kv.second; if (criteria == engine::ARCHIVE_CONF_DAYS) { - size_t usecs = limit * D_SEC * US_PS; + size_t usecs = limit * DAY * US_PS; int64_t now = utils::GetMicroSecTimeStamp(); try { diff --git a/core/unittest/db/test_meta.cpp b/core/unittest/db/test_meta.cpp index b89c73c296..bb8641fac5 100644 --- a/core/unittest/db/test_meta.cpp +++ b/core/unittest/db/test_meta.cpp @@ -141,7 +141,7 @@ TEST_F(MetaTest, ARCHIVE_TEST_DAYS) { status = impl.CreateTableFile(table_file); table_file.file_type_ = milvus::engine::meta::TableFileSchema::NEW; int day = rand_r(&seed) % (days_num * 2); - table_file.created_on_ = ts - day * milvus::engine::meta::D_SEC * milvus::engine::meta::US_PS - 10000; + table_file.created_on_ = ts - day * milvus::engine::meta::DAY * milvus::engine::meta::US_PS - 10000; status = impl.UpdateTableFile(table_file); files.push_back(table_file); days.push_back(day); diff --git a/core/unittest/db/test_meta_mysql.cpp b/core/unittest/db/test_meta_mysql.cpp index 9a52a01b7b..f77defe2f6 100644 --- a/core/unittest/db/test_meta_mysql.cpp +++ b/core/unittest/db/test_meta_mysql.cpp @@ -145,7 +145,7 @@ TEST_F(MySqlMetaTest, ARCHIVE_TEST_DAYS) { status = impl.CreateTableFile(table_file); table_file.file_type_ = milvus::engine::meta::TableFileSchema::NEW; int day = rand_r(&seed) % (days_num * 2); - table_file.created_on_ = ts - day * milvus::engine::meta::D_SEC * milvus::engine::meta::US_PS - 10000; + table_file.created_on_ = ts - day * milvus::engine::meta::DAY * milvus::engine::meta::US_PS - 10000; status = impl.UpdateTableFile(table_file); files.push_back(table_file); days.push_back(day); From 27f6166d50357ba63908e611c9d9dd7a65acc104 Mon Sep 17 00:00:00 2001 From: groot Date: Tue, 3 Dec 2019 10:59:30 +0800 Subject: [PATCH 40/44] fix typo --- core/src/db/DBImpl.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/db/DBImpl.cpp b/core/src/db/DBImpl.cpp index 88689857ca..b51ea698b9 100644 --- a/core/src/db/DBImpl.cpp +++ b/core/src/db/DBImpl.cpp @@ -52,7 +52,7 @@ constexpr uint64_t METRIC_ACTION_INTERVAL = 1; constexpr uint64_t COMPACT_ACTION_INTERVAL = 1; constexpr uint64_t INDEX_ACTION_INTERVAL = 1; -static const Status SHUTDOWN_ERROR = Status(DB_ERROR, "Milsvus server is shutdown!"); +static const Status SHUTDOWN_ERROR = Status(DB_ERROR, "Milvus server is shutdown!"); void TraverseFiles(const meta::DatePartionedTableFilesSchema& date_files, meta::TableFilesSchema& files_array) { From f464b064a854e0ae244c460a5d3a5188c75fde32 Mon Sep 17 00:00:00 2001 From: jielinxu <52057195+jielinxu@users.noreply.github.com> Date: Tue, 3 Dec 2019 11:23:44 +0800 Subject: [PATCH 41/44] [skip ci]Update install.md --- install.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install.md b/install.md index ae9c262f8b..e95153a3a1 100644 --- a/install.md +++ b/install.md @@ -77,7 +77,7 @@ $ ./stop_server.sh ## Compile Milvus on Docker -With this dockerfile, you should be able to compile Milvus on any Linux platform that run Docker. To build a GPU supported Milvus, you neeed to install [NVIDIA Docker](https://github.com/NVIDIA/nvidia-docker/) first. +With the following Docker images, you should be able to compile Milvus on any Linux platform that run Docker. To build a GPU supported Milvus, you neeed to install [NVIDIA Docker](https://github.com/NVIDIA/nvidia-docker/) first. ### Step 1 Pull Milvus Docker images From a320846606f323de0ead7ae9c9b2b55247470c48 Mon Sep 17 00:00:00 2001 From: JinHai-CN Date: Tue, 3 Dec 2019 11:27:49 +0800 Subject: [PATCH 42/44] Add a comment on readonly invertedlist and pin memory --- core/src/index/thirdparty/faiss/InvertedLists.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/core/src/index/thirdparty/faiss/InvertedLists.cpp b/core/src/index/thirdparty/faiss/InvertedLists.cpp index a438020fe9..86a9d6c711 100644 --- a/core/src/index/thirdparty/faiss/InvertedLists.cpp +++ b/core/src/index/thirdparty/faiss/InvertedLists.cpp @@ -20,6 +20,12 @@ namespace faiss { +/* + * Use pin memory to build Readonly Inverted list will accelerate cuda memory copy, but it will downgrade cpu ivf search + * performance. read only inverted list structure will also make ivf search performance not stable. ISSUE 500 mention + * this problem. Best performance is the original inverted list with non pin memory. + */ + PageLockMemory::PageLockMemory(size_t size) : nbytes(size) { CUDA_VERIFY(cudaHostAlloc(&data, size, 0)); } From 7a27e9d455c8319592dab50c0330e4b332a26955 Mon Sep 17 00:00:00 2001 From: groot Date: Tue, 3 Dec 2019 11:57:52 +0800 Subject: [PATCH 43/44] modify log --- core/src/server/grpc_impl/request/CmdRequest.cpp | 2 +- core/src/server/grpc_impl/request/CountTableRequest.cpp | 4 +--- core/src/server/grpc_impl/request/CreateIndexRequest.cpp | 4 +--- core/src/server/grpc_impl/request/CreatePartitionRequest.cpp | 4 +--- core/src/server/grpc_impl/request/CreateTableRequest.cpp | 4 +--- core/src/server/grpc_impl/request/DeleteByDateRequest.cpp | 4 ++-- core/src/server/grpc_impl/request/DescribeIndexRequest.cpp | 4 +--- core/src/server/grpc_impl/request/DescribeTableRequest.cpp | 4 +--- core/src/server/grpc_impl/request/DropIndexRequest.cpp | 4 +--- core/src/server/grpc_impl/request/DropPartitionRequest.cpp | 2 +- core/src/server/grpc_impl/request/DropTableRequest.cpp | 2 +- core/src/server/grpc_impl/request/HasTableRequest.cpp | 4 +--- core/src/server/grpc_impl/request/PreloadTableRequest.cpp | 4 +--- core/src/server/grpc_impl/request/ShowPartitionsRequest.cpp | 2 +- core/src/server/grpc_impl/request/ShowTablesRequest.cpp | 2 +- 15 files changed, 16 insertions(+), 34 deletions(-) diff --git a/core/src/server/grpc_impl/request/CmdRequest.cpp b/core/src/server/grpc_impl/request/CmdRequest.cpp index 4af9db03ef..67e41acd02 100644 --- a/core/src/server/grpc_impl/request/CmdRequest.cpp +++ b/core/src/server/grpc_impl/request/CmdRequest.cpp @@ -38,7 +38,7 @@ CmdRequest::Create(const std::string& cmd, std::string& result) { Status CmdRequest::OnExecute() { std::string hdr = "CmdRequest(cmd=" + cmd_ + ")"; - TimeRecorder rc(hdr); + TimeRecorderAuto rc(hdr); if (cmd_ == "version") { result_ = MILVUS_VERSION; diff --git a/core/src/server/grpc_impl/request/CountTableRequest.cpp b/core/src/server/grpc_impl/request/CountTableRequest.cpp index b90a33bf61..b8bad6ca74 100644 --- a/core/src/server/grpc_impl/request/CountTableRequest.cpp +++ b/core/src/server/grpc_impl/request/CountTableRequest.cpp @@ -40,7 +40,7 @@ Status CountTableRequest::OnExecute() { try { std::string hdr = "CountTableRequest(table=" + table_name_ + ")"; - TimeRecorder rc(hdr); + TimeRecorderAuto rc(hdr); // step 1: check arguments auto status = ValidationUtil::ValidateTableName(table_name_); @@ -60,8 +60,6 @@ CountTableRequest::OnExecute() { } row_count_ = static_cast(row_count); - - rc.ElapseFromBegin("total cost"); } catch (std::exception& ex) { return Status(SERVER_UNEXPECTED_ERROR, ex.what()); } diff --git a/core/src/server/grpc_impl/request/CreateIndexRequest.cpp b/core/src/server/grpc_impl/request/CreateIndexRequest.cpp index c7628048ff..9cb7a4c9e7 100644 --- a/core/src/server/grpc_impl/request/CreateIndexRequest.cpp +++ b/core/src/server/grpc_impl/request/CreateIndexRequest.cpp @@ -45,7 +45,7 @@ Status CreateIndexRequest::OnExecute() { try { std::string hdr = "CreateIndexRequest(table=" + index_param_->table_name() + ")"; - TimeRecorder rc(hdr); + TimeRecorderAuto rc(hdr); // step 1: check arguments std::string table_name_ = index_param_->table_name(); @@ -83,8 +83,6 @@ CreateIndexRequest::OnExecute() { if (!status.ok()) { return status; } - - rc.ElapseFromBegin("totally cost"); } catch (std::exception& ex) { return Status(SERVER_UNEXPECTED_ERROR, ex.what()); } diff --git a/core/src/server/grpc_impl/request/CreatePartitionRequest.cpp b/core/src/server/grpc_impl/request/CreatePartitionRequest.cpp index 1a18cdfcad..892e4ef357 100644 --- a/core/src/server/grpc_impl/request/CreatePartitionRequest.cpp +++ b/core/src/server/grpc_impl/request/CreatePartitionRequest.cpp @@ -46,7 +46,7 @@ CreatePartitionRequest::OnExecute() { std::string hdr = "CreatePartitionRequest(table=" + partition_param_->table_name() + ", partition_name=" + partition_param_->partition_name() + ", partition_tag=" + partition_param_->tag() + ")"; - TimeRecorder rc(hdr); + TimeRecorderAuto rc(hdr); try { // step 1: check arguments @@ -79,8 +79,6 @@ CreatePartitionRequest::OnExecute() { return Status(SERVER_UNEXPECTED_ERROR, ex.what()); } - rc.ElapseFromBegin("totally cost"); - return Status::OK(); } diff --git a/core/src/server/grpc_impl/request/CreateTableRequest.cpp b/core/src/server/grpc_impl/request/CreateTableRequest.cpp index 55d953aa6f..0223890616 100644 --- a/core/src/server/grpc_impl/request/CreateTableRequest.cpp +++ b/core/src/server/grpc_impl/request/CreateTableRequest.cpp @@ -45,7 +45,7 @@ Status CreateTableRequest::OnExecute() { std::string hdr = "CreateTableRequest(table=" + schema_->table_name() + ", dimension=" + std::to_string(schema_->dimension()) + ")"; - TimeRecorder rc(hdr); + TimeRecorderAuto rc(hdr); try { // step 1: check arguments @@ -89,8 +89,6 @@ CreateTableRequest::OnExecute() { return Status(SERVER_UNEXPECTED_ERROR, ex.what()); } - rc.ElapseFromBegin("totally cost"); - return Status::OK(); } diff --git a/core/src/server/grpc_impl/request/DeleteByDateRequest.cpp b/core/src/server/grpc_impl/request/DeleteByDateRequest.cpp index 4a4d414803..d2a5f91ae3 100644 --- a/core/src/server/grpc_impl/request/DeleteByDateRequest.cpp +++ b/core/src/server/grpc_impl/request/DeleteByDateRequest.cpp @@ -46,7 +46,7 @@ DeleteByDateRequest::Create(const ::milvus::grpc::DeleteByDateParam* delete_by_r Status DeleteByDateRequest::OnExecute() { try { - TimeRecorder rc("DeleteByDateRequest"); + TimeRecorderAuto rc("DeleteByDateRequest"); // step 1: check arguments std::string table_name = delete_by_range_param_->table_name(); @@ -67,7 +67,7 @@ DeleteByDateRequest::OnExecute() { } } - rc.ElapseFromBegin("check validation"); + rc.RecordSection("check validation"); // step 3: check date range, and convert to db dates std::vector dates; diff --git a/core/src/server/grpc_impl/request/DescribeIndexRequest.cpp b/core/src/server/grpc_impl/request/DescribeIndexRequest.cpp index d57d47bf2c..a478f28e0a 100644 --- a/core/src/server/grpc_impl/request/DescribeIndexRequest.cpp +++ b/core/src/server/grpc_impl/request/DescribeIndexRequest.cpp @@ -40,7 +40,7 @@ Status DescribeIndexRequest::OnExecute() { try { std::string hdr = "DescribeIndexRequest(table=" + table_name_ + ")"; - TimeRecorder rc(hdr); + TimeRecorderAuto rc(hdr); // step 1: check arguments auto status = ValidationUtil::ValidateTableName(table_name_); @@ -58,8 +58,6 @@ DescribeIndexRequest::OnExecute() { index_param_->set_table_name(table_name_); index_param_->mutable_index()->set_index_type(index.engine_type_); index_param_->mutable_index()->set_nlist(index.nlist_); - - rc.ElapseFromBegin("totally cost"); } catch (std::exception& ex) { return Status(SERVER_UNEXPECTED_ERROR, ex.what()); } diff --git a/core/src/server/grpc_impl/request/DescribeTableRequest.cpp b/core/src/server/grpc_impl/request/DescribeTableRequest.cpp index 3bd97ef7e8..847f81dc19 100644 --- a/core/src/server/grpc_impl/request/DescribeTableRequest.cpp +++ b/core/src/server/grpc_impl/request/DescribeTableRequest.cpp @@ -39,7 +39,7 @@ DescribeTableRequest::Create(const std::string& table_name, ::milvus::grpc::Tabl Status DescribeTableRequest::OnExecute() { std::string hdr = "DescribeTableRequest(table=" + table_name_ + ")"; - TimeRecorder rc(hdr); + TimeRecorderAuto rc(hdr); try { // step 1: check arguments @@ -64,8 +64,6 @@ DescribeTableRequest::OnExecute() { return Status(SERVER_UNEXPECTED_ERROR, ex.what()); } - rc.ElapseFromBegin("totally cost"); - return Status::OK(); } diff --git a/core/src/server/grpc_impl/request/DropIndexRequest.cpp b/core/src/server/grpc_impl/request/DropIndexRequest.cpp index 619ea753ba..d573686f05 100644 --- a/core/src/server/grpc_impl/request/DropIndexRequest.cpp +++ b/core/src/server/grpc_impl/request/DropIndexRequest.cpp @@ -40,7 +40,7 @@ Status DropIndexRequest::OnExecute() { try { std::string hdr = "DropIndexRequest(table=" + table_name_ + ")"; - TimeRecorder rc(hdr); + TimeRecorderAuto rc(hdr); // step 1: check arguments auto status = ValidationUtil::ValidateTableName(table_name_); @@ -63,8 +63,6 @@ DropIndexRequest::OnExecute() { if (!status.ok()) { return status; } - - rc.ElapseFromBegin("totally cost"); } catch (std::exception& ex) { return Status(SERVER_UNEXPECTED_ERROR, ex.what()); } diff --git a/core/src/server/grpc_impl/request/DropPartitionRequest.cpp b/core/src/server/grpc_impl/request/DropPartitionRequest.cpp index 1ec189986a..96913adb9f 100644 --- a/core/src/server/grpc_impl/request/DropPartitionRequest.cpp +++ b/core/src/server/grpc_impl/request/DropPartitionRequest.cpp @@ -42,7 +42,7 @@ DropPartitionRequest::OnExecute() { std::string hdr = "DropPartitionRequest(table=" + partition_param_->table_name() + ", partition_name=" + partition_param_->partition_name() + ", partition_tag=" + partition_param_->tag() + ")"; - TimeRecorder rc(hdr); + TimeRecorderAuto rc(hdr); std::string table_name = partition_param_->table_name(); std::string partition_name = partition_param_->partition_name(); diff --git a/core/src/server/grpc_impl/request/DropTableRequest.cpp b/core/src/server/grpc_impl/request/DropTableRequest.cpp index 6f81a5b2a0..ee4989d6f8 100644 --- a/core/src/server/grpc_impl/request/DropTableRequest.cpp +++ b/core/src/server/grpc_impl/request/DropTableRequest.cpp @@ -61,7 +61,7 @@ DropTableRequest::OnExecute() { } } - rc.ElapseFromBegin("check validation"); + rc.RecordSection("check validation"); // step 3: Drop table std::vector dates; diff --git a/core/src/server/grpc_impl/request/HasTableRequest.cpp b/core/src/server/grpc_impl/request/HasTableRequest.cpp index 434580efdf..d171730073 100644 --- a/core/src/server/grpc_impl/request/HasTableRequest.cpp +++ b/core/src/server/grpc_impl/request/HasTableRequest.cpp @@ -40,7 +40,7 @@ Status HasTableRequest::OnExecute() { try { std::string hdr = "HasTableRequest(table=" + table_name_ + ")"; - TimeRecorder rc(hdr); + TimeRecorderAuto rc(hdr); // step 1: check arguments auto status = ValidationUtil::ValidateTableName(table_name_); @@ -53,8 +53,6 @@ HasTableRequest::OnExecute() { if (!status.ok()) { return status; } - - rc.ElapseFromBegin("totally cost"); } catch (std::exception& ex) { return Status(SERVER_UNEXPECTED_ERROR, ex.what()); } diff --git a/core/src/server/grpc_impl/request/PreloadTableRequest.cpp b/core/src/server/grpc_impl/request/PreloadTableRequest.cpp index 3c46524afe..e659a9359d 100644 --- a/core/src/server/grpc_impl/request/PreloadTableRequest.cpp +++ b/core/src/server/grpc_impl/request/PreloadTableRequest.cpp @@ -40,7 +40,7 @@ Status PreloadTableRequest::OnExecute() { try { std::string hdr = "PreloadTableRequest(table=" + table_name_ + ")"; - TimeRecorder rc(hdr); + TimeRecorderAuto rc(hdr); // step 1: check arguments auto status = ValidationUtil::ValidateTableName(table_name_); @@ -53,8 +53,6 @@ PreloadTableRequest::OnExecute() { if (!status.ok()) { return status; } - - rc.ElapseFromBegin("totally cost"); } catch (std::exception& ex) { return Status(SERVER_UNEXPECTED_ERROR, ex.what()); } diff --git a/core/src/server/grpc_impl/request/ShowPartitionsRequest.cpp b/core/src/server/grpc_impl/request/ShowPartitionsRequest.cpp index 0d0809b171..9ba57785bd 100644 --- a/core/src/server/grpc_impl/request/ShowPartitionsRequest.cpp +++ b/core/src/server/grpc_impl/request/ShowPartitionsRequest.cpp @@ -41,7 +41,7 @@ ShowPartitionsRequest::Create(const std::string& table_name, ::milvus::grpc::Par Status ShowPartitionsRequest::OnExecute() { std::string hdr = "ShowPartitionsRequest(table=" + table_name_ + ")"; - TimeRecorder rc(hdr); + TimeRecorderAuto rc(hdr); auto status = ValidationUtil::ValidateTableName(table_name_); if (!status.ok()) { diff --git a/core/src/server/grpc_impl/request/ShowTablesRequest.cpp b/core/src/server/grpc_impl/request/ShowTablesRequest.cpp index 404e08197e..9ad46a4a83 100644 --- a/core/src/server/grpc_impl/request/ShowTablesRequest.cpp +++ b/core/src/server/grpc_impl/request/ShowTablesRequest.cpp @@ -38,7 +38,7 @@ ShowTablesRequest::Create(::milvus::grpc::TableNameList* table_name_list) { Status ShowTablesRequest::OnExecute() { - TimeRecorder rc("ShowTablesRequest"); + TimeRecorderAuto rc("ShowTablesRequest"); std::vector schema_array; auto statuts = DBWrapper::DB()->AllTables(schema_array); From fab1bbc860c159c1dbeec43d72f7e5288109b944 Mon Sep 17 00:00:00 2001 From: Lizhou Gao Date: Tue, 3 Dec 2019 13:50:22 +0800 Subject: [PATCH 44/44] replace the missing D_SEC --- core/src/db/meta/SqliteMetaImpl.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/db/meta/SqliteMetaImpl.cpp b/core/src/db/meta/SqliteMetaImpl.cpp index cb6797ae47..6a525d27d2 100644 --- a/core/src/db/meta/SqliteMetaImpl.cpp +++ b/core/src/db/meta/SqliteMetaImpl.cpp @@ -1204,7 +1204,7 @@ SqliteMetaImpl::Archive() { auto& criteria = kv.first; auto& limit = kv.second; if (criteria == engine::ARCHIVE_CONF_DAYS) { - int64_t usecs = limit * D_SEC * US_PS; + int64_t usecs = limit * DAY * US_PS; int64_t now = utils::GetMicroSecTimeStamp(); try { // multi-threads call sqlite update may get exception('bad logic', etc), so we add a lock here