From 71fa8bdd05dca378bfbb6ec042c5102436080d92 Mon Sep 17 00:00:00 2001 From: jinhai Date: Thu, 31 Oct 2019 11:03:22 +0000 Subject: [PATCH 1/6] Fix parallel merge issue Former-commit-id: 7eae24c65e50c6c0abc423b841c14bf16a8c0a1d --- core/src/scheduler/job/SearchJob.h | 4 ++++ core/src/scheduler/task/SearchTask.cpp | 5 ++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/core/src/scheduler/job/SearchJob.h b/core/src/scheduler/job/SearchJob.h index 1e586090b9..40cf59e24e 100644 --- a/core/src/scheduler/job/SearchJob.h +++ b/core/src/scheduler/job/SearchJob.h @@ -90,6 +90,10 @@ class SearchJob : public Job { return index_files_; } + std::mutex& mutex() { + return mutex_; + } + private: uint64_t topk_ = 0; uint64_t nq_ = 0; diff --git a/core/src/scheduler/task/SearchTask.cpp b/core/src/scheduler/task/SearchTask.cpp index 1bf1caff76..7daa6268e8 100644 --- a/core/src/scheduler/task/SearchTask.cpp +++ b/core/src/scheduler/task/SearchTask.cpp @@ -219,8 +219,11 @@ XSearchTask::Execute() { // step 3: pick up topk result auto spec_k = index_engine_->Count() < topk ? index_engine_->Count() : topk; - XSearchTask::MergeTopkToResultSet(output_ids, output_distance, spec_k, nq, topk, metric_l2, + { + std::unique_lock lock(search_job->mutex()); + XSearchTask::MergeTopkToResultSet(output_ids, output_distance, spec_k, nq, topk, metric_l2, search_job->GetResult()); + } span = rc.RecordSection(hdr + ", reduce topk"); // search_job->AccumReduceCost(span); From d01c555604554db44dbdb4c7bcca962cce730fe3 Mon Sep 17 00:00:00 2001 From: jinhai Date: Thu, 31 Oct 2019 11:05:04 +0000 Subject: [PATCH 2/6] Fix lint Former-commit-id: 52ff843377f1463bb4cad7fcf67366b3facd8d10 --- core/src/scheduler/job/SearchJob.h | 3 ++- core/src/scheduler/task/SearchTask.cpp | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/core/src/scheduler/job/SearchJob.h b/core/src/scheduler/job/SearchJob.h index 40cf59e24e..90fcf36773 100644 --- a/core/src/scheduler/job/SearchJob.h +++ b/core/src/scheduler/job/SearchJob.h @@ -90,7 +90,8 @@ class SearchJob : public Job { return index_files_; } - std::mutex& mutex() { + std::mutex& + mutex() { return mutex_; } diff --git a/core/src/scheduler/task/SearchTask.cpp b/core/src/scheduler/task/SearchTask.cpp index 7daa6268e8..edeb41bdbe 100644 --- a/core/src/scheduler/task/SearchTask.cpp +++ b/core/src/scheduler/task/SearchTask.cpp @@ -222,7 +222,7 @@ XSearchTask::Execute() { { std::unique_lock lock(search_job->mutex()); XSearchTask::MergeTopkToResultSet(output_ids, output_distance, spec_k, nq, topk, metric_l2, - search_job->GetResult()); + search_job->GetResult()); } span = rc.RecordSection(hdr + ", reduce topk"); From 102aa353501193445a8c5cce306b8002aa87bba7 Mon Sep 17 00:00:00 2001 From: "xiaojun.lin" Date: Fri, 1 Nov 2019 15:54:58 +0800 Subject: [PATCH 3/6] add info Former-commit-id: e2a9b4550ff966180f6fdf3d4046e2886849ab81 --- .../knowhere/knowhere/index/vector_index/IndexIVF.cpp | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/core/src/index/knowhere/knowhere/index/vector_index/IndexIVF.cpp b/core/src/index/knowhere/knowhere/index/vector_index/IndexIVF.cpp index 201071c0a4..dff6dd62cc 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/IndexIVF.cpp +++ b/core/src/index/knowhere/knowhere/index/vector_index/IndexIVF.cpp @@ -221,9 +221,13 @@ IVF::search_impl(int64_t n, const float* data, int64_t k, float* distances, int6 faiss::ivflib::search_with_parameters(index_.get(), n, (float*)data, k, distances, labels, params.get()); stdclock::time_point after = stdclock::now(); double search_cost = (std::chrono::duration(after - before)).count(); - KNOWHERE_LOG_DEBUG << "IVF search cost: " << search_cost - << ", quantization cost: " << faiss::indexIVF_stats.quantization_time - << ", data search cost: " << faiss::indexIVF_stats.search_time; + KNOWHERE_LOG_DEBUG << "K=" << k + << " NQ=" << n + << " NL=" << faiss::indexIVF_stats.nlist + << " ND=" << faiss::indexIVF_stats.ndis + << " NH=" << faiss::indexIVF_stats.nheap_updates + << " Q=" << faiss::indexIVF_stats.quantization_time + << " S=" << faiss::indexIVF_stats.search_time; faiss::indexIVF_stats.quantization_time = 0; faiss::indexIVF_stats.search_time = 0; } From 70f69125333b7b1ce2fd35cb90dc70c06c66e25e Mon Sep 17 00:00:00 2001 From: "xiaojun.lin" Date: Fri, 1 Nov 2019 16:04:45 +0800 Subject: [PATCH 4/6] info lint: Former-commit-id: f3b68279e7473d817745941e2cb056489f3e8e76 --- .../knowhere/knowhere/index/vector_index/IndexIVF.cpp | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/core/src/index/knowhere/knowhere/index/vector_index/IndexIVF.cpp b/core/src/index/knowhere/knowhere/index/vector_index/IndexIVF.cpp index dff6dd62cc..6da5db38ec 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/IndexIVF.cpp +++ b/core/src/index/knowhere/knowhere/index/vector_index/IndexIVF.cpp @@ -221,11 +221,8 @@ IVF::search_impl(int64_t n, const float* data, int64_t k, float* distances, int6 faiss::ivflib::search_with_parameters(index_.get(), n, (float*)data, k, distances, labels, params.get()); stdclock::time_point after = stdclock::now(); double search_cost = (std::chrono::duration(after - before)).count(); - KNOWHERE_LOG_DEBUG << "K=" << k - << " NQ=" << n - << " NL=" << faiss::indexIVF_stats.nlist - << " ND=" << faiss::indexIVF_stats.ndis - << " NH=" << faiss::indexIVF_stats.nheap_updates + KNOWHERE_LOG_DEBUG << "K=" << k << " NQ=" << n << " NL=" << faiss::indexIVF_stats.nlist + << " ND=" << faiss::indexIVF_stats.ndis << " NH=" << faiss::indexIVF_stats.nheap_updates << " Q=" << faiss::indexIVF_stats.quantization_time << " S=" << faiss::indexIVF_stats.search_time; faiss::indexIVF_stats.quantization_time = 0; From fed65830b14f2afc4d1327688319d01f9e07bc5e Mon Sep 17 00:00:00 2001 From: zhenwu Date: Sat, 2 Nov 2019 11:49:31 +0800 Subject: [PATCH 5/6] add test params Former-commit-id: c50a44661a54e0269a6dc2dbe368b9d5b7d5a712 --- ci/jenkinsfile/dev_test.groovy | 4 ++-- ci/jenkinsfile/dev_test_all.groovy | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ci/jenkinsfile/dev_test.groovy b/ci/jenkinsfile/dev_test.groovy index f9df9b4065..89e64efea1 100644 --- a/ci/jenkinsfile/dev_test.groovy +++ b/ci/jenkinsfile/dev_test.groovy @@ -3,7 +3,7 @@ timeout(time: 30, unit: 'MINUTES') { dir ("${PROJECT_NAME}_test") { checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:Test/milvus_test.git", name: 'origin', refspec: "+refs/heads/${SEMVER}:refs/remotes/origin/${SEMVER}"]]]) sh 'python3 -m pip install -r requirements.txt -i http://pypi.douban.com/simple --trusted-host pypi.douban.com' - sh "pytest . --alluredir=\"test_out/dev/single/sqlite\" --level=1 --ip ${env.JOB_NAME}-${env.BUILD_NUMBER}-milvus-gpu-engine.milvus-1.svc.cluster.local" + sh "pytest . --alluredir=\"test_out/dev/single/sqlite\" --level=1 --ip ${env.JOB_NAME}-${env.BUILD_NUMBER}-milvus-gpu-engine.milvus-1.svc.cluster.local --internal" } // mysql database backend test load "${env.WORKSPACE}/ci/jenkinsfile/cleanup_dev.groovy" @@ -19,7 +19,7 @@ timeout(time: 30, unit: 'MINUTES') { } } dir ("${PROJECT_NAME}_test") { - sh "pytest . --alluredir=\"test_out/dev/single/mysql\" --level=1 --ip ${env.JOB_NAME}-${env.BUILD_NUMBER}-milvus-gpu-engine.milvus-2.svc.cluster.local" + sh "pytest . --alluredir=\"test_out/dev/single/mysql\" --level=1 --ip ${env.JOB_NAME}-${env.BUILD_NUMBER}-milvus-gpu-engine.milvus-2.svc.cluster.local --internal" } } catch (exc) { echo 'Milvus Test Failed !' diff --git a/ci/jenkinsfile/dev_test_all.groovy b/ci/jenkinsfile/dev_test_all.groovy index b11ea755b9..0253090942 100644 --- a/ci/jenkinsfile/dev_test_all.groovy +++ b/ci/jenkinsfile/dev_test_all.groovy @@ -3,7 +3,7 @@ timeout(time: 60, unit: 'MINUTES') { dir ("${PROJECT_NAME}_test") { checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:Test/milvus_test.git", name: 'origin', refspec: "+refs/heads/${SEMVER}:refs/remotes/origin/${SEMVER}"]]]) sh 'python3 -m pip install -r requirements.txt -i http://pypi.douban.com/simple --trusted-host pypi.douban.com' - sh "pytest . --alluredir=\"test_out/dev/single/sqlite\" --ip ${env.JOB_NAME}-${env.BUILD_NUMBER}-milvus-gpu-engine.milvus-1.svc.cluster.local" + sh "pytest . --alluredir=\"test_out/dev/single/sqlite\" --ip ${env.JOB_NAME}-${env.BUILD_NUMBER}-milvus-gpu-engine.milvus-1.svc.cluster.local --internal" } // mysql database backend test @@ -20,7 +20,7 @@ timeout(time: 60, unit: 'MINUTES') { } } dir ("${PROJECT_NAME}_test") { - sh "pytest . --alluredir=\"test_out/dev/single/mysql\" --ip ${env.JOB_NAME}-${env.BUILD_NUMBER}-milvus-gpu-engine.milvus-2.svc.cluster.local" + sh "pytest . --alluredir=\"test_out/dev/single/mysql\" --ip ${env.JOB_NAME}-${env.BUILD_NUMBER}-milvus-gpu-engine.milvus-2.svc.cluster.local --internal" } } catch (exc) { echo 'Milvus Test Failed !' From 97a3d974d3085c64983641cbed1d8eedc6629a2e Mon Sep 17 00:00:00 2001 From: zhenwu Date: Sat, 2 Nov 2019 13:19:00 +0800 Subject: [PATCH 6/6] fix test groovy Former-commit-id: cbc94db875e7078902a6238160ad581d7679150f --- ci/jenkinsfile/dev_test.groovy | 4 ++-- ci/jenkinsfile/dev_test_all.groovy | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ci/jenkinsfile/dev_test.groovy b/ci/jenkinsfile/dev_test.groovy index 89e64efea1..9b265ac401 100644 --- a/ci/jenkinsfile/dev_test.groovy +++ b/ci/jenkinsfile/dev_test.groovy @@ -3,7 +3,7 @@ timeout(time: 30, unit: 'MINUTES') { dir ("${PROJECT_NAME}_test") { checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:Test/milvus_test.git", name: 'origin', refspec: "+refs/heads/${SEMVER}:refs/remotes/origin/${SEMVER}"]]]) sh 'python3 -m pip install -r requirements.txt -i http://pypi.douban.com/simple --trusted-host pypi.douban.com' - sh "pytest . --alluredir=\"test_out/dev/single/sqlite\" --level=1 --ip ${env.JOB_NAME}-${env.BUILD_NUMBER}-milvus-gpu-engine.milvus-1.svc.cluster.local --internal" + sh "pytest . --alluredir=\"test_out/dev/single/sqlite\" --level=1 --ip ${env.JOB_NAME}-${env.BUILD_NUMBER}-milvus-gpu-engine.milvus-1.svc.cluster.local --internal=true" } // mysql database backend test load "${env.WORKSPACE}/ci/jenkinsfile/cleanup_dev.groovy" @@ -19,7 +19,7 @@ timeout(time: 30, unit: 'MINUTES') { } } dir ("${PROJECT_NAME}_test") { - sh "pytest . --alluredir=\"test_out/dev/single/mysql\" --level=1 --ip ${env.JOB_NAME}-${env.BUILD_NUMBER}-milvus-gpu-engine.milvus-2.svc.cluster.local --internal" + sh "pytest . --alluredir=\"test_out/dev/single/mysql\" --level=1 --ip ${env.JOB_NAME}-${env.BUILD_NUMBER}-milvus-gpu-engine.milvus-2.svc.cluster.local --internal=true" } } catch (exc) { echo 'Milvus Test Failed !' diff --git a/ci/jenkinsfile/dev_test_all.groovy b/ci/jenkinsfile/dev_test_all.groovy index 0253090942..b82d995d8c 100644 --- a/ci/jenkinsfile/dev_test_all.groovy +++ b/ci/jenkinsfile/dev_test_all.groovy @@ -3,7 +3,7 @@ timeout(time: 60, unit: 'MINUTES') { dir ("${PROJECT_NAME}_test") { checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:Test/milvus_test.git", name: 'origin', refspec: "+refs/heads/${SEMVER}:refs/remotes/origin/${SEMVER}"]]]) sh 'python3 -m pip install -r requirements.txt -i http://pypi.douban.com/simple --trusted-host pypi.douban.com' - sh "pytest . --alluredir=\"test_out/dev/single/sqlite\" --ip ${env.JOB_NAME}-${env.BUILD_NUMBER}-milvus-gpu-engine.milvus-1.svc.cluster.local --internal" + sh "pytest . --alluredir=\"test_out/dev/single/sqlite\" --ip ${env.JOB_NAME}-${env.BUILD_NUMBER}-milvus-gpu-engine.milvus-1.svc.cluster.local --internal=true" } // mysql database backend test @@ -20,7 +20,7 @@ timeout(time: 60, unit: 'MINUTES') { } } dir ("${PROJECT_NAME}_test") { - sh "pytest . --alluredir=\"test_out/dev/single/mysql\" --ip ${env.JOB_NAME}-${env.BUILD_NUMBER}-milvus-gpu-engine.milvus-2.svc.cluster.local --internal" + sh "pytest . --alluredir=\"test_out/dev/single/mysql\" --ip ${env.JOB_NAME}-${env.BUILD_NUMBER}-milvus-gpu-engine.milvus-2.svc.cluster.local --internal=true" } } catch (exc) { echo 'Milvus Test Failed !'