From d089b9d2af3e7b299ccb6cb79c6c1dc80404708a Mon Sep 17 00:00:00 2001 From: starlord Date: Mon, 9 Sep 2019 17:58:48 +0800 Subject: [PATCH 1/3] MS-529 incorrect message Former-commit-id: 2e0c64b0ad9330e53ba185f95f85c128c456e050 --- cpp/src/db/engine/ExecutionEngine.cpp | 22 ---------------------- cpp/src/db/insert/MemTableFile.cpp | 2 +- 2 files changed, 1 insertion(+), 23 deletions(-) delete mode 100644 cpp/src/db/engine/ExecutionEngine.cpp diff --git a/cpp/src/db/engine/ExecutionEngine.cpp b/cpp/src/db/engine/ExecutionEngine.cpp deleted file mode 100644 index 3412eb34bd..0000000000 --- a/cpp/src/db/engine/ExecutionEngine.cpp +++ /dev/null @@ -1,22 +0,0 @@ -/******************************************************************************* - * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved - * Unauthorized copying of this file, via any medium is strictly prohibited. - * Proprietary and confidential. - ******************************************************************************/ -#include "ExecutionEngine.h" - -#include - -namespace zilliz { -namespace milvus { -namespace engine { - -Status ExecutionEngine::AddWithIdArray(const std::vector& vectors, const std::vector& vector_ids) { - long n = (long)vector_ids.size(); - return AddWithIds(n, vectors.data(), vector_ids.data()); -} - - -} // namespace engine -} // namespace milvus -} // namespace zilliz diff --git a/cpp/src/db/insert/MemTableFile.cpp b/cpp/src/db/insert/MemTableFile.cpp index 25abb8db90..5d55bebbae 100644 --- a/cpp/src/db/insert/MemTableFile.cpp +++ b/cpp/src/db/insert/MemTableFile.cpp @@ -49,7 +49,7 @@ Status MemTableFile::Add(const VectorSource::Ptr &source, IDNumbers& vector_ids) std::string err_msg = "MemTableFile::Add: table_file_schema dimension = " + std::to_string(table_file_schema_.dimension_) + ", table_id = " + table_file_schema_.table_id_; ENGINE_LOG_ERROR << err_msg; - return Status(DB_ERROR, err_msg); + return Status(DB_ERROR, "not able to create table file"); } size_t single_vector_mem_size = table_file_schema_.dimension_ * VECTOR_TYPE_SIZE; From d5caefd3348c7d5a9c92a6a488a5ee35f2def5ce Mon Sep 17 00:00:00 2001 From: starlord Date: Mon, 9 Sep 2019 17:59:52 +0800 Subject: [PATCH 2/3] addd unittest for ExecutionImpl Former-commit-id: 67a0c021813a1588ba6779a14bca33de337a9bbf --- cpp/src/db/engine/ExecutionEngine.h | 2 - cpp/src/db/engine/ExecutionEngineImpl.cpp | 16 +-- cpp/unittest/db/engine_test.cpp | 123 ++++++++++++++++++++++ 3 files changed, 127 insertions(+), 14 deletions(-) create mode 100644 cpp/unittest/db/engine_test.cpp diff --git a/cpp/src/db/engine/ExecutionEngine.h b/cpp/src/db/engine/ExecutionEngine.h index 9e63e02a26..4b59baf28e 100644 --- a/cpp/src/db/engine/ExecutionEngine.h +++ b/cpp/src/db/engine/ExecutionEngine.h @@ -31,8 +31,6 @@ enum class MetricType { class ExecutionEngine { public: - virtual Status AddWithIdArray(const std::vector& vectors, const std::vector& vector_ids); - virtual Status AddWithIds(long n, const float *xdata, const long *xids) = 0; virtual size_t Count() const = 0; diff --git a/cpp/src/db/engine/ExecutionEngineImpl.cpp b/cpp/src/db/engine/ExecutionEngineImpl.cpp index 59508ebcef..9f0b825b93 100644 --- a/cpp/src/db/engine/ExecutionEngineImpl.cpp +++ b/cpp/src/db/engine/ExecutionEngineImpl.cpp @@ -138,10 +138,8 @@ Status ExecutionEngineImpl::Load(bool to_cache) { } else { ENGINE_LOG_DEBUG << "Disk io from: " << location_; } - } catch (knowhere::KnowhereException &e) { - ENGINE_LOG_ERROR << e.what(); - return Status(DB_ERROR, e.what()); } catch (std::exception &e) { + ENGINE_LOG_ERROR << e.what(); return Status(DB_ERROR, e.what()); } } @@ -166,10 +164,8 @@ Status ExecutionEngineImpl::CopyToGpu(uint64_t device_id) { try { index_ = index_->CopyToGpu(device_id); ENGINE_LOG_DEBUG << "CPU to GPU" << device_id; - } catch (knowhere::KnowhereException &e) { - ENGINE_LOG_ERROR << e.what(); - return Status(DB_ERROR, e.what()); } catch (std::exception &e) { + ENGINE_LOG_ERROR << e.what(); return Status(DB_ERROR, e.what()); } } @@ -195,10 +191,8 @@ Status ExecutionEngineImpl::CopyToCpu() { try { index_ = index_->CopyToCpu(); ENGINE_LOG_DEBUG << "GPU to CPU"; - } catch (knowhere::KnowhereException &e) { - ENGINE_LOG_ERROR << e.what(); - return Status(DB_ERROR, e.what()); } catch (std::exception &e) { + ENGINE_LOG_ERROR << e.what(); return Status(DB_ERROR, e.what()); } } @@ -233,10 +227,8 @@ Status ExecutionEngineImpl::Merge(const std::string &location) { double physical_size = server::CommonUtil::GetFileSize(location); server::CollectExecutionEngineMetrics metrics(physical_size); to_merge = read_index(location); - } catch (knowhere::KnowhereException &e) { - ENGINE_LOG_ERROR << e.what(); - return Status(DB_ERROR, e.what()); } catch (std::exception &e) { + ENGINE_LOG_ERROR << e.what(); return Status(DB_ERROR, e.what()); } } diff --git a/cpp/unittest/db/engine_test.cpp b/cpp/unittest/db/engine_test.cpp new file mode 100644 index 0000000000..6cd5b5eac6 --- /dev/null +++ b/cpp/unittest/db/engine_test.cpp @@ -0,0 +1,123 @@ +//////////////////////////////////////////////////////////////////////////////// +// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved +// Unauthorized copying of this file, via any medium is strictly prohibited. +// Proprietary and confidential. +//////////////////////////////////////////////////////////////////////////////// +#include +#include + +#include "db/engine/EngineFactory.h" +#include "db/engine/ExecutionEngineImpl.h" +#include "server/ServerConfig.h" + +#include + +using namespace zilliz::milvus; + +TEST(EngineTest, FACTORY_TEST) { + { + auto engine_ptr = engine::EngineFactory::Build( + 512, + "/tmp/milvus_index_1", + engine::EngineType::INVALID, + engine::MetricType::IP, + 1024 + ); + + ASSERT_TRUE(engine_ptr == nullptr); + } + + { + auto engine_ptr = engine::EngineFactory::Build( + 512, + "/tmp/milvus_index_1", + engine::EngineType::FAISS_IDMAP, + engine::MetricType::IP, + 1024 + ); + + ASSERT_TRUE(engine_ptr != nullptr); + } + + { + auto engine_ptr = engine::EngineFactory::Build( + 512, + "/tmp/milvus_index_1", + engine::EngineType::FAISS_IVFFLAT, + engine::MetricType::IP, + 1024 + ); + + ASSERT_TRUE(engine_ptr != nullptr); + } + + { + auto engine_ptr = engine::EngineFactory::Build( + 512, + "/tmp/milvus_index_1", + engine::EngineType::FAISS_IVFSQ8, + engine::MetricType::IP, + 1024 + ); + + ASSERT_TRUE(engine_ptr != nullptr); + } + + { + auto engine_ptr = engine::EngineFactory::Build( + 512, + "/tmp/milvus_index_1", + engine::EngineType::NSG_MIX, + engine::MetricType::IP, + 1024 + ); + + ASSERT_TRUE(engine_ptr != nullptr); + } +} + +TEST(EngineTest, ENGINE_IMPL_TEST) { + uint16_t dimension = 64; + std::string file_path = "/tmp/milvus_index_1"; + auto engine_ptr = engine::EngineFactory::Build( + dimension, + file_path, + engine::EngineType::FAISS_IVFFLAT, + engine::MetricType::IP, + 1024 + ); + + std::vector data; + std::vector ids; + const int row_count = 10000; + data.reserve(row_count*dimension); + ids.reserve(row_count); + for(long i = 0; i < row_count; i++) { + ids.push_back(i); + for(uint16_t k = 0; k < dimension; k++) { + data.push_back(i*dimension + k); + } + } + + auto status = engine_ptr->AddWithIds((long)ids.size(), data.data(), ids.data()); + ASSERT_TRUE(status.ok()); + + ASSERT_EQ(engine_ptr->Dimension(), dimension); + ASSERT_EQ(engine_ptr->Count(), ids.size()); + + server::ConfigNode& config = server::ServerConfig::GetInstance().GetConfig(server::CONFIG_CACHE); + config.AddSequenceItem(server::CONFIG_GPU_IDS, "0"); + + status = engine_ptr->CopyToGpu(0); + //ASSERT_TRUE(status.ok()); + + auto new_engine = engine_ptr->Clone(); + ASSERT_EQ(new_engine->Dimension(), dimension); + ASSERT_EQ(new_engine->Count(), ids.size()); + status = new_engine->CopyToCpu(); + //ASSERT_TRUE(status.ok()); + + auto engine_build = new_engine->BuildIndex("/tmp/milvus_index_2", engine::EngineType::FAISS_IVFSQ8); + //ASSERT_TRUE(status.ok()); + +} From 7bb2d98fe2d90a64ee7c4f5943e2ea3be5fd2431 Mon Sep 17 00:00:00 2001 From: starlord Date: Mon, 9 Sep 2019 19:22:53 +0800 Subject: [PATCH 3/3] reduce unittest run time Former-commit-id: d643a9bdc49bc704cf6e9e6994776b22270476e7 --- cpp/src/core/test/test_nsg/test_nsg.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cpp/src/core/test/test_nsg/test_nsg.cpp b/cpp/src/core/test/test_nsg/test_nsg.cpp index 50eccab272..3c8d47770f 100644 --- a/cpp/src/core/test/test_nsg/test_nsg.cpp +++ b/cpp/src/core/test/test_nsg/test_nsg.cpp @@ -26,7 +26,7 @@ class NSGInterfaceTest : public DataGen, public TestWithParam<::std::tuple(); std::tie(train_cfg, search_cfg) = GetParam(); } @@ -44,7 +44,7 @@ class NSGInterfaceTest : public DataGen, public TestWithParam<::std::tuple out_degree - Config::object{{"nlist", 16384}, {"nprobe", 50}, {"knng", 100}, {"metric_type", "L2"}, + Config::object{{"nlist", 128}, {"nprobe", 50}, {"knng", 100}, {"metric_type", "L2"}, {"search_length", 60}, {"out_degree", 70}, {"candidate_pool_size", 500}}, Config::object{{"k", 20}, {"search_length", 30}})) );