Merge branch 'add_unittest' into 'branch-0.4.0'

MS-526 1. Fix knowhere unittest

See merge request megasearch/milvus!517

Former-commit-id: 948cfc7a2378f88670512732f064742b353392e1

cpp/src/core/include/knowhere/index/vector_index/nsg/nsg.h

@@ -138,8 +138,8 @@ class NsgIndex {
 
     void FindUnconnectedNode(boost::dynamic_bitset<> &flags, int64_t &root);
 
- private:
+ //private:
-    void GetKnnGraphFromFile();
+ //   void GetKnnGraphFromFile();
 };
 
 }

cpp/src/core/src/knowhere/index/vector_index/nsg/nsg.cpp

@@ -722,30 +722,30 @@ void NsgIndex::SetKnnGraph(Graph &g) {
     knng = std::move(g);
 }
 
-void NsgIndex::GetKnnGraphFromFile() {
+//void NsgIndex::GetKnnGraphFromFile() {
-    //std::string filename = "/home/zilliz/opt/workspace/wook/efanna_graph/tests/sift.1M.50NN.graph";
+//    //std::string filename = "/home/zilliz/opt/workspace/wook/efanna_graph/tests/sift.1M.50NN.graph";
-    std::string filename = "/home/zilliz/opt/workspace/wook/efanna_graph/tests/sift.50NN.graph";
+//    std::string filename = "/home/zilliz/opt/workspace/wook/efanna_graph/tests/sift.50NN.graph";
-
+//
-    std::ifstream in(filename, std::ios::binary);
+//    std::ifstream in(filename, std::ios::binary);
-    unsigned k;
+//    unsigned k;
-    in.read((char *) &k, sizeof(unsigned));
+//    in.read((char *) &k, sizeof(unsigned));
-    in.seekg(0, std::ios::end);
+//    in.seekg(0, std::ios::end);
-    std::ios::pos_type ss = in.tellg();
+//    std::ios::pos_type ss = in.tellg();
-    size_t fsize = (size_t) ss;
+//    size_t fsize = (size_t) ss;
-    size_t num = (unsigned) (fsize / (k + 1) / 4);
+//    size_t num = (unsigned) (fsize / (k + 1) / 4);
-    in.seekg(0, std::ios::beg);
+//    in.seekg(0, std::ios::beg);
-
+//
-    knng.resize(num);
+//    knng.resize(num);
-    knng.reserve(num);
+//    knng.reserve(num);
-    unsigned kk = (k + 3) / 4 * 4;
+//    unsigned kk = (k + 3) / 4 * 4;
-    for (size_t i = 0; i < num; i++) {
+//    for (size_t i = 0; i < num; i++) {
-        in.seekg(4, std::ios::cur);
+//        in.seekg(4, std::ios::cur);
-        knng[i].resize(k);
+//        knng[i].resize(k);
-        knng[i].reserve(kk);
+//        knng[i].reserve(kk);
-        in.read((char *) knng[i].data(), k * sizeof(unsigned));
+//        in.read((char *) knng[i].data(), k * sizeof(unsigned));
-    }
+//    }
-    in.close();
+//    in.close();
-}
+//}
 
 }
 }

cpp/src/core/test/CMakeLists.txt

@@ -78,6 +78,6 @@ install(TARGETS test_ivf DESTINATION unittest)
 install(TARGETS test_idmap DESTINATION unittest)
 install(TARGETS test_kdt DESTINATION unittest)
 
-add_subdirectory(faiss_ori)
+#add_subdirectory(faiss_ori)
 add_subdirectory(test_nsg)
 

cpp/src/core/test/test_idmap.cpp

@@ -148,10 +148,6 @@ TEST_F(IDMAPTest, copy_test) {
 
     {
         // cpu to gpu
-        static int64_t device_id = 0;
-        FaissGpuResourceMgr::GetInstance().InitDevice(0);
-        FaissGpuResourceMgr::GetInstance().InitDevice(1);
-
         auto clone_index = CopyCpuToGpu(index_, device_id, Config());
         auto clone_result = clone_index->Search(query_dataset, Config::object{{"k", k}});
         AssertAnns(clone_result, nq, k);
@@ -169,7 +165,7 @@ TEST_F(IDMAPTest, copy_test) {
         assert(std::static_pointer_cast<IDMAP>(host_index)->GetRawIds() != nullptr);
 
         // gpu to gpu
-        auto device_index = CopyCpuToGpu(index_, 1, Config());
+        auto device_index = CopyCpuToGpu(index_, device_id, Config());
         auto device_result = device_index->Search(query_dataset, Config::object{{"k", k}});
         AssertAnns(device_result, nq, k);
         //assert(std::static_pointer_cast<GPUIDMAP>(device_index)->GetRawVectors() != nullptr);

cpp/src/core/test/test_ivf.cpp

@@ -52,9 +52,9 @@ class IVFTest
     void SetUp() override {
         std::tie(index_type, preprocess_cfg, train_cfg, add_cfg, search_cfg) = GetParam();
         //Init_with_default();
-        Generate(128, 1000000/5, 10);
+        Generate(128, 1000000/10, 10);
         index_ = IndexFactory(index_type);
-        FaissGpuResourceMgr::GetInstance().InitDevice(device_id, 1024*1024*200, 1024*1024*300, 2);
+        FaissGpuResourceMgr::GetInstance().InitDevice(device_id, 1024*1024*200, 1024*1024*600, 2);
     }
     void TearDown() override {
         FaissGpuResourceMgr::GetInstance().Free();
@@ -77,21 +77,21 @@ INSTANTIATE_TEST_CASE_P(IVFParameters, IVFTest,
                                             Config::object{{"nlist", 100}, {"metric_type", "L2"}},
                                             Config(),
                                             Config::object{{"k", 10}}),
-                            //std::make_tuple("IVFPQ",
+                            std::make_tuple("IVFPQ",
-                            //                Config(),
+                                            Config(),
-                            //                Config::object{{"nlist", 100}, {"M", 8}, {"nbits", 8}, {"metric_type", "L2"}},
+                                            Config::object{{"nlist", 100}, {"M", 8}, {"nbits", 8}, {"metric_type", "L2"}},
-                            //                Config(),
+                                            Config(),
-                            //                Config::object{{"k", 10}}),
+                                            Config::object{{"k", 10}}),
                             std::make_tuple("GPUIVF",
                                             Config(),
                                             Config::object{{"nlist", 1638}, {"gpu_id", device_id}, {"metric_type", "L2"}},
                                             Config(),
                                             Config::object{{"k", 10}}),
-                            //std::make_tuple("GPUIVFPQ",
+                            std::make_tuple("GPUIVFPQ",
-                            //                Config(),
+                                            Config(),
-                            //                Config::object{{"gpu_id", device_id}, {"nlist", 100}, {"M", 8}, {"nbits", 8}, {"metric_type", "L2"}},
+                                            Config::object{{"gpu_id", device_id}, {"nlist", 100}, {"M", 8}, {"nbits", 8}, {"metric_type", "L2"}},
-                            //                Config(),
+                                            Config(),
-                            //                Config::object{{"k", 10}}),
+                                            Config::object{{"k", 10}}),
                             std::make_tuple("IVFSQ",
                                             Config(),
                                             Config::object{{"nlist", 100}, {"nbits", 8}, {"metric_type", "L2"}},

cpp/src/core/test/test_nsg/test_nsg.cpp

@@ -7,6 +7,7 @@
 #include <gtest/gtest.h>
 #include <memory>
 
+#include "knowhere/index/vector_index/gpu_ivf.h"
 #include "knowhere/index/vector_index/nsg_index.h"
 #include "knowhere/index/vector_index/nsg/nsg_io.h"
 
@@ -18,15 +19,22 @@ using ::testing::TestWithParam;
 using ::testing::Values;
 using ::testing::Combine;
 
+constexpr int64_t DEVICE_ID = 0;
+
 class NSGInterfaceTest : public DataGen, public TestWithParam<::std::tuple<Config, Config>> {
  protected:
     void SetUp() override {
         //Init_with_default();
+        FaissGpuResourceMgr::GetInstance().InitDevice(DEVICE_ID, 1024*1024*200, 1024*1024*300, 2);
         Generate(256, 1000000, 1);
         index_ = std::make_shared<NSG>();
         std::tie(train_cfg, search_cfg) = GetParam();
     }
 
+    void TearDown() override {
+        FaissGpuResourceMgr::GetInstance().Free();
+    }
+
  protected:
     std::shared_ptr<NSG> index_;
     Config train_cfg;