diff --git a/CHANGELOG.md b/CHANGELOG.md index 5376288534..7d1d377e9e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,8 +7,8 @@ Please mark all change in change log and use the issue from GitHub ## Bug - \#1705 Limit the insert data batch size - \#1929 Skip MySQL meta schema field width check +- \#1997 Index file missed after compact - \#2073 Fix CheckDBConfigBackendUrl error message - - \#2076 CheckMetricConfigAddress error message ## Feature diff --git a/core/src/db/DBImpl.cpp b/core/src/db/DBImpl.cpp index 0d136c7500..2497691335 100644 --- a/core/src/db/DBImpl.cpp +++ b/core/src/db/DBImpl.cpp @@ -986,18 +986,16 @@ DBImpl::CompactFile(const std::string& collection_id, const meta::SegmentSchema& return status; } - // Update collection files state - // if index type isn't IDMAP, set file type to TO_INDEX if file size exceed index_file_size - // else set file type to RAW, no need to build index - if (!utils::IsRawIndexType(compacted_file.engine_type_)) { - compacted_file.file_type_ = (segment_writer_ptr->Size() >= compacted_file.index_file_size_) - ? meta::SegmentSchema::TO_INDEX - : meta::SegmentSchema::RAW; + // Update compacted file state, if origin file is backup or to_index, set compected file to to_index + compacted_file.file_size_ = segment_writer_ptr->Size(); + compacted_file.row_count_ = segment_writer_ptr->VectorCount(); + if ((file.file_type_ == (int32_t)meta::SegmentSchema::BACKUP || + file.file_type_ == (int32_t)meta::SegmentSchema::TO_INDEX) && + (compacted_file.row_count_ > meta::BUILD_INDEX_THRESHOLD)) { + compacted_file.file_type_ = meta::SegmentSchema::TO_INDEX; } else { compacted_file.file_type_ = meta::SegmentSchema::RAW; } - compacted_file.file_size_ = segment_writer_ptr->Size(); - compacted_file.row_count_ = segment_writer_ptr->VectorCount(); if (compacted_file.row_count_ == 0) { LOG_ENGINE_DEBUG_ << "Compacted segment is empty. Mark it as TO_DELETE"; diff --git a/core/src/server/delivery/request/SearchByIDRequest.cpp b/core/src/server/delivery/request/SearchByIDRequest.cpp index c8490055ac..88806f8ba6 100644 --- a/core/src/server/delivery/request/SearchByIDRequest.cpp +++ b/core/src/server/delivery/request/SearchByIDRequest.cpp @@ -104,23 +104,7 @@ SearchByIDRequest::OnExecute() { return status; } - // step 6: check whether GPU search resource is enabled -#ifdef MILVUS_GPU_VERSION - Config& config = Config::GetInstance(); - bool gpu_enable; - config.GetGpuResourceConfigEnable(gpu_enable); - if (gpu_enable) { - std::vector search_resources; - config.GetGpuResourceConfigSearchResources(search_resources); - if (!search_resources.empty()) { - std::string err_msg = "SearchByID cannot be executed on GPU"; - LOG_SERVER_ERROR_ << err_msg; - return Status(SERVER_UNSUPPORTED_ERROR, err_msg); - } - } -#endif - - // step 7: check collection's index type supports search by id + // step 6: check collection's index type supports search by id if (collection_schema.engine_type_ != (int32_t)engine::EngineType::FAISS_IDMAP && collection_schema.engine_type_ != (int32_t)engine::EngineType::FAISS_BIN_IDMAP && collection_schema.engine_type_ != (int32_t)engine::EngineType::FAISS_IVFFLAT && @@ -134,7 +118,7 @@ SearchByIDRequest::OnExecute() { rc.RecordSection("check validation"); - // step 8: search vectors + // step 7: search vectors engine::ResultIds result_ids; engine::ResultDistances result_distances; @@ -161,7 +145,7 @@ SearchByIDRequest::OnExecute() { return Status::OK(); // empty collection } - // step 9: construct result array + // step 8: construct result array milvus::server::ContextChild tracer(context_, "Constructing result"); result_.row_num_ = id_array_.size(); result_.distance_list_.swap(result_distances); diff --git a/core/unittest/db/test_search_by_id.cpp b/core/unittest/db/test_search_by_id.cpp index 71360a3e15..d26802469c 100644 --- a/core/unittest/db/test_search_by_id.cpp +++ b/core/unittest/db/test_search_by_id.cpp @@ -37,7 +37,7 @@ std::string GetCollectionName() { auto now = std::chrono::system_clock::now(); auto micros = std::chrono::duration_cast(now.time_since_epoch()).count(); - static std::string collection_name = std::to_string(micros); + std::string collection_name = std::to_string(micros); return collection_name; } @@ -460,9 +460,7 @@ TEST_F(GetVectorByIdTest, WITH_DELETE_TEST) { } TEST_F(SearchByIdTest, BINARY_TEST) { - milvus::engine::meta::CollectionSchema collection_info; - collection_info.dimension_ = COLLECTION_DIM; - collection_info.collection_id_ = GetCollectionName(); + milvus::engine::meta::CollectionSchema collection_info = BuildCollectionSchema(); collection_info.engine_type_ = (int)milvus::engine::EngineType::FAISS_BIN_IDMAP; collection_info.metric_type_ = (int32_t)milvus::engine::MetricType::JACCARD; auto stat = db_->CreateCollection(collection_info); @@ -474,7 +472,7 @@ TEST_F(SearchByIdTest, BINARY_TEST) { ASSERT_TRUE(stat.ok()); ASSERT_EQ(collection_info_get.dimension_, COLLECTION_DIM); - int insert_loop = 10; + int insert_loop = 100; int64_t nb = 1000; for (int k = 0; k < insert_loop; ++k) { diff --git a/sdk/examples/simple/src/ClientTest.cpp b/sdk/examples/simple/src/ClientTest.cpp index 36f33737d6..edb77d763b 100644 --- a/sdk/examples/simple/src/ClientTest.cpp +++ b/sdk/examples/simple/src/ClientTest.cpp @@ -298,10 +298,10 @@ ClientTest::Test() { DescribeCollection(collection_name); InsertEntities(collection_name, dim); - BuildSearchEntities(NQ, dim); Flush(collection_name); ShowCollectionInfo(collection_name); + BuildSearchEntities(NQ, dim); GetEntitiesByID(collection_name, search_id_array_); // SearchEntities(collection_name, TOP_K, NPROBE); SearchEntitiesByID(collection_name, TOP_K, NPROBE); @@ -309,11 +309,11 @@ ClientTest::Test() { CreateIndex(collection_name, INDEX_TYPE, NLIST); ShowCollectionInfo(collection_name); - PreloadCollection(collection_name); - std::vector delete_ids = {search_id_array_[0], search_id_array_[1]}; DeleteByIds(collection_name, delete_ids); CompactCollection(collection_name); + + PreloadCollection(collection_name); SearchEntities(collection_name, TOP_K, NPROBE); // this line get two search error since we delete two entities DropIndex(collection_name);