diff --git a/CHANGELOG.md b/CHANGELOG.md index bb4d896fdb..9d20d9dcc7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,7 @@ Please mark all change in change log and use the issue from GitHub - \#2141 Fix server start failed if wal directory exist - \#2169 Fix SingleIndexTest.IVFSQHybrid unittest - \#2196 Fix server start failed if wal is disabled +- \#2231 Use server_config to define hard-delete delay time for segment files ## Feature - \#1751 Add api SearchByID diff --git a/core/conf/server_config.template b/core/conf/server_config.template index 9db7cb0252..9d7cbd8e09 100644 --- a/core/conf/server_config.template +++ b/core/conf/server_config.template @@ -68,9 +68,13 @@ db_config: # secondary_path | A semicolon-separated list of secondary directories used | Path | | # | to save vector data and index data. | | | #----------------------+------------------------------------------------------------+------------+-----------------+ +# file_cleanup_timeout | time gap between soft-delete and hard-delete | Integer | 10 (s) | +# | range [0, 3600] | | | +#----------------------+------------------------------------------------------------+------------+-----------------+ storage_config: primary_path: @MILVUS_DB_PATH@ secondary_path: + file_cleanup_timeout: 10 #----------------------+------------------------------------------------------------+------------+-----------------+ # Metric Config | Description | Type | Default | diff --git a/core/src/config/Config.cpp b/core/src/config/Config.cpp index 0c1f7c2f37..9ab1d34ae1 100644 --- a/core/src/config/Config.cpp +++ b/core/src/config/Config.cpp @@ -74,6 +74,8 @@ const char* CONFIG_STORAGE_PRIMARY_PATH = "primary_path"; const char* CONFIG_STORAGE_PRIMARY_PATH_DEFAULT = "/tmp/milvus"; const char* CONFIG_STORAGE_SECONDARY_PATH = "secondary_path"; const char* CONFIG_STORAGE_SECONDARY_PATH_DEFAULT = ""; +const char* CONFIG_STORAGE_FILE_CLEANUP_TIMEOUT = "file_cleanup_timeout"; +const char* CONFIG_STORAGE_FILE_CLEANUP_TIMEOUT_DEFAULT = "10"; const char* CONFIG_STORAGE_S3_ENABLE = "s3_enable"; const char* CONFIG_STORAGE_S3_ENABLE_DEFAULT = "false"; const char* CONFIG_STORAGE_S3_ADDRESS = "s3_address"; @@ -443,6 +445,7 @@ Config::ResetDefaultConfig() { /* storage config */ CONFIG_CHECK(SetStorageConfigPrimaryPath(CONFIG_STORAGE_PRIMARY_PATH_DEFAULT)); CONFIG_CHECK(SetStorageConfigSecondaryPath(CONFIG_STORAGE_SECONDARY_PATH_DEFAULT)); + CONFIG_CHECK(SetStorageConfigFileCleanupTimeout(CONFIG_STORAGE_FILE_CLEANUP_TIMEOUT_DEFAULT)); CONFIG_CHECK(SetStorageConfigS3Enable(CONFIG_STORAGE_S3_ENABLE_DEFAULT)); CONFIG_CHECK(SetStorageConfigS3Address(CONFIG_STORAGE_S3_ADDRESS_DEFAULT)); CONFIG_CHECK(SetStorageConfigS3Port(CONFIG_STORAGE_S3_PORT_DEFAULT)); @@ -1088,6 +1091,32 @@ Config::CheckStorageConfigSecondaryPath(const std::string& value) { return Status::OK(); } +Status +Config::CheckStorageConfigFileCleanupTimeout(const std::string& value) { + auto status = Status::OK(); + + if (value.empty()) { + return status; + } + + if (!ValidationUtil::ValidateStringIsNumber(value).ok()) { + std::string msg = "Invalid file cleanup timeout: " + value + + ". Possible reason: storage_config.file_cleanup_timeout is not a positive integer."; + return Status(SERVER_INVALID_ARGUMENT, msg); + } else { + const int64_t min = 0, max = 3600; + int64_t file_cleanup_timeout = std::stoll(value); + if (file_cleanup_timeout < min || file_cleanup_timeout > max) { + std::string msg = "Invalid file cleanup timeout: " + value + + ". Possible reason: storage_config.file_cleanup_timeout is not in range [" + + std::to_string(min) + ", " + std::to_string(max) + "]."; + return Status(SERVER_INVALID_ARGUMENT, msg); + } + } + + return Status::OK(); +} + Status Config::CheckStorageConfigS3Enable(const std::string& value) { if (!ValidationUtil::ValidateStringIsBool(value).ok()) { @@ -1501,6 +1530,7 @@ Config::CheckGpuResourceConfigBuildIndexResources(const std::vector } #endif + /* tracing config */ Status Config::CheckTracingConfigJsonConfigPath(const std::string& value) { @@ -1854,6 +1884,15 @@ Config::GetStorageConfigSecondaryPath(std::string& value) { return CheckStorageConfigSecondaryPath(value); } +Status +Config::GetStorageConfigFileCleanupTimeup(int64_t& value) { + std::string str = + GetConfigStr(CONFIG_STORAGE, CONFIG_STORAGE_FILE_CLEANUP_TIMEOUT, CONFIG_STORAGE_FILE_CLEANUP_TIMEOUT_DEFAULT); + CONFIG_CHECK(CheckStorageConfigFileCleanupTimeout(str)); + value = std::stoll(str); + return Status::OK(); +} + Status Config::GetStorageConfigS3Enable(bool& value) { std::string str = GetConfigStr(CONFIG_STORAGE, CONFIG_STORAGE_S3_ENABLE, CONFIG_STORAGE_S3_ENABLE_DEFAULT); @@ -2307,6 +2346,12 @@ Config::SetStorageConfigSecondaryPath(const std::string& value) { return SetConfigValueInMem(CONFIG_STORAGE, CONFIG_STORAGE_SECONDARY_PATH, value); } +Status +Config::SetStorageConfigFileCleanupTimeout(const std::string& value) { + CONFIG_CHECK(CheckStorageConfigFileCleanupTimeout(value)); + return SetConfigValueInMem(CONFIG_STORAGE, CONFIG_STORAGE_FILE_CLEANUP_TIMEOUT, value); +} + Status Config::SetStorageConfigS3Enable(const std::string& value) { CONFIG_CHECK(CheckStorageConfigS3Enable(value)); @@ -2498,12 +2543,14 @@ Config::SetLogsDeleteExceeds(const std::string& value) { } #ifdef MILVUS_GPU_VERSION + Status Config::SetEngineConfigGpuSearchThreshold(const std::string& value) { CONFIG_CHECK(CheckEngineConfigGpuSearchThreshold(value)); CONFIG_CHECK(SetConfigValueInMem(CONFIG_ENGINE, CONFIG_ENGINE_GPU_SEARCH_THRESHOLD, value)); return ExecCallBacks(CONFIG_ENGINE, CONFIG_ENGINE_GPU_SEARCH_THRESHOLD, value); } + #endif /* gpu resource config */ diff --git a/core/src/config/Config.h b/core/src/config/Config.h index 6bdd28f471..c3306581b0 100644 --- a/core/src/config/Config.h +++ b/core/src/config/Config.h @@ -70,6 +70,7 @@ extern const char* CONFIG_STORAGE_PRIMARY_PATH; extern const char* CONFIG_STORAGE_PRIMARY_PATH_DEFAULT; extern const char* CONFIG_STORAGE_SECONDARY_PATH; extern const char* CONFIG_STORAGE_SECONDARY_PATH_DEFAULT; +extern const char* CONFIG_STORAGE_FILE_CLEANUP_TIMEOUT; extern const char* CONFIG_STORAGE_S3_ENABLE; extern const char* CONFIG_STORAGE_S3_ENABLE_DEFAULT; extern const char* CONFIG_STORAGE_S3_ADDRESS; @@ -256,6 +257,8 @@ class Config { Status CheckStorageConfigSecondaryPath(const std::string& value); Status + CheckStorageConfigFileCleanupTimeout(const std::string& value); + Status CheckStorageConfigS3Enable(const std::string& value); Status CheckStorageConfigS3Address(const std::string& value); @@ -389,6 +392,8 @@ class Config { Status GetStorageConfigSecondaryPath(std::string& value); Status + GetStorageConfigFileCleanupTimeup(int64_t& value); + Status GetStorageConfigS3Enable(bool& value); Status GetStorageConfigS3Address(std::string& value); @@ -514,6 +519,8 @@ class Config { Status SetStorageConfigSecondaryPath(const std::string& value); Status + SetStorageConfigFileCleanupTimeout(const std::string& value); + Status SetStorageConfigS3Enable(const std::string& value); Status SetStorageConfigS3Address(const std::string& value); diff --git a/core/src/db/DBImpl.cpp b/core/src/db/DBImpl.cpp index 7216c3693e..999f76a678 100644 --- a/core/src/db/DBImpl.cpp +++ b/core/src/db/DBImpl.cpp @@ -2155,11 +2155,8 @@ DBImpl::BackgroundMerge(std::set collection_ids) { meta_ptr_->Archive(); { - uint64_t ttl = 10 * meta::SECOND; // default: file will be hard-deleted few seconds after soft-deleted - if (options_.mode_ == DBOptions::MODE::CLUSTER_WRITABLE) { - ttl = meta::HOUR; - } - + uint64_t timeout = (options_.file_cleanup_timeout_ > 0) ? options_.file_cleanup_timeout_ : 10; + uint64_t ttl = timeout * meta::SECOND; // default: file will be hard-deleted few seconds after soft-deleted meta_ptr_->CleanUpFilesWithTTL(ttl); } diff --git a/core/src/db/Options.h b/core/src/db/Options.h index 473b18614d..b446d049ec 100644 --- a/core/src/db/Options.h +++ b/core/src/db/Options.h @@ -72,6 +72,7 @@ struct DBOptions { bool insert_cache_immediately_ = false; int64_t auto_flush_interval_ = 1; + int64_t file_cleanup_timeout_ = 10; // wal relative configurations bool wal_enable_ = true; diff --git a/core/src/server/DBWrapper.cpp b/core/src/server/DBWrapper.cpp index 1ec00fb17e..f1ca8aa8dc 100644 --- a/core/src/server/DBWrapper.cpp +++ b/core/src/server/DBWrapper.cpp @@ -64,6 +64,12 @@ DBWrapper::StartService() { StringHelpFunctions::SplitStringByDelimeter(db_slave_path, ";", opt.meta_.slave_paths_); + s = config.GetStorageConfigFileCleanupTimeup(opt.file_cleanup_timeout_); + if (!s.ok()) { + std::cerr << s.ToString() << std::endl; + return s; + } + // cache config s = config.GetCacheConfigCacheInsertData(opt.insert_cache_immediately_); if (!s.ok()) { @@ -71,6 +77,14 @@ DBWrapper::StartService() { return s; } + int64_t insert_buffer_size = 1 * engine::GB; + s = config.GetCacheConfigInsertBufferSize(insert_buffer_size); + if (!s.ok()) { + std::cerr << s.ToString() << std::endl; + return s; + } + opt.insert_buffer_size_ = insert_buffer_size * engine::GB; + std::string mode; s = config.GetServerConfigDeployMode(mode); if (!s.ok()) { diff --git a/core/src/server/delivery/request/SearchCombineRequest.cpp b/core/src/server/delivery/request/SearchCombineRequest.cpp index 996cfaecc3..fa0c666640 100644 --- a/core/src/server/delivery/request/SearchCombineRequest.cpp +++ b/core/src/server/delivery/request/SearchCombineRequest.cpp @@ -384,6 +384,22 @@ SearchCombineRequest::OnExecute() { return status; } + // avoid memcpy crash, check id count = target vector count * topk + if (result_ids.size() != total_count * search_topk_) { + status = Status(DB_ERROR, "Result count doesn't match target vectors count"); + // let all request return + FreeRequests(status); + return status; + } + + // avoid memcpy crash, check distance count = id count + if (result_distances.size() != result_ids.size()) { + status = Status(DB_ERROR, "Result distance and id count doesn't match"); + // let all request return + FreeRequests(status); + return status; + } + // step 5: construct result array offset = 0; for (auto& request : request_list_) { diff --git a/sdk/examples/qps/src/ClientTest.cpp b/sdk/examples/qps/src/ClientTest.cpp index 7f6700a7b9..65420b9f2f 100644 --- a/sdk/examples/qps/src/ClientTest.cpp +++ b/sdk/examples/qps/src/ClientTest.cpp @@ -174,7 +174,7 @@ ClientTest::InsertEntities(std::shared_ptr& conn) { milvus_sdk::TimeRecorder rc(title); milvus::Status stat = conn->Insert(parameters_.collection_name_, "", entity_array, record_ids); if (!stat.ok()) { - std::cout << "CreateIndex function call status: " << stat.message() << std::endl; + std::cout << "Insert function call status: " << stat.message() << std::endl; } // std::cout << "InsertEntities function call status: " << stat.message() << std::endl; // std::cout << "Returned id array count: " << record_ids.size() << std::endl;