file cleanup timeout config (#2250)

* write error Signed-off-by: groot <yihua.mo@zilliz.com> * out of storage Signed-off-by: groot <yihua.mo@zilliz.com> * clang format Signed-off-by: groot <yihua.mo@zilliz.com> * fix ut Signed-off-by: groot <yihua.mo@zilliz.com> * fix #1955 Signed-off-by: groot <yihua.mo@zilliz.com> * refine code Signed-off-by: groot <yihua.mo@zilliz.com> * compact threashold Signed-off-by: groot <yihua.mo@zilliz.com> * changelog Signed-off-by: groot <yihua.mo@zilliz.com> * search by id for hnsw/pq/annoy Signed-off-by: groot <yihua.mo@zilliz.com> * fix python test Signed-off-by: yhmo <yihua.mo@zilliz.com> * file cleanup timeout config Signed-off-by: yhmo <yihua.mo@zilliz.com> * typo Signed-off-by: yhmo <yihua.mo@zilliz.com> * changelog Signed-off-by: yhmo <yihua.mo@zilliz.com>
2025-12-29 06:55:27 +08:00 · 2020-05-08 01:24:29 -05:00 · 2020-05-08 01:24:29 -05:00 · da55622042
commit da55622042
parent 56055d9d00
9 changed files with 93 additions and 6 deletions
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@ -18,6 +18,7 @@ Please mark all change in change log and use the issue from GitHub
 -   \#2141 Fix server start failed if wal directory exist
 -   \#2169 Fix SingleIndexTest.IVFSQHybrid unittest
 -   \#2196 Fix server start failed if wal is disabled
+-   \#2231 Use server_config to define hard-delete delay time for segment files

 ## Feature
 -   \#1751 Add api SearchByID
--- a/core/conf/server_config.template
+++ b/core/conf/server_config.template
@ -68,9 +68,13 @@ db_config:
 # secondary_path       | A semicolon-separated list of secondary directories used   | Path       |                 |
 #                      | to save vector data and index data.                        |            |                 |
 #----------------------+------------------------------------------------------------+------------+-----------------+
+# file_cleanup_timeout | time gap between soft-delete and hard-delete               | Integer    | 10 (s)          |
+#                      | range [0, 3600]                                            |            |                 |
+#----------------------+------------------------------------------------------------+------------+-----------------+
 storage_config:
  primary_path: @MILVUS_DB_PATH@
  secondary_path:
+  file_cleanup_timeout: 10

 #----------------------+------------------------------------------------------------+------------+-----------------+
 # Metric Config        | Description                                                | Type       | Default         |
--- a/core/src/config/Config.cpp
+++ b/core/src/config/Config.cpp
@ -74,6 +74,8 @@ const char* CONFIG_STORAGE_PRIMARY_PATH = "primary_path";
 const char* CONFIG_STORAGE_PRIMARY_PATH_DEFAULT = "/tmp/milvus";
 const char* CONFIG_STORAGE_SECONDARY_PATH = "secondary_path";
 const char* CONFIG_STORAGE_SECONDARY_PATH_DEFAULT = "";
+const char* CONFIG_STORAGE_FILE_CLEANUP_TIMEOUT = "file_cleanup_timeout";
+const char* CONFIG_STORAGE_FILE_CLEANUP_TIMEOUT_DEFAULT = "10";
 const char* CONFIG_STORAGE_S3_ENABLE = "s3_enable";
 const char* CONFIG_STORAGE_S3_ENABLE_DEFAULT = "false";
 const char* CONFIG_STORAGE_S3_ADDRESS = "s3_address";
@ -443,6 +445,7 @@ Config::ResetDefaultConfig() {
    /* storage config */
    CONFIG_CHECK(SetStorageConfigPrimaryPath(CONFIG_STORAGE_PRIMARY_PATH_DEFAULT));
    CONFIG_CHECK(SetStorageConfigSecondaryPath(CONFIG_STORAGE_SECONDARY_PATH_DEFAULT));
+    CONFIG_CHECK(SetStorageConfigFileCleanupTimeout(CONFIG_STORAGE_FILE_CLEANUP_TIMEOUT_DEFAULT));
    CONFIG_CHECK(SetStorageConfigS3Enable(CONFIG_STORAGE_S3_ENABLE_DEFAULT));
    CONFIG_CHECK(SetStorageConfigS3Address(CONFIG_STORAGE_S3_ADDRESS_DEFAULT));
    CONFIG_CHECK(SetStorageConfigS3Port(CONFIG_STORAGE_S3_PORT_DEFAULT));
@ -1088,6 +1091,32 @@ Config::CheckStorageConfigSecondaryPath(const std::string& value) {
    return Status::OK();
 }

+Status
+Config::CheckStorageConfigFileCleanupTimeout(const std::string& value) {
+    auto status = Status::OK();
+
+    if (value.empty()) {
+        return status;
+    }
+
+    if (!ValidationUtil::ValidateStringIsNumber(value).ok()) {
+        std::string msg = "Invalid file cleanup timeout: " + value +
+                          ". Possible reason: storage_config.file_cleanup_timeout is not a positive integer.";
+        return Status(SERVER_INVALID_ARGUMENT, msg);
+    } else {
+        const int64_t min = 0, max = 3600;
+        int64_t file_cleanup_timeout = std::stoll(value);
+        if (file_cleanup_timeout < min || file_cleanup_timeout > max) {
+            std::string msg = "Invalid file cleanup timeout: " + value +
+                              ". Possible reason: storage_config.file_cleanup_timeout is not in range [" +
+                              std::to_string(min) + ", " + std::to_string(max) + "].";
+            return Status(SERVER_INVALID_ARGUMENT, msg);
+        }
+    }
+
+    return Status::OK();
+}
+
 Status
 Config::CheckStorageConfigS3Enable(const std::string& value) {
    if (!ValidationUtil::ValidateStringIsBool(value).ok()) {
@ -1501,6 +1530,7 @@ Config::CheckGpuResourceConfigBuildIndexResources(const std::vector<std::string>
 }

 #endif
+
 /* tracing config */
 Status
 Config::CheckTracingConfigJsonConfigPath(const std::string& value) {
@ -1854,6 +1884,15 @@ Config::GetStorageConfigSecondaryPath(std::string& value) {
    return CheckStorageConfigSecondaryPath(value);
 }

+Status
+Config::GetStorageConfigFileCleanupTimeup(int64_t& value) {
+    std::string str =
+        GetConfigStr(CONFIG_STORAGE, CONFIG_STORAGE_FILE_CLEANUP_TIMEOUT, CONFIG_STORAGE_FILE_CLEANUP_TIMEOUT_DEFAULT);
+    CONFIG_CHECK(CheckStorageConfigFileCleanupTimeout(str));
+    value = std::stoll(str);
+    return Status::OK();
+}
+
 Status
 Config::GetStorageConfigS3Enable(bool& value) {
    std::string str = GetConfigStr(CONFIG_STORAGE, CONFIG_STORAGE_S3_ENABLE, CONFIG_STORAGE_S3_ENABLE_DEFAULT);
@ -2307,6 +2346,12 @@ Config::SetStorageConfigSecondaryPath(const std::string& value) {
    return SetConfigValueInMem(CONFIG_STORAGE, CONFIG_STORAGE_SECONDARY_PATH, value);
 }

+Status
+Config::SetStorageConfigFileCleanupTimeout(const std::string& value) {
+    CONFIG_CHECK(CheckStorageConfigFileCleanupTimeout(value));
+    return SetConfigValueInMem(CONFIG_STORAGE, CONFIG_STORAGE_FILE_CLEANUP_TIMEOUT, value);
+}
+
 Status
 Config::SetStorageConfigS3Enable(const std::string& value) {
    CONFIG_CHECK(CheckStorageConfigS3Enable(value));
@ -2498,12 +2543,14 @@ Config::SetLogsDeleteExceeds(const std::string& value) {
 }

 #ifdef MILVUS_GPU_VERSION
+
 Status
 Config::SetEngineConfigGpuSearchThreshold(const std::string& value) {
    CONFIG_CHECK(CheckEngineConfigGpuSearchThreshold(value));
    CONFIG_CHECK(SetConfigValueInMem(CONFIG_ENGINE, CONFIG_ENGINE_GPU_SEARCH_THRESHOLD, value));
    return ExecCallBacks(CONFIG_ENGINE, CONFIG_ENGINE_GPU_SEARCH_THRESHOLD, value);
 }
+
 #endif

 /* gpu resource config */
--- a/core/src/config/Config.h
+++ b/core/src/config/Config.h
@ -70,6 +70,7 @@ extern const char* CONFIG_STORAGE_PRIMARY_PATH;
 extern const char* CONFIG_STORAGE_PRIMARY_PATH_DEFAULT;
 extern const char* CONFIG_STORAGE_SECONDARY_PATH;
 extern const char* CONFIG_STORAGE_SECONDARY_PATH_DEFAULT;
+extern const char* CONFIG_STORAGE_FILE_CLEANUP_TIMEOUT;
 extern const char* CONFIG_STORAGE_S3_ENABLE;
 extern const char* CONFIG_STORAGE_S3_ENABLE_DEFAULT;
 extern const char* CONFIG_STORAGE_S3_ADDRESS;
@ -256,6 +257,8 @@ class Config {
    Status
    CheckStorageConfigSecondaryPath(const std::string& value);
    Status
+    CheckStorageConfigFileCleanupTimeout(const std::string& value);
+    Status
    CheckStorageConfigS3Enable(const std::string& value);
    Status
    CheckStorageConfigS3Address(const std::string& value);
@ -389,6 +392,8 @@ class Config {
    Status
    GetStorageConfigSecondaryPath(std::string& value);
    Status
+    GetStorageConfigFileCleanupTimeup(int64_t& value);
+    Status
    GetStorageConfigS3Enable(bool& value);
    Status
    GetStorageConfigS3Address(std::string& value);
@ -514,6 +519,8 @@ class Config {
    Status
    SetStorageConfigSecondaryPath(const std::string& value);
    Status
+    SetStorageConfigFileCleanupTimeout(const std::string& value);
+    Status
    SetStorageConfigS3Enable(const std::string& value);
    Status
    SetStorageConfigS3Address(const std::string& value);
--- a/core/src/db/DBImpl.cpp
+++ b/core/src/db/DBImpl.cpp
@ -2155,11 +2155,8 @@ DBImpl::BackgroundMerge(std::set<std::string> collection_ids) {
    meta_ptr_->Archive();

    {
-        uint64_t ttl = 10 * meta::SECOND;  // default: file will be hard-deleted few seconds after soft-deleted
-        if (options_.mode_ == DBOptions::MODE::CLUSTER_WRITABLE) {
-            ttl = meta::HOUR;
-        }
-
+        uint64_t timeout = (options_.file_cleanup_timeout_ > 0) ? options_.file_cleanup_timeout_ : 10;
+        uint64_t ttl = timeout * meta::SECOND;  // default: file will be hard-deleted few seconds after soft-deleted
        meta_ptr_->CleanUpFilesWithTTL(ttl);
    }

--- a/core/src/db/Options.h
+++ b/core/src/db/Options.h
@ -72,6 +72,7 @@ struct DBOptions {
    bool insert_cache_immediately_ = false;

    int64_t auto_flush_interval_ = 1;
+    int64_t file_cleanup_timeout_ = 10;

    // wal relative configurations
    bool wal_enable_ = true;
--- a/core/src/server/DBWrapper.cpp
+++ b/core/src/server/DBWrapper.cpp
@ -64,6 +64,12 @@ DBWrapper::StartService() {

    StringHelpFunctions::SplitStringByDelimeter(db_slave_path, ";", opt.meta_.slave_paths_);

+    s = config.GetStorageConfigFileCleanupTimeup(opt.file_cleanup_timeout_);
+    if (!s.ok()) {
+        std::cerr << s.ToString() << std::endl;
+        return s;
+    }
+
    // cache config
    s = config.GetCacheConfigCacheInsertData(opt.insert_cache_immediately_);
    if (!s.ok()) {
@ -71,6 +77,14 @@ DBWrapper::StartService() {
        return s;
    }

+    int64_t insert_buffer_size = 1 * engine::GB;
+    s = config.GetCacheConfigInsertBufferSize(insert_buffer_size);
+    if (!s.ok()) {
+        std::cerr << s.ToString() << std::endl;
+        return s;
+    }
+    opt.insert_buffer_size_ = insert_buffer_size * engine::GB;
+
    std::string mode;
    s = config.GetServerConfigDeployMode(mode);
    if (!s.ok()) {
--- a/core/src/server/delivery/request/SearchCombineRequest.cpp
+++ b/core/src/server/delivery/request/SearchCombineRequest.cpp
@ -384,6 +384,22 @@ SearchCombineRequest::OnExecute() {
            return status;
        }

+        // avoid memcpy crash, check id count = target vector count * topk
+        if (result_ids.size() != total_count * search_topk_) {
+            status = Status(DB_ERROR, "Result count doesn't match target vectors count");
+            // let all request return
+            FreeRequests(status);
+            return status;
+        }
+
+        // avoid memcpy crash, check distance count = id count
+        if (result_distances.size() != result_ids.size()) {
+            status = Status(DB_ERROR, "Result distance and id count doesn't match");
+            // let all request return
+            FreeRequests(status);
+            return status;
+        }
+
        // step 5: construct result array
        offset = 0;
        for (auto& request : request_list_) {
--- a/sdk/examples/qps/src/ClientTest.cpp
+++ b/sdk/examples/qps/src/ClientTest.cpp
@ -174,7 +174,7 @@ ClientTest::InsertEntities(std::shared_ptr<milvus::Connection>& conn) {
        milvus_sdk::TimeRecorder rc(title);
        milvus::Status stat = conn->Insert(parameters_.collection_name_, "", entity_array, record_ids);
        if (!stat.ok()) {
-            std::cout << "CreateIndex function call status: " << stat.message() << std::endl;
+            std::cout << "Insert function call status: " << stat.message() << std::endl;
        }
 //        std::cout << "InsertEntities function call status: " << stat.message() << std::endl;
 //        std::cout << "Returned id array count: " << record_ids.size() << std::endl;