From 9b6dd23f8ede46cf874c97a1aacf2ce04ca35404 Mon Sep 17 00:00:00 2001 From: smellthemoon <64083300+smellthemoon@users.noreply.github.com> Date: Thu, 7 Nov 2024 11:22:25 +0800 Subject: [PATCH] fix: wrong path spelling when use rootpath in segcore (#37453) #36532 Signed-off-by: lixinguo Co-authored-by: lixinguo --- .../core/src/clustering/KmeansClustering.h | 37 +++++++++++-------- .../core/src/index/InvertedIndexTantivy.cpp | 14 +++---- internal/core/src/storage/ChunkManager.h | 1 + internal/core/src/storage/FileManager.h | 31 ++++++++++------ internal/core/src/storage/Util.cpp | 34 +++++++++++------ 5 files changed, 72 insertions(+), 45 deletions(-) diff --git a/internal/core/src/clustering/KmeansClustering.h b/internal/core/src/clustering/KmeansClustering.h index 500613ea0a..151ef5e3f5 100644 --- a/internal/core/src/clustering/KmeansClustering.h +++ b/internal/core/src/clustering/KmeansClustering.h @@ -17,9 +17,11 @@ #pragma once #include +#include #include #include +#include "boost/filesystem/path.hpp" #include "storage/MemFileManagerImpl.h" #include "pb/clustering.pb.h" #include "knowhere/cluster/cluster_factory.h" @@ -61,27 +63,32 @@ class KmeansClustering { GetRemoteCentroidsObjectPrefix() const { auto index_meta_ = file_manager_->GetIndexMeta(); auto field_meta_ = file_manager_->GetFieldDataMeta(); - return file_manager_->GetChunkManager()->GetRootPath() + "/" + - std::string(ANALYZE_ROOT_PATH) + "/" + - std::to_string(index_meta_.build_id) + "/" + - std::to_string(index_meta_.index_version) + "/" + - std::to_string(field_meta_.collection_id) + "/" + - std::to_string(field_meta_.partition_id) + "/" + - std::to_string(field_meta_.field_id); + boost::filesystem::path prefix = + file_manager_->GetChunkManager()->GetRootPath(); + boost::filesystem::path path = + std::to_string(index_meta_.build_id) + "/" + + std::to_string(index_meta_.index_version) + "/" + + std::to_string(field_meta_.collection_id) + "/" + + std::to_string(field_meta_.partition_id) + "/" + + std::to_string(field_meta_.field_id); + return (prefix / path).string(); } inline std::string GetRemoteCentroidIdMappingObjectPrefix(int64_t segment_id) const { auto index_meta_ = file_manager_->GetIndexMeta(); auto field_meta_ = file_manager_->GetFieldDataMeta(); - return file_manager_->GetChunkManager()->GetRootPath() + "/" + - std::string(ANALYZE_ROOT_PATH) + "/" + - std::to_string(index_meta_.build_id) + "/" + - std::to_string(index_meta_.index_version) + "/" + - std::to_string(field_meta_.collection_id) + "/" + - std::to_string(field_meta_.partition_id) + "/" + - std::to_string(field_meta_.field_id) + "/" + - std::to_string(segment_id); + boost::filesystem::path prefix = + file_manager_->GetChunkManager()->GetRootPath(); + boost::filesystem::path path = std::string(ANALYZE_ROOT_PATH); + boost::filesystem::path path1 = + std::to_string(index_meta_.build_id) + "/" + + std::to_string(index_meta_.index_version) + "/" + + std::to_string(field_meta_.collection_id) + "/" + + std::to_string(field_meta_.partition_id) + "/" + + std::to_string(field_meta_.field_id) + "/" + + std::to_string(segment_id); + return (prefix / path / path1).string(); } ~KmeansClustering() = default; diff --git a/internal/core/src/index/InvertedIndexTantivy.cpp b/internal/core/src/index/InvertedIndexTantivy.cpp index 1db1fa9d01..663564dd95 100644 --- a/internal/core/src/index/InvertedIndexTantivy.cpp +++ b/internal/core/src/index/InvertedIndexTantivy.cpp @@ -188,15 +188,15 @@ InvertedIndexTantivy::Load(milvus::tracer::TraceContext ctx, }), files_value.end()); - auto index_valid_data_file = - mem_file_manager_->GetRemoteIndexObjectPrefix() + - std::string("/index_null_offset"); - auto it = std::find( - files_value.begin(), files_value.end(), index_valid_data_file); + auto it = std::find_if( + files_value.begin(), files_value.end(), [](const std::string& file) { + return file.substr(file.find_last_of('/') + 1) == + "index_null_offset"; + }); if (it != files_value.end()) { - files_value.erase(it); std::vector file; - file.push_back(index_valid_data_file); + file.push_back(*it); + files_value.erase(it); auto index_datas = mem_file_manager_->LoadIndexToMemory(file); AssembleIndexDatas(index_datas); BinarySet binary_set; diff --git a/internal/core/src/storage/ChunkManager.h b/internal/core/src/storage/ChunkManager.h index 9f51154ee6..fa4a39b2ec 100644 --- a/internal/core/src/storage/ChunkManager.h +++ b/internal/core/src/storage/ChunkManager.h @@ -117,6 +117,7 @@ class ChunkManager { /** * @brief Get the Root Path * @return std::string + * Note: when join path, please check the training '/' */ virtual std::string GetRootPath() const = 0; diff --git a/internal/core/src/storage/FileManager.h b/internal/core/src/storage/FileManager.h index 18f4b798c8..d2e71a39ee 100644 --- a/internal/core/src/storage/FileManager.h +++ b/internal/core/src/storage/FileManager.h @@ -21,6 +21,7 @@ #include #include "common/Consts.h" +#include "boost/filesystem/path.hpp" #include "knowhere/file_manager.h" #include "log/Log.h" #include "storage/ChunkManager.h" @@ -129,11 +130,14 @@ class FileManagerImpl : public knowhere::FileManager { virtual std::string GetRemoteIndexObjectPrefix() const { - return rcm_->GetRootPath() + "/" + std::string(INDEX_ROOT_PATH) + "/" + - std::to_string(index_meta_.build_id) + "/" + - std::to_string(index_meta_.index_version) + "/" + - std::to_string(field_meta_.partition_id) + "/" + - std::to_string(field_meta_.segment_id); + boost::filesystem::path prefix = rcm_->GetRootPath(); + boost::filesystem::path path = std::string(INDEX_ROOT_PATH); + boost::filesystem::path path1 = + std::to_string(index_meta_.build_id) + "/" + + std::to_string(index_meta_.index_version) + "/" + + std::to_string(field_meta_.partition_id) + "/" + + std::to_string(field_meta_.segment_id); + return (prefix / path / path1).string(); } virtual std::string @@ -147,13 +151,16 @@ class FileManagerImpl : public knowhere::FileManager { virtual std::string GetRemoteTextLogPrefix() const { - return rcm_->GetRootPath() + "/" + std::string(TEXT_LOG_ROOT_PATH) + - "/" + std::to_string(index_meta_.build_id) + "/" + - std::to_string(index_meta_.index_version) + "/" + - std::to_string(field_meta_.collection_id) + "/" + - std::to_string(field_meta_.partition_id) + "/" + - std::to_string(field_meta_.segment_id) + "/" + - std::to_string(field_meta_.field_id); + boost::filesystem::path prefix = rcm_->GetRootPath(); + boost::filesystem::path path = std::string(TEXT_LOG_ROOT_PATH); + boost::filesystem::path path1 = + std::to_string(index_meta_.build_id) + "/" + + std::to_string(index_meta_.index_version) + "/" + + std::to_string(field_meta_.collection_id) + "/" + + std::to_string(field_meta_.partition_id) + "/" + + std::to_string(field_meta_.segment_id) + "/" + + std::to_string(field_meta_.field_id); + return (prefix / path / path1).string(); } protected: diff --git a/internal/core/src/storage/Util.cpp b/internal/core/src/storage/Util.cpp index 0ccf13b45f..5e13720772 100644 --- a/internal/core/src/storage/Util.cpp +++ b/internal/core/src/storage/Util.cpp @@ -502,8 +502,11 @@ std::string GenIndexPathPrefix(ChunkManagerPtr cm, int64_t build_id, int64_t index_version) { - return cm->GetRootPath() + "/" + std::string(INDEX_ROOT_PATH) + "/" + - GenIndexPathIdentifier(build_id, index_version); + boost::filesystem::path prefix = cm->GetRootPath(); + boost::filesystem::path path = std::string(INDEX_ROOT_PATH); + boost::filesystem::path path1 = + GenIndexPathIdentifier(build_id, index_version); + return (prefix / path / path1).string(); } std::string @@ -512,29 +515,38 @@ GenTextIndexPathPrefix(ChunkManagerPtr cm, int64_t index_version, int64_t segment_id, int64_t field_id) { - return cm->GetRootPath() + "/" + std::string(TEXT_LOG_ROOT_PATH) + "/" + - GenTextIndexPathIdentifier( - build_id, index_version, segment_id, field_id); + boost::filesystem::path prefix = cm->GetRootPath(); + boost::filesystem::path path = std::string(TEXT_LOG_ROOT_PATH); + boost::filesystem::path path1 = GenTextIndexPathIdentifier( + build_id, index_version, segment_id, field_id); + return (prefix / path / path1).string(); } std::string GetIndexPathPrefixWithBuildID(ChunkManagerPtr cm, int64_t build_id) { - return cm->GetRootPath() + "/" + std::string(INDEX_ROOT_PATH) + "/" + - std::to_string(build_id); + boost::filesystem::path prefix = cm->GetRootPath(); + boost::filesystem::path path = std::string(INDEX_ROOT_PATH); + boost::filesystem::path path1 = std::to_string(build_id); + return (prefix / path / path1).string(); } std::string GenFieldRawDataPathPrefix(ChunkManagerPtr cm, int64_t segment_id, int64_t field_id) { - return cm->GetRootPath() + "/" + std::string(RAWDATA_ROOT_PATH) + "/" + - std::to_string(segment_id) + "/" + std::to_string(field_id) + "/"; + boost::filesystem::path prefix = cm->GetRootPath(); + boost::filesystem::path path = std::string(RAWDATA_ROOT_PATH); + boost::filesystem::path path1 = + std::to_string(segment_id) + "/" + std::to_string(field_id) + "/"; + return (prefix / path / path1).string(); } std::string GetSegmentRawDataPathPrefix(ChunkManagerPtr cm, int64_t segment_id) { - return cm->GetRootPath() + "/" + std::string(RAWDATA_ROOT_PATH) + "/" + - std::to_string(segment_id); + boost::filesystem::path prefix = cm->GetRootPath(); + boost::filesystem::path path = std::string(RAWDATA_ROOT_PATH); + boost::filesystem::path path1 = std::to_string(segment_id); + return (prefix / path / path1).string(); } std::unique_ptr