Use OpenDAL to access object store (#25642)

Signed-off-by: Enwei Jiao <enwei.jiao@zilliz.com>

.github/workflows/code-checker.yaml

@@ -35,6 +35,14 @@ jobs:
     strategy:
       fail-fast: false
     steps:
+      - name: Maximize build space
+        uses: easimon/maximize-build-space@master
+        with:
+          root-reserve-mb: 10240
+          swap-size-mb: 1024
+          remove-dotnet: 'true'
+          remove-android: 'true'
+          remove-haskell: 'true'
       - name: Checkout
         uses: actions/checkout@v2
       - name: 'Generate CCache Hash'
@@ -74,6 +82,14 @@ jobs:
     runs-on: ubuntu-latest
     timeout-minutes: 180
     steps:
+      - name: Maximize build space
+        uses: easimon/maximize-build-space@master
+        with:
+          root-reserve-mb: 10240
+          swap-size-mb: 1024
+          remove-dotnet: 'true'
+          remove-android: 'true'
+          remove-haskell: 'true'
       - name: Checkout
         uses: actions/checkout@v2
       - name: 'Generate CCache Hash'

.github/workflows/main.yaml

@@ -47,6 +47,14 @@ jobs:
     env:
       UBUNTU: ${{ matrix.ubuntu }}
     steps:
+      - name: Maximize build space
+        uses: easimon/maximize-build-space@master
+        with:
+          root-reserve-mb: 10240
+          swap-size-mb: 1024
+          remove-dotnet: 'true'
+          remove-android: 'true'
+          remove-haskell: 'true'
       - name: Checkout
         uses: actions/checkout@v2
         with:
@@ -92,7 +100,7 @@ jobs:
         run: |
           ./build/builder.sh /bin/bash -c "make USE_ASAN=${{env.useasan}} build-cpp-with-coverage"
       - run: |
-          zip -r code.zip . -x "./.docker/*"
+          zip -r code.zip . -x "./.docker/*" -x "./cmake_build/thirdparty/*"
       - name: Archive code
         uses: actions/upload-artifact@v3
         with:
@@ -127,8 +135,6 @@ jobs:
         shell: bash
         run: |
           docker-compose up -d azurite
-#      - name: 'Setup upterm session'
-#        uses: lhotari/action-upterm@v1
       - name: UT
         run: |
           chmod +x build/builder.sh
@@ -173,8 +179,6 @@ jobs:
         shell: bash
         run: |
           docker-compose up -d pulsar etcd minio azurite
-#      - name: 'Setup upterm session'
-#        uses: lhotari/action-upterm@v1
       - name: UT
         run: |
           chmod +x build/builder.sh

configs/milvus.yaml

@@ -91,7 +91,7 @@ minio:
   # Supported level:  off, fatal, error, warn, info, debug, trace
   logLevel: fatal
   # Cloud data center region
-  region: ""
+  region: ''
   # Cloud whether use virtual host bucket mode
   useVirtualHost: false
   # timeout for request time in milliseconds
@@ -500,7 +500,7 @@ common:
     BeamWidthRatio: 4
   gracefulTime: 5000 # milliseconds. it represents the interval (in ms) by which the request arrival time needs to be subtracted in the case of Bounded Consistency.
   gracefulStopTimeout: 1800 # seconds. it will force quit the server if the graceful stop process is not completed during this time.
-  storageType: minio # please adjust in embedded Milvus: local
+  storageType: opendal # please adjust in embedded Milvus: local
   # Default value: auto
   # Valid values: [auto, avx512, avx2, avx, sse4_2]
   # This configuration is only used by querynode and indexnode, it selects CPU instruction set for Searching and Index-building.
@@ -520,9 +520,9 @@ common:
     enabled: false
     # support pre-created topics
     # the name of pre-created topics
-    names: ["topic1", "topic2"]
+    names: ['topic1', 'topic2']
     # need to set a separated topic to stand for currently consumed timestamp for each channel
-    timeticker: "timetick-channel"
+    timeticker: 'timetick-channel'
 
   ImportMaxFileSize: 17179869184 # 16 * 1024 * 1024 * 1024
   # max file size to import for bulkInsert

internal/core/conanfile.py

@@ -6,7 +6,7 @@ class MilvusConan(ConanFile):
     requires = (
         "rocksdb/6.29.5",
         "boost/1.82.0",
-        "onetbb/2021.7.0",
+        "onetbb/2021.9.0",
         "nlohmann_json/3.11.2",
         "zstd/1.5.4",
         "lz4/1.9.4",

internal/core/src/common/CMakeLists.txt

@@ -33,6 +33,7 @@ target_link_libraries(milvus_common
         yaml-cpp
         boost_bitset_ext
         simdjson
+        opendal
         ${CONAN_LIBS}
         )
 

internal/core/src/indexbuilder/CMakeLists.txt

@@ -20,17 +20,7 @@ set(INDEXBUILDER_FILES
 milvus_add_pkg_config("milvus_indexbuilder")
 add_library(milvus_indexbuilder SHARED ${INDEXBUILDER_FILES})
 
-find_library(TBB NAMES tbb)
-set(PLATFORM_LIBS dl)
-if (MSYS)
-set(PLATFORM_LIBS -Wl,--allow-multiple-definition)
-endif ()
-
 # link order matters
-target_link_libraries(milvus_indexbuilder
+target_link_libraries(milvus_indexbuilder milvus_index)
-        milvus_index
-        ${TBB}
-        ${PLATFORM_LIBS}
-        )
 
 install(TARGETS milvus_indexbuilder DESTINATION "${CMAKE_INSTALL_LIBDIR}")

internal/core/src/indexbuilder/index_c.cpp

@@ -310,6 +310,8 @@ NewBuildIndexInfo(CBuildIndexInfo* c_build_index_info,
             std::string(c_storage_config.cloud_provider);
         storage_config.iam_endpoint =
             std::string(c_storage_config.iam_endpoint);
+        storage_config.cloud_provider =
+            std::string(c_storage_config.cloud_provider);
         storage_config.useSSL = c_storage_config.useSSL;
         storage_config.useIAM = c_storage_config.useIAM;
         storage_config.region = c_storage_config.region;

internal/core/src/segcore/CMakeLists.txt

@@ -41,18 +41,6 @@ set(SEGCORE_FILES
         ConcurrentVector.cpp)
 add_library(milvus_segcore SHARED ${SEGCORE_FILES})
 
-find_library(TBB NAMES tbb)
+target_link_libraries(milvus_segcore milvus_query ${OpenMP_CXX_FLAGS})
-set(PLATFORM_LIBS dl)
-
-if (MSYS)
-  set(PLATFORM_LIBS )
-endif()
-
-target_link_libraries(milvus_segcore
-        milvus_query
-        ${PLATFORM_LIBS}
-        ${TBB}
-        ${OpenMP_CXX_FLAGS}
-        )
 
 install(TARGETS milvus_segcore DESTINATION "${CMAKE_INSTALL_LIBDIR}")

internal/core/src/storage/CMakeLists.txt

@@ -47,8 +47,9 @@ set(STORAGE_FILES
     Event.cpp
     ThreadPool.cpp
     storage_c.cpp
+    ChunkManager.cpp
     MinioChunkManager.cpp
-    ChunkManagers.cpp
+    OpenDALChunkManager.cpp
     AliyunSTSClient.cpp
     AliyunCredentialsProvider.cpp
     MemFileManagerImpl.cpp

internal/core/src/storage/ChunkManager.h

@@ -129,6 +129,7 @@ enum class ChunkManagerType : int8_t {
     Local = 1,
     Minio = 2,
     Remote = 3,
+    OpenDAL = 4,
 };
 
 extern std::map<std::string, ChunkManagerType> ChunkManagerType_Map;

internal/core/src/storage/OpenDALChunkManager.cpp

@@ -0,0 +1,206 @@
+// Licensed to the LF AI & Data foundation under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <algorithm>
+#include <fstream>
+#include <string>
+
+#include "log/Log.h"
+#include "opendal.h"
+#include "common/EasyAssert.h"
+#include "storage/Util.h"
+#include "storage/OpenDALChunkManager.h"
+
+namespace milvus::storage {
+
+std::string
+ToString(opendal_bytes* bs) {
+    return {reinterpret_cast<const char*>(bs->data), bs->len};
+}
+
+#define THROWOPENDALERROR(err, msg)                                            \
+    do {                                                                       \
+        auto exception = SegcoreError(                                         \
+            S3Error, fmt::format("{}: {}", (msg), ToString(&(err)->message))); \
+        opendal_error_free(err);                                               \
+        throw exception;                                                       \
+    } while (0)
+
+// std::once_flag init_flag_;
+
+OpenDALChunkManager::OpenDALChunkManager(const StorageConfig& storage_config)
+    : default_bucket_name_(storage_config.bucket_name) {
+    // std::call_once(init_flag_, []() { opendal_init_logger(); });
+    remote_root_path_ = storage_config.root_path;
+    std::string storageType;
+    if (storage_config.cloud_provider == "gcp") {
+        storageType = "gcs";
+    } else if (storage_config.cloud_provider == "aliyun") {
+        storageType = "oss";
+    } else if (storage_config.cloud_provider == "azure") {
+        storageType = "azblob";
+    } else {
+        storageType = "s3";
+    }
+
+    opendal_operator_options* op_options_ = opendal_operator_options_new();
+    if (!storage_config.access_key_id.empty() &&
+        !storage_config.access_key_value.empty()) {
+        opendal_operator_options_set(
+            op_options_, "access_key_id", storage_config.access_key_id.c_str());
+        opendal_operator_options_set(op_options_,
+                                     "secret_access_key",
+                                     storage_config.access_key_value.c_str());
+        storageType = "s3";
+    }
+    opendal_operator_options_set(op_options_, "root", "/");
+    opendal_operator_options_set(
+        op_options_, "bucket", storage_config.bucket_name.c_str());
+    opendal_operator_options_set(op_options_,
+                                 "region",
+                                 storage_config.region.empty()
+                                     ? "us-east-1"
+                                     : storage_config.region.c_str());
+    opendal_operator_options_set(
+        op_options_,
+        "endpoint",
+        ((storage_config.useSSL ? "https://" : "http://") +
+         storage_config.address)
+            .c_str());
+    opendal_operator_options_set(
+        op_options_,
+        "enable_virtual_host_style",
+        storage_config.useVirtualHost ? "true" : "false");
+
+    auto op = opendal_operator_new(storageType.c_str(), op_options_);
+    if (op.error != nullptr) {
+        THROWOPENDALERROR(op.error, "Init opendal error");
+    }
+    op_ptr_ = op.op;
+    opendal_operator_options_free(op_options_);
+    LOG_SEGCORE_INFO_ << "init OpenDALChunkManager with parameter[storage: '"
+                      << storageType << ", " << storage_config.cloud_provider
+                      << "', endpoint: '" << storage_config.address
+                      << "', default_bucket_name:'"
+                      << storage_config.bucket_name << "', use_secure:'"
+                      << std::boolalpha << storage_config.useSSL << "']";
+}
+
+OpenDALChunkManager::~OpenDALChunkManager() {
+    opendal_operator_free(op_ptr_);
+}
+
+uint64_t
+OpenDALChunkManager::Size(const std::string& filepath) {
+    auto ret = opendal_operator_stat(op_ptr_, filepath.c_str());
+    if (ret.error != nullptr) {
+        THROWOPENDALERROR(ret.error, "GetObjectSize");
+    }
+    auto size = opendal_metadata_content_length(ret.meta);
+    opendal_metadata_free(ret.meta);
+    return size;
+}
+
+bool
+OpenDALChunkManager::Exist(const std::string& filepath) {
+    auto ret = opendal_operator_is_exist(op_ptr_, filepath.c_str());
+    if (ret.error != nullptr) {
+        THROWOPENDALERROR(ret.error, "ObjectExists");
+    }
+    return ret.is_exist;
+}
+
+void
+OpenDALChunkManager::Remove(const std::string& filepath) {
+    auto ret = opendal_operator_delete(op_ptr_, filepath.c_str());
+    if (ret != nullptr) {
+        THROWOPENDALERROR(ret, "RemoveObject");
+    }
+}
+
+std::vector<std::string>
+OpenDALChunkManager::ListWithPrefix(const std::string& filepath) {
+    auto ret = opendal_operator_list(op_ptr_, filepath.c_str());
+    if (ret.error != nullptr) {
+        THROWOPENDALERROR(ret.error, "ListObjects");
+    }
+    auto lister = OpendalLister(ret.lister);
+    std::vector<std::string> objects;
+    opendal_result_lister_next result = opendal_lister_next(lister.Get());
+    if (result.error != nullptr) {
+        THROWOPENDALERROR(result.error, "ListObjects");
+    }
+    auto entry = result.entry;
+    while (entry) {
+        const char* de_path = opendal_entry_path(entry);
+        objects.push_back(std::string(de_path));
+        opendal_entry_free(entry);
+        result = opendal_lister_next(lister.Get());
+        if (result.error != nullptr) {
+            THROWOPENDALERROR(result.error, "ListObjects");
+        }
+        entry = result.entry;
+    }
+    return objects;
+}
+
+uint64_t
+OpenDALChunkManager::Read(const std::string& filepath,
+                          void* buf,
+                          uint64_t size) {
+    auto ret = opendal_operator_reader(op_ptr_, filepath.c_str());
+    if (ret.error != nullptr) {
+        THROWOPENDALERROR(ret.error, "GetObjectBuffer");
+    }
+    auto reader = OpendalReader(ret.reader);
+    uint64_t buf_size = 16 * 1024;
+    uint64_t buf_index = 0;
+    while (true) {
+        auto read_ret =
+            opendal_reader_read(reader.Get(),
+                                reinterpret_cast<uint8_t*>(buf) + buf_index,
+                                buf_size);
+        buf_index += read_ret.size;
+        if (read_ret.error != nullptr) {
+            THROWOPENDALERROR(read_ret.error, "GetObjectBuffer");
+        }
+        if (read_ret.size == 0) {
+            break;
+        }
+    }
+    if (buf_index != size) {
+        throw SegcoreError(
+            S3Error,
+            fmt::format(
+                "Read size mismatch, target size is {}, actual size is {}",
+                size,
+                buf_index));
+    }
+    return buf_index;
+}
+
+void
+OpenDALChunkManager::Write(const std::string& filepath,
+                           void* buf,
+                           uint64_t size) {
+    auto ret = opendal_operator_write(
+        op_ptr_, filepath.c_str(), {reinterpret_cast<uint8_t*>(buf), size});
+    if (ret != nullptr) {
+        THROWOPENDALERROR(ret, "Write");
+    }
+}
+
+}  // namespace milvus::storage

internal/core/src/storage/OpenDALChunkManager.h

@@ -0,0 +1,106 @@
+#pragma once
+
+#include <map>
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "common/EasyAssert.h"
+#include "storage/ChunkManager.h"
+#include "storage/Types.h"
+#include "opendal.h"
+
+namespace milvus::storage {
+class OpenDALChunkManager : public ChunkManager {
+ public:
+    OpenDALChunkManager() = default;
+    explicit OpenDALChunkManager(const StorageConfig& storage_config);
+
+    OpenDALChunkManager(const OpenDALChunkManager&);
+    OpenDALChunkManager&
+    operator=(const OpenDALChunkManager&);
+
+ public:
+    virtual ~OpenDALChunkManager();
+
+    bool
+    Exist(const std::string& filepath) override;
+
+    uint64_t
+    Size(const std::string& filepath) override;
+
+    uint64_t
+    Read(const std::string& filepath,
+         uint64_t offset,
+         void* buf,
+         uint64_t len) override {
+        throw SegcoreError(NotImplemented,
+                           GetName() + "Read with offset not implement");
+    }
+
+    void
+    Write(const std::string& filepath,
+          uint64_t offset,
+          void* buf,
+          uint64_t len) override {
+        throw SegcoreError(NotImplemented,
+                           GetName() + "Write with offset not implement");
+    }
+
+    uint64_t
+    Read(const std::string& filepath, void* buf, uint64_t len) override;
+
+    void
+    Write(const std::string& filepath, void* buf, uint64_t len) override;
+
+    std::vector<std::string>
+    ListWithPrefix(const std::string& filepath) override;
+
+    void
+    Remove(const std::string& filepath) override;
+
+    std::string
+    GetName() const override {
+        return "OpenDALChunkManager";
+    }
+
+    std::string
+    GetRootPath() const override {
+        return remote_root_path_;
+    }
+
+ private:
+    std::string default_bucket_name_;
+    std::string remote_root_path_;
+
+    const opendal_operator* op_ptr_;
+};
+struct OpendalReader {
+    explicit OpendalReader(opendal_reader* reader) : reader_(reader) {
+    }
+    ~OpendalReader() {
+        opendal_reader_free(reader_);
+    }
+    opendal_reader*
+    Get() {
+        return reader_;
+    }
+
+ private:
+    opendal_reader* reader_;
+};
+struct OpendalLister {
+    explicit OpendalLister(opendal_lister* lister) : lister_(lister) {
+    }
+    ~OpendalLister() {
+        opendal_lister_free(lister_);
+    }
+    opendal_lister*
+    Get() {
+        return lister_;
+    }
+
+ private:
+    opendal_lister* lister_;
+};
+}  // namespace milvus::storage

internal/core/src/storage/RemoteChunkManagerSingleton.h

@@ -20,6 +20,7 @@
 #include <shared_mutex>
 
 #include "storage/Util.h"
+#include "opendal.h"
 
 namespace milvus::storage {
 

internal/core/src/storage/Util.cpp

@@ -22,6 +22,7 @@
 #include "common/EasyAssert.h"
 #include "common/Consts.h"
 #include "fmt/format.h"
+#include "storage/ChunkManager.h"
 #ifdef AZURE_BUILD_DIR
 #include "storage/AzureChunkManager.h"
 #endif
@@ -31,6 +32,7 @@
 #include "storage/ThreadPools.h"
 #include "storage/LocalChunkManager.h"
 #include "storage/MinioChunkManager.h"
+#include "storage/OpenDALChunkManager.h"
 #include "storage/MemFileManagerImpl.h"
 #include "storage/DiskFileManagerImpl.h"
 
@@ -39,7 +41,8 @@ namespace milvus::storage {
 std::map<std::string, ChunkManagerType> ChunkManagerType_Map = {
     {"local", ChunkManagerType::Local},
     {"minio", ChunkManagerType::Minio},
-    {"remote", ChunkManagerType::Remote}};
+    {"remote", ChunkManagerType::Remote},
+    {"opendal", ChunkManagerType::OpenDAL}};
 
 enum class CloudProviderType : int8_t {
     UNKNOWN = 0,
@@ -448,45 +451,6 @@ EncodeAndUploadFieldSlice(ChunkManager* chunk_manager,
     return std::make_pair(std::move(object_key), serialized_index_size);
 }
 
-// /**
-//  * Returns the current resident set size (physical memory use) measured
-//  * in bytes, or zero if the value cannot be determined on this OS.
-//  */
-// size_t
-// getCurrentRSS() {
-// #if defined(_WIN32)
-//     /* Windows -------------------------------------------------- */
-//     PROCESS_MEMORY_COUNTERS info;
-//     GetProcessMemoryInfo(GetCurrentProcess(), &info, sizeof(info));
-//     return (size_t)info.WorkingSetSize;
-
-// #elif defined(__APPLE__) && defined(__MACH__)
-//     /* OSX ------------------------------------------------------ */
-//     struct mach_task_basic_info info;
-//     mach_msg_type_number_t infoCount = MACH_TASK_BASIC_INFO_COUNT;
-//     if (task_info(mach_task_self(), MACH_TASK_BASIC_INFO, (task_info_t)&info, &infoCount) != KERN_SUCCESS)
-//         return (size_t)0L; /* Can't access? */
-//     return (size_t)info.resident_size;
-
-// #elif defined(__linux__) || defined(__linux) || defined(linux) || defined(__gnu_linux__)
-//     /* Linux ---------------------------------------------------- */
-//     long rss = 0L;
-//     FILE* fp = NULL;
-//     if ((fp = fopen("/proc/self/statm", "r")) == NULL)
-//         return (size_t)0L; /* Can't open? */
-//     if (fscanf(fp, "%*s%ld", &rss) != 1) {
-//         fclose(fp);
-//         return (size_t)0L; /* Can't read? */
-//     }
-//     fclose(fp);
-//     return (size_t)rss * (size_t)sysconf(_SC_PAGESIZE);
-
-// #else
-//     /* AIX, BSD, Solaris, and Unknown OS ------------------------ */
-//     return (size_t)0L; /* Unsupported. */
-// #endif
-// }
-
 std::vector<FieldDataPtr>
 GetObjectData(ChunkManager* remote_chunk_manager,
               const std::vector<std::string>& remote_files) {
@@ -608,6 +572,9 @@ CreateChunkManager(const StorageConfig& storage_config) {
                 }
             }
         }
+        case ChunkManagerType::OpenDAL: {
+            return std::make_shared<OpenDALChunkManager>(storage_config);
+        }
 
         default: {
             PanicInfo(ConfigInvalid,

internal/core/src/storage/storage_c.cpp

@@ -66,6 +66,8 @@ InitRemoteChunkManagerSingleton(CStorageConfig c_storage_config) {
             std::string(c_storage_config.cloud_provider);
         storage_config.iam_endpoint =
             std::string(c_storage_config.iam_endpoint);
+        storage_config.cloud_provider =
+            std::string(c_storage_config.cloud_provider);
         storage_config.log_level = std::string(c_storage_config.log_level);
         storage_config.useSSL = c_storage_config.useSSL;
         storage_config.useIAM = c_storage_config.useIAM;

internal/core/thirdparty/CMakeLists.txt

@@ -36,6 +36,7 @@ add_subdirectory(boost_ext)
 add_subdirectory(rocksdb)
 add_subdirectory(rdkafka)
 add_subdirectory(simdjson)
+add_subdirectory(opendal)
 
 if (LINUX)
     add_subdirectory(jemalloc)

internal/core/thirdparty/opendal/CMakeLists.txt

@@ -0,0 +1,31 @@
+#-------------------------------------------------------------------------------
+# Copyright (C) 2019-2020 Zilliz. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+# or implied. See the License for the specific language governing permissions and limitations under the License.
+#-------------------------------------------------------------------------------
+
+# ----------------------------------------------------------------------
+message(STATUS "Building (vendored) opendal from source")
+
+set(OPENDAL_LIB_DIR "${CMAKE_INSTALL_PREFIX}/lib")
+set(OPENDAL_INCLUDE_DIR "${CMAKE_INSTALL_PREFIX}/include")
+set(OPENDAL_NAME "libopendal_c${CMAKE_STATIC_LIBRARY_SUFFIX}")
+
+add_library(opendal STATIC IMPORTED)
+set_target_properties(opendal
+    PROPERTIES
+    IMPORTED_GLOBAL TRUE
+    IMPORTED_LOCATION "${OPENDAL_LIB_DIR}/${OPENDAL_NAME}"
+    INTERFACE_INCLUDE_DIRECTORIES "${CMAKE_INSTALL_PREFIX}/include")
+
+get_target_property(OPENDAL_IMPORTED_LOCATION opendal IMPORTED_LOCATION)
+get_target_property(OPENDAL_INTERFACE_INCLUDE_DIRECTORIES opendal INTERFACE_INCLUDE_DIRECTORIES)
+message("OPENDAL_IMPORTED_LOCATION: ${OPENDAL_IMPORTED_LOCATION}")
+message("OPENDAL_INTERFACE_INCLUDE_DIRECTORIES: ${OPENDAL_INTERFACE_INCLUDE_DIRECTORIES}")

internal/core/unittest/test_sealed.cpp

@@ -1181,7 +1181,7 @@ TEST(Sealed, GetVectorFromChunkCache) {
     auto sc = milvus::storage::StorageConfig{};
     milvus::storage::RemoteChunkManagerSingleton::GetInstance().Init(sc);
     auto mcm = std::make_unique<milvus::storage::MinioChunkManager>(sc);
-    mcm->CreateBucket(sc.bucket_name);
+    // mcm->CreateBucket(sc.bucket_name);
     milvus::storage::ChunkCacheSingleton::GetInstance().Init(mmap_dir,
                                                              "willneed");
 

internal/storage/factory.go

@@ -48,7 +48,7 @@ func (f *ChunkManagerFactory) newChunkManager(ctx context.Context, engine string
 	switch engine {
 	case "local":
 		return NewLocalChunkManager(RootPath(f.config.rootPath)), nil
-	case "minio":
+	case "minio", "opendal":
 		return newMinioChunkManagerWithConfig(ctx, f.config)
 	case "remote":
 		return NewRemoteChunkManager(ctx, f.config)

internal/util/indexcgowrapper/build_index_info.go

@@ -48,18 +48,18 @@ func NewBuildIndexInfo(config *indexpb.StorageConfig) (*BuildIndexInfo, error) {
 	cAccessValue := C.CString(config.SecretAccessKey)
 	cRootPath := C.CString(config.RootPath)
 	cStorageType := C.CString(config.StorageType)
-	cCloudProvider := C.CString(config.CloudProvider)
 	cIamEndPoint := C.CString(config.IAMEndpoint)
 	cRegion := C.CString(config.Region)
+	cCloudProvider := C.CString(config.CloudProvider)
 	defer C.free(unsafe.Pointer(cAddress))
 	defer C.free(unsafe.Pointer(cBucketName))
 	defer C.free(unsafe.Pointer(cAccessKey))
 	defer C.free(unsafe.Pointer(cAccessValue))
 	defer C.free(unsafe.Pointer(cRootPath))
 	defer C.free(unsafe.Pointer(cStorageType))
-	defer C.free(unsafe.Pointer(cCloudProvider))
 	defer C.free(unsafe.Pointer(cIamEndPoint))
 	defer C.free(unsafe.Pointer(cRegion))
+	defer C.free(unsafe.Pointer(cCloudProvider))
 	storageConfig := C.CStorageConfig{
 		address:          cAddress,
 		bucket_name:      cBucketName,
@@ -67,8 +67,8 @@ func NewBuildIndexInfo(config *indexpb.StorageConfig) (*BuildIndexInfo, error) {
 		access_key_value: cAccessValue,
 		root_path:        cRootPath,
 		storage_type:     cStorageType,
-		cloud_provider:   cCloudProvider,
 		iam_endpoint:     cIamEndPoint,
+		cloud_provider:   cCloudProvider,
 		useSSL:           C.bool(config.UseSSL),
 		useIAM:           C.bool(config.UseIAM),
 		region:           cRegion,

internal/util/initcore/init_core.go

@@ -62,8 +62,8 @@ func InitRemoteChunkManager(params *paramtable.ComponentParam) error {
 	cAccessValue := C.CString(params.MinioCfg.SecretAccessKey.GetValue())
 	cRootPath := C.CString(params.MinioCfg.RootPath.GetValue())
 	cStorageType := C.CString(params.CommonCfg.StorageType.GetValue())
-	cCloudProvider := C.CString(params.MinioCfg.CloudProvider.GetValue())
 	cIamEndPoint := C.CString(params.MinioCfg.IAMEndpoint.GetValue())
+	cCloudProvider := C.CString(params.MinioCfg.CloudProvider.GetValue())
 	cLogLevel := C.CString(params.MinioCfg.LogLevel.GetValue())
 	cRegion := C.CString(params.MinioCfg.Region.GetValue())
 	defer C.free(unsafe.Pointer(cAddress))
@@ -72,10 +72,10 @@ func InitRemoteChunkManager(params *paramtable.ComponentParam) error {
 	defer C.free(unsafe.Pointer(cAccessValue))
 	defer C.free(unsafe.Pointer(cRootPath))
 	defer C.free(unsafe.Pointer(cStorageType))
-	defer C.free(unsafe.Pointer(cCloudProvider))
 	defer C.free(unsafe.Pointer(cIamEndPoint))
 	defer C.free(unsafe.Pointer(cLogLevel))
 	defer C.free(unsafe.Pointer(cRegion))
+	defer C.free(unsafe.Pointer(cCloudProvider))
 	storageConfig := C.CStorageConfig{
 		address:          cAddress,
 		bucket_name:      cBucketName,
@@ -83,8 +83,8 @@ func InitRemoteChunkManager(params *paramtable.ComponentParam) error {
 		access_key_value: cAccessValue,
 		root_path:        cRootPath,
 		storage_type:     cStorageType,
-		cloud_provider:   cCloudProvider,
 		iam_endpoint:     cIamEndPoint,
+		cloud_provider:   cCloudProvider,
 		useSSL:           C.bool(params.MinioCfg.UseSSL.GetAsBool()),
 		useIAM:           C.bool(params.MinioCfg.UseIAM.GetAsBool()),
 		log_level:        cLogLevel,

pkg/util/paramtable/component_param.go

@@ -480,8 +480,8 @@ This configuration is only used by querynode and indexnode, it selects CPU instr
 	p.StorageType = ParamItem{
 		Key:          "common.storageType",
 		Version:      "2.0.0",
-		DefaultValue: "minio",
+		DefaultValue: "opendal",
-		Doc:          "please adjust in embedded Milvus: local",
+		Doc:          "please adjust in embedded Milvus: local, available values are [local, minio, remote, opendal]]",
 		Export:       true,
 	}
 	p.StorageType.Init(base.mgr)

scripts/3rdparty_build.sh

@@ -60,3 +60,25 @@ case "${unameOut}" in
 esac
 
 popd
+
+pushd ${ROOT_DIR}/cmake_build/thirdparty
+
+# git clone https://github.com/jiaoew1991/opendal.git opendal
+git clone https://github.com/apache/incubator-opendal.git opendal
+cd opendal
+# git checkout blocking-layer
+if command -v cargo >/dev/null 2>&1; then
+    echo "cargo exists"
+else
+    bash -c "curl https://sh.rustup.rs -sSf | sh -s -- -y" || { echo 'rustup install failed'; exit 1;}
+    source $HOME/.cargo/env
+fi
+pushd bindings/c
+cargo build || { echo 'opendal_c build failed'; exit 1; }
+popd
+mkdir -p ${ROOT_DIR}/internal/core/output/lib
+mkdir -p ${ROOT_DIR}/internal/core/output/include
+cp target/debug/libopendal_c.a ${ROOT_DIR}/internal/core/output/lib/libopendal_c.a
+cp bindings/c/include/opendal.h ${ROOT_DIR}/internal/core/output/include/opendal.h
+
+popd

scripts/run_cpp_codecov.sh

@@ -81,16 +81,17 @@ ${LCOV_CMD} -a ${FILE_INFO_BASE} -a ${FILE_INFO_UT} -o ${FILE_INFO_COMBINE}
 # remove unnecessary info
 ${LCOV_CMD} -r "${FILE_INFO_COMBINE}" -o "${FILE_INFO_OUTPUT}" \
     "/usr/*" \
+    "*/llvm/*" \
     "*/src/pb/*" \
     "*/src/core/bench/*" \
-    "*/faiss_ep-prefix/*" \
-    "*/boost/*" \
     "*/unittest/*" \
-    "*/thirdparty/*"
+    "*/thirdparty/*" \
+    "*/3rdparty_download/*" \
+    "*/.conan/data/*"
 
 # generate html report
-#${LCOV_GEN_CMD} ${FILE_INFO_OUTPUT} --output-directory ${DIR_LCOV_OUTPUT}/
+${LCOV_GEN_CMD} ${FILE_INFO_OUTPUT} --output-directory ${DIR_LCOV_OUTPUT}/
-#echo "Generate cpp code coverage report to ${DIR_LCOV_OUTPUT}"
+echo "Generate cpp code coverage report to ${DIR_LCOV_OUTPUT}"
 
 
 endTime=`date +%s`